Example #1
0
def run_sky_subtraction(night, expid, cameras, basedir, nsky_list, reps=None):
    '''Runs sky subtraction with new sky models and frame files.
    Args:
        night: YYYYMMDD (float)
        expid: exposure id without padding zeros (float)
        cameras: list of cameras corresponding to frame and sky files in given directory, example ['r3', 'z3', 'b3'] (list or array)
        basedir: where to look for frame files
        nsky_list: list with different numbers of fibers frame files and sky files were generated with.
    Options:
        rep: number of different frame/sky files for each camera and nsky combination. Default is 5
    '''
    if reps == None:
        reps = 5

    for cam in cameras:
        framefile = desispec.io.findfile('frame', night, expid, camera=cam)
        header = fitsio.read_header(framefile)
        fiberflatfile = findcalibfile([
            header,
        ], 'FIBERFLAT')
        fiberflat = desispec.io.read_fiberflat(fiberflatfile)
        for N in range(reps):
            for n in nsky_list:
                newframefile = basedir + '/frame-{}-{:08d}-{}-{}.fits'.format(
                    cam, expid, n, N)
                skyfile = basedir + '/sky-{}-{:08d}-{}-{}.fits'.format(
                    cam, expid, n, N)
                sframe = desispec.io.read_frame(newframefile)
                sky = desispec.io.read_sky(skyfile)
                apply_fiberflat(sframe, fiberflat)
                subtract_sky(sframe, sky)

                sframefile = basedir + '/sframe-{}-{:08d}-{}-{}.fits'.format(
                    cam, expid, n, N)
                desispec.io.write_frame(sframefile, sframe)
Example #2
0
def pick_sky_fibers(frame, fiberflat, nsky=100):
    '''
    Updates frame in-place with a new set of sky fibers
    
    The particular cuts here only work for early commissioning observations
    where we randomly point the telescope and hope that most fibers don't
    hit a star or galaxy.
    '''
    tmpframe = deepcopy(frame)
    apply_fiberflat(tmpframe, fiberflat)

    #- select fibers whose flux is between 1 and 90th percentile
    sumflux = np.sum(tmpframe.flux, axis=1)
    sumivar = np.sum(tmpframe.ivar, axis=1)
    fluxlo, fluxhi = np.percentile(sumflux, [5, 85])
    iisky = (fluxlo < sumflux) & (sumflux < fluxhi) & (sumivar > 0) & (sumflux
                                                                       > 0)

    #- Pick a random subset for calling "SKY" fibers
    skysubset = np.random.choice(np.where(iisky)[0], size=nsky, replace=False)

    #- everything is a target unless told otherwise
    frame.fibermap['OBJTYPE'] = 'TGT'

    #- Flag the too-bright or too-faint as "BAD" so we know not to use them
    #- for sky subtraction studies
    frame.fibermap['OBJTYPE'][~iisky] = 'BAD'

    #- Flag the subset as "SKY"
    frame.fibermap['OBJTYPE'][skysubset] = 'SKY'
Example #3
0
def main():

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--infile',
                        type=str,
                        default=None,
                        required=True,
                        help='path of DESI exposure frame fits file')
    parser.add_argument('--fibermap',
                        type=str,
                        default=None,
                        required=True,
                        help='path of DESI exposure frame fits file')
    parser.add_argument('--fiberflat',
                        type=str,
                        default=None,
                        required=True,
                        help='path of DESI fiberflat fits file')
    parser.add_argument('--outfile',
                        type=str,
                        default=None,
                        required=True,
                        help='path of DESI sky fits file')

    args = parser.parse_args()
    log = get_logger()

    log.info("starting")

    # read exposure to load data and get range of spectra
    frame = read_frame(args.infile)
    specmin = frame.header["SPECMIN"]
    specmax = frame.header["SPECMAX"]

    # read fibermap to locate sky fibers
    fibermap = read_fibermap(args.fibermap)
    selection = np.where((fibermap["OBJTYPE"] == "SKY")
                         & (fibermap["FIBER"] >= specmin)
                         & (fibermap["FIBER"] <= specmax))[0]
    if selection.size == 0:
        log.error("no sky fiber in fibermap %s" % args.fibermap)
        sys.exit(12)

    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat to sky fibers
    apply_fiberflat(frame, fiberflat)

    # compute sky model
    skymodel = compute_sky(frame, fibermap)

    # write result
    write_sky(args.outfile, skymodel, frame.header)

    log.info("successfully wrote %s" % args.outfile)
Example #4
0
def main(args) :

    log=get_logger()

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel=read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux,model_wave,model_fibers=read_stdstar_models(args.models)

    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap
    model_fibers = model_fibers%500
    if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'):
        for i in model_fibers:
            log.error("inconsistency with spectrum %d, OBJTYPE='%s' in fibermap"%(i,fibermap["OBJTYPE"][i]))
        sys.exit(12)

    fluxcalib = compute_flux_calibration(frame, model_wave, model_flux)

    # QA
    if (args.qafile is not None):
        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
        # Run
        #import pdb; pdb.set_trace()
        qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s"%args.outfile)
Example #5
0
def main() :

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--infile', type = str, default = None, required=True,
                        help = 'path of DESI exposure frame fits file')
    parser.add_argument('--fiberflat', type = str, default = None,
                        help = 'path of DESI fiberflat fits file')
    parser.add_argument('--sky', type = str, default = None,
                        help = 'path of DESI sky fits file')
    parser.add_argument('--calib', type = str, default = None,
                        help = 'path of DESI calibration fits file')
    parser.add_argument('--outfile', type = str, default = None, required=True,
                        help = 'path of DESI sky fits file')
    # add calibration here when exists

    args = parser.parse_args()
    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    frame = read_frame(args.infile)
    
    if args.fiberflat!=None :
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to sky fibers
        apply_fiberflat(frame, fiberflat)

    if args.sky!=None :
        log.info("subtract sky")
        # read sky
        skymodel=read_sky(args.sky)
        # subtract sky
        subtract_sky(frame, skymodel)

    if args.calib!=None :
        log.info("calibrate")
        # read calibration
        fluxcalib=read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)


    # save output
    write_frame(args.outfile, frame)

    log.info("successfully wrote %s"%args.outfile)
Example #6
0
    def run_pa(self,input_frame,fiberflat,outputfile):
        from desispec.fiberflat import apply_fiberflat
        from desispec.sky import compute_sky
        from desispec.io.sky import write_sky

        #- First apply fiberflat to sky fibers
        apply_fiberflat(input_frame,fiberflat)

        #- calculate the model
        skymodel=compute_sky(input_frame)
        write_sky(outputfile,skymodel,input_frame.meta)
        log.info("Sky Model file wrtten. Exiting pipeline for this configuration")
        sys.exit(0)
Example #7
0
    def run_pa(self,input_frame,fiberflat,outputfile):
        from desispec.fiberflat import apply_fiberflat
        from desispec.sky import compute_sky
        from desispec.io.sky import write_sky

        #- First apply fiberflat to sky fibers
        apply_fiberflat(input_frame,fiberflat)

        #- calculate the model
        skymodel=compute_sky(input_frame)
        write_sky(outputfile,skymodel,input_frame.meta)
        log.info("Sky Model file wrtten. Exiting pipeline for this configuration")
        sys.exit(0)
Example #8
0
def main(args) :

    log=get_logger()

    log.info("starting")

    # read exposure to load data and get range of spectra
    frame = read_frame(args.infile)
    specmin, specmax = np.min(frame.fibers), np.max(frame.fibers)

    if args.cosmics_nsig>0 : # Reject cosmics
        reject_cosmic_rays_1d(frame,args.cosmics_nsig)

    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat to sky fibers
    apply_fiberflat(frame, fiberflat)

    # compute sky model
    skymodel = compute_sky(frame,add_variance=(not args.no_extra_variance),\
                           angular_variation_deg=args.angular_variation_deg,\
                           chromatic_variation_deg=args.chromatic_variation_deg,\
                           adjust_wavelength=args.adjust_wavelength,\
                           adjust_lsf=args.adjust_lsf)

    # QA
    if (args.qafile is not None) or (args.qafig is not None):
        log.info("performing skysub QA")
        # Load
        qaframe = load_qa_frame(args.qafile, frame_meta=frame.meta, flavor=frame.meta['FLAVOR'])
        # Run
        qaframe.run_qa('SKYSUB', (frame, skymodel))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe)

    # record inputs
    frame.meta['IN_FRAME'] = shorten_filename(args.infile)
    frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)

    # write result
    write_sky(args.outfile, skymodel, frame.meta)
    log.info("successfully wrote %s"%args.outfile)
Example #9
0
def main() :

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--infile', type = str, default = None, required=True,
                        help = 'path of DESI exposure frame fits file')
    parser.add_argument('--fibermap', type = str, default = None, required=True,
                        help = 'path of DESI exposure frame fits file')
    parser.add_argument('--fiberflat', type = str, default = None, required=True,
                        help = 'path of DESI fiberflat fits file')
    parser.add_argument('--outfile', type = str, default = None, required=True,
                        help = 'path of DESI sky fits file')


    args = parser.parse_args()
    log=get_logger()

    log.info("starting")

    # read exposure to load data and get range of spectra
    frame = read_frame(args.infile)
    specmin=frame.header["SPECMIN"]
    specmax=frame.header["SPECMAX"]

    # read fibermap to locate sky fibers
    fibermap = read_fibermap(args.fibermap)
    selection=np.where((fibermap["OBJTYPE"]=="SKY")&(fibermap["FIBER"]>=specmin)&(fibermap["FIBER"]<=specmax))[0]
    if selection.size == 0 :
        log.error("no sky fiber in fibermap %s"%args.fibermap)
        sys.exit(12)

    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat to sky fibers
    apply_fiberflat(frame, fiberflat)

    # compute sky model
    skymodel = compute_sky(frame, fibermap)

    # write result
    write_sky(args.outfile, skymodel, frame.header)

    log.info("successfully wrote %s"%args.outfile)
Example #10
0
    def test_apply_fiberflat(self):
        '''test apply_fiberflat interface and changes to flux and mask'''
        wave = np.arange(5000, 5050)
        nwave = len(wave)
        nspec = 3
        flux = np.random.uniform(size=(nspec, nwave))
        ivar = np.ones_like(flux)
        frame = Frame(wave, flux, ivar, spectrograph=0)
        
        fiberflat = np.ones_like(flux)
        ffivar = 2*np.ones_like(flux)
        ffmask = np.zeros_like(flux)
        fiberflat[0] *= 0.8
        fiberflat[1] *= 1.2
        fiberflat[2, 0:10] = 0  #- bad fiberflat
        ffivar[2, 10:20] = 0    #- bad fiberflat
        ffmask[2, 20:30] = 1    #- bad fiberflat
        
        ff = FiberFlat(wave, fiberflat, ffivar)

        origframe = copy.deepcopy(frame)
        apply_fiberflat(frame, ff)

        #- was fiberflat applied?
        self.assertTrue(np.all(frame.flux[0] == origframe.flux[0]/0.8))
        self.assertTrue(np.all(frame.flux[1] == origframe.flux[1]/1.2))
        self.assertTrue(np.all(frame.flux[2] == origframe.flux[2]))

        #- did mask get set?
        ii = (ff.fiberflat == 0)
        self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))
        ii = (ff.ivar == 0)
        self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))
        ii = (ff.mask != 0)
        self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))

        #- Should fail if frame and ff don't have a common wavelength grid
        frame.wave = frame.wave + 0.1
        with self.assertRaises(ValueError):
            apply_fiberflat(frame, ff)
Example #11
0
    def test_apply_fiberflat(self):
        '''test apply_fiberflat interface and changes to flux and mask'''
        wave = np.arange(5000, 5050)
        nwave = len(wave)
        nspec = 3
        flux = np.random.uniform(size=(nspec, nwave))
        ivar = np.ones_like(flux)
        frame = Frame(wave, flux, ivar, spectrograph=0)

        fiberflat = np.ones_like(flux)
        ffivar = 2 * np.ones_like(flux)
        ffmask = np.zeros_like(flux)
        fiberflat[0] *= 0.8
        fiberflat[1] *= 1.2
        fiberflat[2, 0:10] = 0  #- bad fiberflat
        ffivar[2, 10:20] = 0  #- bad fiberflat
        ffmask[2, 20:30] = 1  #- bad fiberflat

        ff = FiberFlat(wave, fiberflat, ffivar)

        origframe = copy.deepcopy(frame)
        apply_fiberflat(frame, ff)

        #- was fiberflat applied?
        self.assertTrue(np.all(frame.flux[0] == origframe.flux[0] / 0.8))
        self.assertTrue(np.all(frame.flux[1] == origframe.flux[1] / 1.2))
        self.assertTrue(np.all(frame.flux[2] == origframe.flux[2]))

        #- did mask get set?
        ii = (ff.fiberflat == 0)
        self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))
        ii = (ff.ivar == 0)
        self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))
        ii = (ff.mask != 0)
        self.assertTrue(np.all((frame.mask[ii] & specmask.BADFIBERFLAT) != 0))

        #- Should fail if frame and ff don't have a common wavelength grid
        frame.wave = frame.wave + 0.1
        with self.assertRaises(ValueError):
            apply_fiberflat(frame, ff)
Example #12
0
    def run_pa(self,input_frame,fiberflat,dump=False,dumpfile=None): 
     
        from desispec.quicklook.quickfiberflat import apply_fiberflat 
        fframe=apply_fiberflat(input_frame,fiberflat)

        if dump and dumpfile is not None:
            from desispec import io
            night = fframe.meta['NIGHT']
            expid = fframe.meta['EXPID']
            io.write_frame(dumpfile, fframe)
            log.info("Wrote intermediate file %s after %s"%(dumpfile,self.name))

        return fframe
Example #13
0
    def run_pa(self,input_frame,fiberflat,dump=False,dumpfile=None): 
     
        from desispec.quicklook.quickfiberflat import apply_fiberflat 
        fframe=apply_fiberflat(input_frame,fiberflat)

        if dump and dumpfile is not None:
            from desispec import io
            night = fframe.meta['NIGHT']
            expid = fframe.meta['EXPID']
            io.write_frame(dumpfile, fframe)
            log.info("Wrote intermediate file %s after %s"%(dumpfile,self.name))

        return fframe
Example #14
0
def main(args):

    log = get_logger()

    log.info("starting")

    # read exposure to load data and get range of spectra
    frame = read_frame(args.infile)
    specmin, specmax = np.min(frame.fibers), np.max(frame.fibers)

    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat to sky fibers
    apply_fiberflat(frame, fiberflat)

    # compute sky model
    skymodel = compute_sky(frame)

    # QA
    if (args.qafile is not None) or (args.qafig is not None):
        log.info("performing skysub QA")
        # Load
        qaframe = load_qa_frame(args.qafile,
                                frame,
                                flavor=frame.meta['FLAVOR'])
        # Run
        qaframe.run_qa('SKYSUB', (frame, skymodel))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe)

    # write result
    write_sky(args.outfile, skymodel, frame.meta)
    log.info("successfully wrote %s" % args.outfile)
Example #15
0
def main(args):

    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    frame = read_frame(args.infile)

    if args.fiberflat!=None :
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to sky fibers
        apply_fiberflat(frame, fiberflat)

    if args.sky!=None :
        log.info("subtract sky")
        # read sky
        skymodel=read_sky(args.sky)
        # subtract sky
        subtract_sky(frame, skymodel)

    if args.calib!=None :
        log.info("calibrate")
        # read calibration
        fluxcalib=read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)


    # save output
    write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 A)')

    log.info("successfully wrote %s"%args.outfile)
Example #16
0
def main(args) :

    log=get_logger()

    log.info("starting")

    # read exposure to load data and get range of spectra
    frame = read_frame(args.infile)
    specmin, specmax = np.min(frame.fibers), np.max(frame.fibers)

    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat to sky fibers
    apply_fiberflat(frame, fiberflat)

    # compute sky model
    skymodel = compute_sky(frame)

    # QA
    if (args.qafile is not None) or (args.qafig is not None):
        log.info("performing skysub QA")
        # Load
        qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
        # Run
        qaframe.run_qa('SKYSUB', (frame, skymodel))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_skyres(args.qafig, frame, skymodel, qaframe)

    # write result
    write_sky(args.outfile, skymodel, frame.meta)
    log.info("successfully wrote %s"%args.outfile)
Example #17
0
    def test_apply_fiberflat_ivar(self):
        '''test error propagation in apply_fiberflat'''
        wave = np.arange(5000, 5010)
        nwave = len(wave)
        nspec = 3
        flux = np.random.uniform(0.9, 1.0, size=(nspec, nwave))
        ivar = np.ones_like(flux)
        origframe = Frame(wave, flux, ivar, spectrograph=0)

        fiberflat = np.ones_like(flux)
        ffmask = np.zeros_like(flux)
        fiberflat[0] *= 0.5
        fiberflat[1] *= 1.5

        #- ff with essentially no error
        ffivar = 1e20 * np.ones_like(flux)
        ff = FiberFlat(wave, fiberflat, ffivar)
        frame = copy.deepcopy(origframe)
        apply_fiberflat(frame, ff)
        self.assertTrue(np.allclose(frame.ivar, fiberflat**2))

        #- ff with large error
        ffivar = np.ones_like(flux)
        ff = FiberFlat(wave, fiberflat, ffivar)
        frame = copy.deepcopy(origframe)
        apply_fiberflat(frame, ff)

        #- c = a/b
        #- (sigma_c/c)^2 = (sigma_a/a)^2 + (sigma_b/b)^2
        var = frame.flux**2 * (1.0/(origframe.ivar * origframe.flux**2) + \
                               1.0/(ff.ivar * ff.fiberflat**2))
        self.assertTrue(np.allclose(frame.ivar, 1 / var))

        #- ff.ivar=0 should result in frame.ivar=0, even if ff.fiberflat=0 too
        ffivar = np.ones_like(flux)
        ffivar[0, 0:5] = 0.0
        fiberflat[0, 0:5] = 0.0
        ff = FiberFlat(wave, fiberflat, ffivar)
        frame = copy.deepcopy(origframe)
        apply_fiberflat(frame, ff)

        self.assertTrue(np.all(frame.ivar[0, 0:5] == 0.0))
Example #18
0
    def test_apply_fiberflat_ivar(self):
        '''test error propagation in apply_fiberflat'''
        wave = np.arange(5000, 5010)
        nwave = len(wave)
        nspec = 3
        flux = np.random.uniform(0.9, 1.0, size=(nspec, nwave))
        ivar = np.ones_like(flux)
        origframe = Frame(wave, flux, ivar, spectrograph=0)

        fiberflat = np.ones_like(flux)
        ffmask = np.zeros_like(flux)
        fiberflat[0] *= 0.5
        fiberflat[1] *= 1.5

        #- ff with essentially no error
        ffivar = 1e20 * np.ones_like(flux)
        ff = FiberFlat(wave, fiberflat, ffivar)
        frame = copy.deepcopy(origframe)
        apply_fiberflat(frame, ff)
        self.assertTrue(np.allclose(frame.ivar, fiberflat**2))

        #- ff with large error
        ffivar = np.ones_like(flux)
        ff = FiberFlat(wave, fiberflat, ffivar)
        frame = copy.deepcopy(origframe)
        apply_fiberflat(frame, ff)
        
        #- c = a/b
        #- (sigma_c/c)^2 = (sigma_a/a)^2 + (sigma_b/b)^2
        var = frame.flux**2 * (1.0/(origframe.ivar * origframe.flux**2) + \
                               1.0/(ff.ivar * ff.fiberflat**2))
        self.assertTrue(np.allclose(frame.ivar, 1/var))

        #- ff.ivar=0 should result in frame.ivar=0, even if ff.fiberflat=0 too
        ffivar = np.ones_like(flux)
        ffivar[0, 0:5] = 0.0
        fiberflat[0, 0:5] = 0.0
        ff = FiberFlat(wave, fiberflat, ffivar)
        frame = copy.deepcopy(origframe)
        apply_fiberflat(frame, ff)

        self.assertTrue(np.all(frame.ivar[0, 0:5] == 0.0))
Example #19
0
def main(args):

    log = get_logger()

    cmd = [
        'desi_compute_fluxcalibration',
    ]
    for key, value in args.__dict__.items():
        if value is not None:
            cmd += ['--' + key, str(value)]
    cmd = ' '.join(cmd)
    log.info(cmd)

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    # Set fibermask flagged spectra to have 0 flux and variance
    frame = get_fiberbitmasked_frame(frame,
                                     bitmask='flux',
                                     ivar_framemask=True)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel = read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
        args.models)

    ok = np.ones(len(model_metadata), dtype=bool)

    if args.chi2cut > 0:
        log.info("apply cut CHI2DOF<{}".format(args.chi2cut))
        good = (model_metadata["CHI2DOF"] < args.chi2cut)
        bad = ~good
        ok &= good
        if np.any(bad):
            log.info(" discard {} stars with CHI2DOF= {}".format(
                np.sum(bad), list(model_metadata["CHI2DOF"][bad])))

    legacy_filters = ('G-R', 'R-Z')
    gaia_filters = ('GAIA-BP-RP', 'GAIA-G-RP')
    model_column_list = model_metadata.columns.names
    if args.color is None:
        if 'MODEL_G-R' in model_column_list:
            color = 'G-R'
        elif 'MODEL_GAIA-BP-RP' in model_column_list:
            log.info('Using Gaia filters')
            color = 'GAIA-BP-RP'
        else:
            log.error(
                "Can't find either G-R or BP-RP color in the model file.")
            sys.exit(15)
    else:
        if args.color not in legacy_filters and args.color not in gaia_filters:
            log.error(
                'Color name {} is not allowed, must be one of {} {}'.format(
                    args.color, legacy_filters, gaia_filters))
            sys.exit(14)
        color = args.color
        if color not in model_column_list:
            # This should't happen
            log.error(
                'The color {} was not computed in the models'.format(color))
            sys.exit(16)

    if args.delta_color_cut > 0:
        log.info("apply cut |delta color|<{}".format(args.delta_color_cut))
        good = (np.abs(model_metadata["MODEL_" + color] -
                       model_metadata["DATA_" + color]) < args.delta_color_cut)
        bad = ok & (~good)
        ok &= good
        if np.any(bad):
            vals = model_metadata["MODEL_" +
                                  color][bad] - model_metadata["DATA_" +
                                                               color][bad]
            log.info(" discard {} stars with dcolor= {}".format(
                np.sum(bad), list(vals)))

    if args.min_color is not None:
        log.info("apply cut DATA_{}>{}".format(color, args.min_color))
        good = (model_metadata["DATA_{}".format(color)] > args.min_color)
        bad = ok & (~good)
        ok &= good
        if np.any(bad):
            vals = model_metadata["DATA_{}".format(color)][bad]
            log.info(" discard {} stars with {}= {}".format(
                np.sum(bad), color, list(vals)))

    if args.chi2cut_nsig > 0:
        # automatically reject stars that ar chi2 outliers
        mchi2 = np.median(model_metadata["CHI2DOF"])
        rmschi2 = np.std(model_metadata["CHI2DOF"])
        maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
        log.info("apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
            maxchi2, args.chi2cut_nsig))
        good = (model_metadata["CHI2DOF"] <= maxchi2)
        bad = ok & (~good)
        ok &= good
        if np.any(bad):
            log.info(" discard {} stars with CHI2DOF={}".format(
                np.sum(bad), list(model_metadata["CHI2DOF"][bad])))

    ok = np.where(ok)[0]
    if ok.size == 0:
        log.error("selection cuts discarded all stars")
        sys.exit(12)
    nstars = model_flux.shape[0]
    nbad = nstars - ok.size
    if nbad > 0:
        log.warning("discarding %d star(s) out of %d because of cuts" %
                    (nbad, nstars))
        model_flux = model_flux[ok]
        model_fibers = model_fibers[ok]
        model_metadata = model_metadata[:][ok]

    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap

    ## check whether star fibers from args.models are consistent with fibers from fibermap
    ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
    fibermap_std_indices = np.where(isStdStar(fibermap))[0]
    if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
        target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
        colname = target_colnames[0]
        for i in model_fibers % 500:
            log.error(
                "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
                .format(i, fibermap["OBJTYPE"][i], colname,
                        fibermap[colname][i]))
        sys.exit(12)

    # Make sure the fibers of interest aren't entirely masked.
    if np.sum(
            np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
            frame.nwave) == len(model_fibers):
        log.warning('All standard-star spectra are masked!')
        return

    fluxcalib = compute_flux_calibration(
        frame,
        model_wave,
        model_flux,
        model_fibers % 500,
        highest_throughput_nstars=args.highest_throughput,
        exposure_seeing_fwhm=args.seeing_fwhm)

    # QA
    if (args.qafile is not None):

        from desispec.io import write_qa_frame
        from desispec.io.qa import load_qa_frame
        from desispec.qa import qa_plots

        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile,
                                frame_meta=frame.meta,
                                flavor=frame.meta['FLAVOR'])
        # Run
        #import pdb; pdb.set_trace()
        qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)

    # record inputs
    frame.meta['IN_FRAME'] = shorten_filename(args.infile)
    frame.meta['IN_SKY'] = shorten_filename(args.sky)
    frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)
    frame.meta['STDMODEL'] = shorten_filename(args.models)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s" % args.outfile)
Example #20
0
def main(args):

    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    frame = read_frame(args.infile)

    #- Raw scores already added in extraction, but just in case they weren't
    #- it is harmless to rerun to make sure we have them.
    compute_and_append_frame_scores(frame,suffix="RAW")

    if args.cosmics_nsig>0 and args.sky==None : # Reject cosmics (otherwise do it after sky subtraction)
        log.info("cosmics ray 1D rejection")
        reject_cosmic_rays_1d(frame,args.cosmics_nsig)

    if args.fiberflat!=None :
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to all fibers
        apply_fiberflat(frame, fiberflat)
        compute_and_append_frame_scores(frame,suffix="FFLAT")

    if args.sky!=None :

        # read sky
        skymodel=read_sky(args.sky)

        if args.cosmics_nsig>0 :

            # use a copy the frame (not elegant but robust)
            copied_frame = copy.deepcopy(frame)
            
            # first subtract sky without throughput correction
            subtract_sky(copied_frame, skymodel, apply_throughput_correction = False)

            # then find cosmics
            log.info("cosmics ray 1D rejection after sky subtraction")
            reject_cosmic_rays_1d(copied_frame,args.cosmics_nsig)

            # copy mask
            frame.mask = copied_frame.mask
            
            # and (re-)subtract sky, but just the correction term
            subtract_sky(frame, skymodel, apply_throughput_correction = (not args.no_sky_throughput_correction) )

        else :
            # subtract sky
            subtract_sky(frame, skymodel, apply_throughput_correction = (not args.no_sky_throughput_correction) )

        compute_and_append_frame_scores(frame,suffix="SKYSUB")

    if args.calib!=None :
        log.info("calibrate")
        # read calibration
        fluxcalib=read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)

        # Ensure that ivars are set to 0 for all values if any designated
        # fibermask bit is set. Also flips a bits for each frame.mask value using specmask.BADFIBER
        frame = get_fiberbitmasked_frame(frame,bitmask="flux",ivar_framemask=True)
        compute_and_append_frame_scores(frame,suffix="CALIB")


    # save output
    write_frame(args.outfile, frame, units='10**-17 erg/(s cm2 Angstrom)')
    log.info("successfully wrote %s"%args.outfile)
Example #21
0
def qaframe_from_frame(frame_file,
                       specprod_dir=None,
                       make_plots=False,
                       qaprod_dir=None,
                       output_dir=None,
                       clobber=True):
    """  Generate a qaframe object from an input frame_file name (and night)

    Write QA to disk
    Will also make plots if directed
    Args:
        frame_file: str
        specprod_dir: str, optional
        qa_dir: str, optional -- Location of QA
        make_plots: bool, optional
        output_dir: str, optional

    Returns:

    """
    import glob
    import os

    from desispec.io import read_frame
    from desispec.io import meta
    from desispec.io.qa import load_qa_frame, write_qa_frame
    from desispec.io.qa import qafile_from_framefile
    from desispec.io.frame import search_for_framefile
    from desispec.io.fiberflat import read_fiberflat
    from desispec.fiberflat import apply_fiberflat
    from desispec.qa import qa_plots
    from desispec.io.sky import read_sky
    from desispec.io.fluxcalibration import read_flux_calibration

    if '/' in frame_file:  # If present, assume full path is used here
        pass
    else:  # Find the frame file in the desispec hierarchy?
        frame_file = search_for_framefile(frame_file)

    # Load frame
    frame = read_frame(frame_file)
    frame_meta = frame.meta
    night = frame_meta['NIGHT'].strip()
    camera = frame_meta['CAMERA'].strip()
    expid = frame_meta['EXPID']
    spectro = int(frame_meta['CAMERA'][-1])

    # Filename
    qafile, qatype = qafile_from_framefile(frame_file,
                                           qaprod_dir=qaprod_dir,
                                           output_dir=output_dir)
    qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
    # Flat QA
    if frame_meta['FLAVOR'] in ['flat']:
        fiberflat_fil = meta.findfile('fiberflat',
                                      night=night,
                                      camera=camera,
                                      expid=expid,
                                      specprod_dir=specprod_dir)
        try:  # Backwards compatibility
            fiberflat = read_fiberflat(fiberflat_fil)
        except FileNotFoundError:
            fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
            path, basen = os.path.split(fiberflat_fil)
            path, _ = os.path.split(path)
            fiberflat_fil = os.path.join(path, basen)
            fiberflat = read_fiberflat(fiberflat_fil)
        qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber)
        if make_plots:
            # Do it
            qafig = meta.findfile('qa_flat_fig',
                                  night=night,
                                  camera=camera,
                                  expid=expid,
                                  specprod_dir=specprod_dir,
                                  outdir=output_dir)
            qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
    # SkySub QA
    if qatype == 'qa_data':
        sky_fil = meta.findfile('sky',
                                night=night,
                                camera=camera,
                                expid=expid,
                                specprod_dir=specprod_dir)

        fiberflat_fil = meta.findfile('fiberflatnight',
                                      night=night,
                                      camera=camera)
        if not os.path.exists(fiberflat_fil):
            # Backwards compatibility (for now)
            dummy_fiberflat_fil = meta.findfile(
                'fiberflat',
                night=night,
                camera=camera,
                expid=expid,
                specprod_dir=specprod_dir)  # This is dummy
            path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
            fiberflat_files = glob.glob(
                os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
            if len(fiberflat_files) == 0:
                path = path.replace('exposures', 'calib2d')
                path, _ = os.path.split(path)  # Remove night
                fiberflat_files = glob.glob(
                    os.path.join(path, 'fiberflat-' + camera + '*.fits'))

            # Sort and take the first (same as old pipeline)
            fiberflat_files.sort()
            fiberflat_fil = fiberflat_files[0]

        fiberflat = read_fiberflat(fiberflat_fil)
        apply_fiberflat(frame, fiberflat)
        # Load sky model and run
        try:
            skymodel = read_sky(sky_fil)
        except FileNotFoundError:
            warnings.warn(
                "Sky file {:s} not found.  Skipping..".format(sky_fil))
        else:
            qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber)
            if make_plots:
                qafig = meta.findfile('qa_sky_fig',
                                      night=night,
                                      camera=camera,
                                      expid=expid,
                                      specprod_dir=specprod_dir,
                                      outdir=output_dir)
                qafig2 = meta.findfile('qa_skychi_fig',
                                       night=night,
                                       camera=camera,
                                       expid=expid,
                                       specprod_dir=specprod_dir,
                                       outdir=output_dir)
                qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
                #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)
    # FluxCalib QA
    if qatype == 'qa_data':
        # Standard stars
        stdstar_fil = meta.findfile('stdstars',
                                    night=night,
                                    camera=camera,
                                    expid=expid,
                                    specprod_dir=specprod_dir,
                                    spectrograph=spectro)
        # try:
        #    model_tuple=read_stdstar_models(stdstar_fil)
        # except FileNotFoundError:
        #    warnings.warn("Standard star file {:s} not found.  Skipping..".format(stdstar_fil))
        # else:
        flux_fil = meta.findfile('calib',
                                 night=night,
                                 camera=camera,
                                 expid=expid,
                                 specprod_dir=specprod_dir)
        try:
            fluxcalib = read_flux_calibration(flux_fil)
        except FileNotFoundError:
            warnings.warn(
                "Flux file {:s} not found.  Skipping..".format(flux_fil))
        else:
            qaframe.run_qa(
                'FLUXCALIB',
                (frame, fluxcalib))  # , model_tuple))#, indiv_stars))
            if make_plots:
                qafig = meta.findfile('qa_flux_fig',
                                      night=night,
                                      camera=camera,
                                      expid=expid,
                                      specprod_dir=specprod_dir,
                                      outdir=output_dir)
                qa_plots.frame_fluxcalib(qafig, qaframe, frame,
                                         fluxcalib)  # , model_tuple)
    # Write
    write_qa_frame(qafile, qaframe, verbose=True)
    return qaframe
Example #22
0
def main():

    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--infile',
                        type=str,
                        default=None,
                        required=True,
                        help='path of DESI exposure frame fits file')
    parser.add_argument('--fiberflat',
                        type=str,
                        default=None,
                        help='path of DESI fiberflat fits file')
    parser.add_argument('--sky',
                        type=str,
                        default=None,
                        help='path of DESI sky fits file')
    parser.add_argument('--calib',
                        type=str,
                        default=None,
                        help='path of DESI calibration fits file')
    parser.add_argument('--outfile',
                        type=str,
                        default=None,
                        required=True,
                        help='path of DESI sky fits file')
    # add calibration here when exists

    args = parser.parse_args()
    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is
                                                            None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    frame = read_frame(args.infile)

    if args.fiberflat != None:
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to sky fibers
        apply_fiberflat(frame, fiberflat)

    if args.sky != None:
        log.info("subtract sky")
        # read sky
        skymodel = read_sky(args.sky)
        # subtract sky
        subtract_sky(frame, skymodel)

    if args.calib != None:
        log.info("calibrate")
        # read calibration
        fluxcalib = read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)

    # save output
    write_frame(args.outfile, frame)

    log.info("successfully wrote %s" % args.outfile)
Example #23
0
def main(args):

    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is
                                                            None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    frame = read_frame(args.infile)

    #- Raw scores already added in extraction, but just in case they weren't
    #- it is harmless to rerun to make sure we have them.
    compute_and_append_frame_scores(frame, suffix="RAW")

    if args.cosmics_nsig > 0 and args.sky == None:  # Reject cosmics (otherwise do it after sky subtraction)
        log.info("cosmics ray 1D rejection")
        reject_cosmic_rays_1d(frame, args.cosmics_nsig)

    if args.fiberflat != None:
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to all fibers
        apply_fiberflat(frame, fiberflat)
        compute_and_append_frame_scores(frame, suffix="FFLAT")

    if args.sky != None:

        # read sky
        skymodel = read_sky(args.sky)

        if args.cosmics_nsig > 0:

            # first subtract sky without throughput correction
            subtract_sky(frame, skymodel, throughput_correction=False)

            # then find cosmics
            log.info("cosmics ray 1D rejection after sky subtraction")
            reject_cosmic_rays_1d(frame, args.cosmics_nsig)

            if args.sky_throughput_correction:
                # and (re-)subtract sky, but just the correction term
                subtract_sky(frame,
                             skymodel,
                             throughput_correction=True,
                             default_throughput_correction=0.)

        else:
            # subtract sky
            subtract_sky(frame,
                         skymodel,
                         throughput_correction=args.sky_throughput_correction)

        compute_and_append_frame_scores(frame, suffix="SKYSUB")

    if args.calib != None:
        log.info("calibrate")
        # read calibration
        fluxcalib = read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)
        compute_and_append_frame_scores(frame, suffix="CALIB")

    # save output
    write_frame(args.outfile, frame, units='1e-17 erg/(s cm2 Angstrom)')

    log.info("successfully wrote %s" % args.outfile)
Example #24
0
def qaframe_from_frame(frame_file,
                       specprod_dir=None,
                       make_plots=False,
                       qaprod_dir=None,
                       output_dir=None,
                       clobber=True):
    """  Generate a qaframe object from an input frame_file name (and night)

    Write QA to disk
    Will also make plots if directed
    Args:
        frame_file: str
        specprod_dir: str, optional
        qa_dir: str, optional -- Location of QA
        make_plots: bool, optional
        output_dir: str, optional

    Returns:

    """
    import glob
    import os

    from desispec.io import read_frame
    from desispec.io import meta
    from desispec.io.qa import load_qa_frame, write_qa_frame
    from desispec.io.qa import qafile_from_framefile
    from desispec.io.frame import search_for_framefile
    from desispec.io.fiberflat import read_fiberflat
    from desispec.fiberflat import apply_fiberflat
    from desispec.qa import qa_plots
    from desispec.io.sky import read_sky
    from desispec.io.fluxcalibration import read_flux_calibration
    from desispec.qa import qa_plots_ql

    if '/' in frame_file:  # If present, assume full path is used here
        pass
    else:  # Find the frame file in the desispec hierarchy?
        frame_file = search_for_framefile(frame_file)

    # Load frame
    frame = read_frame(frame_file)
    frame_meta = frame.meta
    night = frame_meta['NIGHT'].strip()
    camera = frame_meta['CAMERA'].strip()
    expid = frame_meta['EXPID']
    spectro = int(frame_meta['CAMERA'][-1])

    # Filename
    qafile, qatype = qafile_from_framefile(frame_file,
                                           qaprod_dir=qaprod_dir,
                                           output_dir=output_dir)
    if os.path.isfile(qafile) and (not clobber):
        write = False
    else:
        write = True
    qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
    # Flat QA
    if frame_meta['FLAVOR'] in ['flat']:
        fiberflat_fil = meta.findfile('fiberflat',
                                      night=night,
                                      camera=camera,
                                      expid=expid,
                                      specprod_dir=specprod_dir)
        try:  # Backwards compatibility
            fiberflat = read_fiberflat(fiberflat_fil)
        except FileNotFoundError:
            fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
            path, basen = os.path.split(fiberflat_fil)
            path, _ = os.path.split(path)
            fiberflat_fil = os.path.join(path, basen)
            fiberflat = read_fiberflat(fiberflat_fil)
        if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
            write = True
        if make_plots:
            # Do it
            qafig = meta.findfile('qa_flat_fig',
                                  night=night,
                                  camera=camera,
                                  expid=expid,
                                  qaprod_dir=qaprod_dir,
                                  specprod_dir=specprod_dir,
                                  outdir=output_dir)
            if (not os.path.isfile(qafig)) or clobber:
                qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
    # SkySub QA
    if qatype == 'qa_data':
        sky_fil = meta.findfile('sky',
                                night=night,
                                camera=camera,
                                expid=expid,
                                specprod_dir=specprod_dir)

        fiberflat_fil = meta.findfile('fiberflatnight',
                                      night=night,
                                      camera=camera)
        if not os.path.exists(fiberflat_fil):
            # Backwards compatibility (for now)
            dummy_fiberflat_fil = meta.findfile(
                'fiberflat',
                night=night,
                camera=camera,
                expid=expid,
                specprod_dir=specprod_dir)  # This is dummy
            path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
            fiberflat_files = glob.glob(
                os.path.join(path, '*', 'fiberflat-' + camera + '*.fits'))
            if len(fiberflat_files) == 0:
                path = path.replace('exposures', 'calib2d')
                path, _ = os.path.split(path)  # Remove night
                fiberflat_files = glob.glob(
                    os.path.join(path, 'fiberflat-' + camera + '*.fits'))

            # Sort and take the first (same as old pipeline)
            fiberflat_files.sort()
            fiberflat_fil = fiberflat_files[0]

        fiberflat = read_fiberflat(fiberflat_fil)
        apply_fiberflat(frame, fiberflat)
        # Load sky model and run
        try:
            skymodel = read_sky(sky_fil)
        except FileNotFoundError:
            warnings.warn(
                "Sky file {:s} not found.  Skipping..".format(sky_fil))
        else:
            if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
                write = True
            if make_plots:
                qafig = meta.findfile('qa_sky_fig',
                                      night=night,
                                      camera=camera,
                                      expid=expid,
                                      specprod_dir=specprod_dir,
                                      outdir=output_dir,
                                      qaprod_dir=qaprod_dir)
                qafig2 = meta.findfile('qa_skychi_fig',
                                       night=night,
                                       camera=camera,
                                       expid=expid,
                                       specprod_dir=specprod_dir,
                                       outdir=output_dir,
                                       qaprod_dir=qaprod_dir)
                if (not os.path.isfile(qafig)) or clobber:
                    qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
                #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)

    # S/N QA on cframe
    if qatype == 'qa_data':
        # cframe
        cframe_file = frame_file.replace('frame-', 'cframe-')
        cframe = read_frame(cframe_file)
        if qaframe.run_qa('S2N', (cframe, ), clobber=clobber):
            write = True
        # Figure?
        if make_plots:
            s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
            qafig = meta.findfile('qa_s2n_fig',
                                  night=night,
                                  camera=camera,
                                  expid=expid,
                                  specprod_dir=specprod_dir,
                                  outdir=output_dir,
                                  qaprod_dir=qaprod_dir)
            #badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist()
            #sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE')
            coeff = s2n_dict['METRICS']['FITCOEFF_TGT']  #[sci_idx]
            # Add an item or two for the QL method
            s2n_dict['CAMERA'] = camera
            s2n_dict['EXPID'] = expid
            s2n_dict['PANAME'] = 'SNRFit'
            s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA']
            s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC']
            objlist = s2n_dict['METRICS']['OBJLIST']
            # Deal with YAML list instead of ndarray
            s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(
                s2n_dict['METRICS']['MEDIAN_SNR'])
            # Generate
            if (not os.path.isfile(qafig)) or clobber:
                qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist,
                                     [[]] * len(objlist), coeff)

    # FluxCalib QA
    if qatype == 'qa_data':
        # Standard stars
        stdstar_fil = meta.findfile('stdstars',
                                    night=night,
                                    camera=camera,
                                    expid=expid,
                                    specprod_dir=specprod_dir,
                                    spectrograph=spectro)
        # try:
        #    model_tuple=read_stdstar_models(stdstar_fil)
        # except FileNotFoundError:
        #    warnings.warn("Standard star file {:s} not found.  Skipping..".format(stdstar_fil))
        # else:
        flux_fil = meta.findfile('calib',
                                 night=night,
                                 camera=camera,
                                 expid=expid,
                                 specprod_dir=specprod_dir)
        try:
            fluxcalib = read_flux_calibration(flux_fil)
        except FileNotFoundError:
            warnings.warn(
                "Flux file {:s} not found.  Skipping..".format(flux_fil))
        else:
            if qaframe.run_qa(
                    'FLUXCALIB', (frame, fluxcalib),
                    clobber=clobber):  # , model_tuple))#, indiv_stars))
                write = True
            if make_plots:
                qafig = meta.findfile('qa_flux_fig',
                                      night=night,
                                      camera=camera,
                                      expid=expid,
                                      specprod_dir=specprod_dir,
                                      outdir=output_dir,
                                      qaprod_dir=qaprod_dir)
                if (not os.path.isfile(qafig)) or clobber:
                    qa_plots.frame_fluxcalib(qafig, qaframe, frame,
                                             fluxcalib)  # , model_tuple)
    # Write
    if write:
        write_qa_frame(qafile, qaframe, verbose=True)
    return qaframe
Example #25
0
 def run_pa(self,input_frame,fiberflat): 
  
     from desispec.fiberflat import apply_fiberflat 
     apply_fiberflat(input_frame,fiberflat)
     return input_frame
Example #26
0
 def run_pa(self,input_frame,fiberflat): 
  
     from desispec.quicklook.quickfiberflat import apply_fiberflat 
     fframe=apply_fiberflat(input_frame,fiberflat)
     return fframe
Example #27
0
def main(args) :

    log=get_logger()

    cmd = ['desi_compute_fluxcalibration',]
    for key, value in args.__dict__.items():
        if value is not None:
            cmd += ['--'+key, str(value)]
    cmd = ' '.join(cmd)
    log.info(cmd)

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel=read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux,model_wave,model_fibers,model_metadata=read_stdstar_models(args.models)

    if args.chi2cut > 0 :
        ok = np.where(model_metadata["CHI2DOF"]<args.chi2cut)[0]
        if ok.size == 0 :
            log.error("chi2cut has discarded all stars")
            sys.exit(12)
        nstars=model_flux.shape[0]
        nbad=nstars-ok.size
        if nbad>0 :
            log.warning("discarding %d star(s) out of %d because of chi2cut"%(nbad,nstars))
            model_flux=model_flux[ok]
            model_fibers=model_fibers[ok]
            model_metadata=model_metadata[:][ok]
    
    if args.delta_color_cut > 0 :
        ok = np.where(np.abs(model_metadata["MODEL_G-R"]-model_metadata["DATA_G-R"])<args.delta_color_cut)[0]
        nstars=model_flux.shape[0]
        nbad=nstars-ok.size
        if nbad>0 :
            log.warning("discarding %d star(s) out of %d because |delta_color|>%f"%(nbad,nstars,args.delta_color_cut))
            model_flux=model_flux[ok]
            model_fibers=model_fibers[ok]
            model_metadata=model_metadata[:][ok]
    

    # automatically reject stars that ar chi2 outliers
    if args.chi2cut_nsig > 0 :
        mchi2=np.median(model_metadata["CHI2DOF"])
        rmschi2=np.std(model_metadata["CHI2DOF"])
        maxchi2=mchi2+args.chi2cut_nsig*rmschi2
        ok=np.where(model_metadata["CHI2DOF"]<=maxchi2)[0]
        nstars=model_flux.shape[0]
        nbad=nstars-ok.size
        if nbad>0 :
            log.warning("discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"%(nbad,nstars,args.chi2cut_nsig,maxchi2))
            model_flux=model_flux[ok]
            model_fibers=model_fibers[ok]
            model_metadata=model_metadata[:][ok]
    
    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap

    ## check whether star fibers from args.models are consistent with fibers from fibermap
    ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
    fibermap_std_indices = np.where(isStdStar(fibermap['DESI_TARGET']))[0]
    if np.any(~np.in1d(model_fibers%500, fibermap_std_indices)):
        for i in model_fibers%500:
            log.error("inconsistency with spectrum {}, OBJTYPE='{}', DESI_TARGET={} in fibermap".format(
                (i, fibermap["OBJTYPE"][i], fibermap["DESI_TARGET"][i])))
        sys.exit(12)

    fluxcalib = compute_flux_calibration(frame, model_wave, model_flux, model_fibers%500)

    # QA
    if (args.qafile is not None):
        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile, frame, flavor=frame.meta['FLAVOR'])
        # Run
        #import pdb; pdb.set_trace()
        qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s"%args.outfile)
Example #28
0
def main(args):

    log = get_logger()

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel = read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux, model_wave, model_fibers = read_stdstar_models(args.models)
    model_tuple = model_flux, model_wave, model_fibers

    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap
    model_fibers = model_fibers % 500
    if np.any(fibermap['OBJTYPE'][model_fibers] != 'STD'):
        for i in model_fibers:
            log.error(
                "inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" %
                (i, fibermap["OBJTYPE"][i]))
        sys.exit(12)

    #fluxcalib, indiv_stars = compute_flux_calibration(frame, model_wave, model_flux)
    fluxcalib = compute_flux_calibration(frame, model_wave, model_flux)

    # QA
    if (args.qafile is not None):
        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile,
                                frame,
                                flavor=frame.meta['FLAVOR'])
        # Run
        qaframe.run_qa('FLUXCALIB',
                       (frame, fluxcalib, model_tuple))  #, indiv_stars))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib,
                                     model_tuple)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s" % args.outfile)
def main() :

    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument('--infile', type = str, default = None, required=True,
                        help = 'path of DESI exposure frame fits file')
    parser.add_argument('--fibermap', type = str, default = None, required=True,
                        help = 'path of DESI exposure frame fits file')
    parser.add_argument('--fiberflat', type = str, default = None, required=True,
                        help = 'path of DESI fiberflat fits file')
    parser.add_argument('--sky', type = str, default = None, required=True,
                        help = 'path of DESI sky fits file')
    parser.add_argument('--models', type = str, default = None, required=True,
                        help = 'path of spetro-photometric stellar spectra fits file')
    parser.add_argument('--outfile', type = str, default = None, required=True,
                        help = 'path of DESI flux calbration fits file')


    args = parser.parse_args()
    log=get_logger()

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)
    
    log.info("subtract sky")
    # read sky
    skymodel=read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux,model_wave,model_fibers=read_stdstar_models(args.models)

    # select fibers
    SPECMIN=frame.header["SPECMIN"]
    SPECMAX=frame.header["SPECMAX"]
    selec=np.where((model_fibers>=SPECMIN)&(model_fibers<=SPECMAX))[0]
    if selec.size == 0 :
        log.error("no stellar models for this spectro")
        sys.exit(12)
    fibers=model_fibers[selec]-frame.header["SPECMIN"]
    log.info("star fibers= %s"%str(fibers))

    table = read_fibermap(args.fibermap)
    bad=np.where(table["OBJTYPE"][fibers]!="STD")[0]
    if bad.size > 0 :
        for fiber in fibers[bad] :
            log.error("inconsistency with fiber %d, OBJTYPE='%s' in fibermap"%(fiber,table["OBJTYPE"][fiber]))
        sys.exit(12)

    fluxcalib = compute_flux_calibration(frame, fibers, model_wave, model_flux)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.header)


    log.info("successfully wrote %s"%args.outfile)
Example #30
0
def qaframe_from_frame(frame_file, specprod_dir=None, make_plots=False, qaprod_dir=None,
                       output_dir=None, clobber=True):
    """  Generate a qaframe object from an input frame_file name (and night)

    Write QA to disk
    Will also make plots if directed
    Args:
        frame_file: str
        specprod_dir: str, optional
        qa_dir: str, optional -- Location of QA
        make_plots: bool, optional
        output_dir: str, optional

    Returns:

    """
    import glob
    import os

    from desispec.io import read_frame
    from desispec.io import meta
    from desispec.io.qa import load_qa_frame, write_qa_frame
    from desispec.io.qa import qafile_from_framefile
    from desispec.io.frame import search_for_framefile
    from desispec.io.fiberflat import read_fiberflat
    from desispec.fiberflat import apply_fiberflat
    from desispec.qa import qa_plots
    from desispec.io.sky import read_sky
    from desispec.io.fluxcalibration import read_flux_calibration
    from desispec.qa import qa_plots_ql

    if '/' in frame_file:  # If present, assume full path is used here
        pass
    else: # Find the frame file in the desispec hierarchy?
        frame_file = search_for_framefile(frame_file)

    # Load frame
    frame = read_frame(frame_file)
    frame_meta = frame.meta
    night = frame_meta['NIGHT'].strip()
    camera = frame_meta['CAMERA'].strip()
    expid = frame_meta['EXPID']
    spectro = int(frame_meta['CAMERA'][-1])

    # Filename
    qafile, qatype = qafile_from_framefile(frame_file, qaprod_dir=qaprod_dir, output_dir=output_dir)
    if os.path.isfile(qafile) and (not clobber):
        write = False
    else:
        write = True
    qaframe = load_qa_frame(qafile, frame, flavor=frame.meta['FLAVOR'])
    # Flat QA
    if frame_meta['FLAVOR'] in ['flat']:
        fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid,
                                      specprod_dir=specprod_dir)
        try: # Backwards compatibility
            fiberflat = read_fiberflat(fiberflat_fil)
        except FileNotFoundError:
            fiberflat_fil = fiberflat_fil.replace('exposures', 'calib2d')
            path, basen = os.path.split(fiberflat_fil)
            path,_ = os.path.split(path)
            fiberflat_fil = os.path.join(path, basen)
            fiberflat = read_fiberflat(fiberflat_fil)
        if qaframe.run_qa('FIBERFLAT', (frame, fiberflat), clobber=clobber):
            write = True
        if make_plots:
            # Do it
            qafig = meta.findfile('qa_flat_fig', night=night, camera=camera, expid=expid,
                                  qaprod_dir=qaprod_dir, specprod_dir=specprod_dir, outdir=output_dir)
            if (not os.path.isfile(qafig)) or clobber:
                qa_plots.frame_fiberflat(qafig, qaframe, frame, fiberflat)
    # SkySub QA
    if qatype == 'qa_data':
        sky_fil = meta.findfile('sky', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir)

        fiberflat_fil = meta.findfile('fiberflatnight', night=night, camera=camera)
        if not os.path.exists(fiberflat_fil):
            # Backwards compatibility (for now)
            dummy_fiberflat_fil = meta.findfile('fiberflat', night=night, camera=camera, expid=expid,
                                            specprod_dir=specprod_dir) # This is dummy
            path = os.path.dirname(os.path.dirname(dummy_fiberflat_fil))
            fiberflat_files = glob.glob(os.path.join(path,'*','fiberflat-'+camera+'*.fits'))
            if len(fiberflat_files) == 0:
                path = path.replace('exposures', 'calib2d')
                path,_ = os.path.split(path) # Remove night
                fiberflat_files = glob.glob(os.path.join(path,'fiberflat-'+camera+'*.fits'))

            # Sort and take the first (same as old pipeline)
            fiberflat_files.sort()
            fiberflat_fil = fiberflat_files[0]

        fiberflat = read_fiberflat(fiberflat_fil)
        apply_fiberflat(frame, fiberflat)
        # Load sky model and run
        try:
            skymodel = read_sky(sky_fil)
        except FileNotFoundError:
            warnings.warn("Sky file {:s} not found.  Skipping..".format(sky_fil))
        else:
            if qaframe.run_qa('SKYSUB', (frame, skymodel), clobber=clobber):
                write=True
            if make_plots:
                qafig = meta.findfile('qa_sky_fig', night=night, camera=camera, expid=expid,
                                      specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
                qafig2 = meta.findfile('qa_skychi_fig', night=night, camera=camera, expid=expid,
                                      specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
                if (not os.path.isfile(qafig)) or clobber:
                    qa_plots.frame_skyres(qafig, frame, skymodel, qaframe)
                #qa_plots.frame_skychi(qafig2, frame, skymodel, qaframe)

    # S/N QA on cframe
    if qatype == 'qa_data':
        # cframe
        cframe_file = frame_file.replace('frame-', 'cframe-')
        cframe = read_frame(cframe_file)
        if qaframe.run_qa('S2N', (cframe,), clobber=clobber):
            write=True
        # Figure?
        if make_plots:
            s2n_dict = copy.deepcopy(qaframe.qa_data['S2N'])
            qafig = meta.findfile('qa_s2n_fig', night=night, camera=camera, expid=expid,
                              specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
            #badfibs = np.where(np.isnan(s2n_dict['METRICS']['MEDIAN_SNR']))[0].tolist()
            #sci_idx = s2n_dict['METRICS']['OBJLIST'].index('SCIENCE')
            coeff = s2n_dict['METRICS']['FITCOEFF_TGT']#[sci_idx]
            # Add an item or two for the QL method
            s2n_dict['CAMERA'] = camera
            s2n_dict['EXPID'] = expid
            s2n_dict['PANAME'] = 'SNRFit'
            s2n_dict['METRICS']['RA'] = frame.fibermap['FIBER_RA']
            s2n_dict['METRICS']['DEC'] = frame.fibermap['FIBER_DEC']
            objlist = s2n_dict['METRICS']['OBJLIST']
            # Deal with YAML list instead of ndarray
            s2n_dict['METRICS']['MEDIAN_SNR'] = np.array(s2n_dict['METRICS']['MEDIAN_SNR'])
            # Generate
            if (not os.path.isfile(qafig)) or clobber:
                qa_plots_ql.plot_SNR(s2n_dict, qafig, objlist, [[]]*len(objlist), coeff)

    # FluxCalib QA
    if qatype == 'qa_data':
        # Standard stars
        stdstar_fil = meta.findfile('stdstars', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir,
                                    spectrograph=spectro)
        # try:
        #    model_tuple=read_stdstar_models(stdstar_fil)
        # except FileNotFoundError:
        #    warnings.warn("Standard star file {:s} not found.  Skipping..".format(stdstar_fil))
        # else:
        flux_fil = meta.findfile('calib', night=night, camera=camera, expid=expid, specprod_dir=specprod_dir)
        try:
            fluxcalib = read_flux_calibration(flux_fil)
        except FileNotFoundError:
            warnings.warn("Flux file {:s} not found.  Skipping..".format(flux_fil))
        else:
            if qaframe.run_qa('FLUXCALIB', (frame, fluxcalib), clobber=clobber):  # , model_tuple))#, indiv_stars))
                write = True
            if make_plots:
                qafig = meta.findfile('qa_flux_fig', night=night, camera=camera, expid=expid,
                                      specprod_dir=specprod_dir, outdir=output_dir, qaprod_dir=qaprod_dir)
                if (not os.path.isfile(qafig)) or clobber:
                    qa_plots.frame_fluxcalib(qafig, qaframe, frame, fluxcalib)  # , model_tuple)
    # Write
    if write:
        write_qa_frame(qafile, qaframe, verbose=True)
    return qaframe
Example #31
0
def main(args):

    log = get_logger()

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel = read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
        args.models)

    if args.chi2cut > 0:
        ok = np.where(model_metadata["CHI2DOF"] < args.chi2cut)[0]
        if ok.size == 0:
            log.error("chi2cut has discarded all stars")
            sys.exit(12)
        nstars = model_flux.shape[0]
        nbad = nstars - ok.size
        if nbad > 0:
            log.warning("discarding %d star(s) out of %d because of chi2cut" %
                        (nbad, nstars))
            model_flux = model_flux[ok]
            model_fibers = model_fibers[ok]
            model_metadata = model_metadata[:][ok]

    if args.delta_color_cut > 0:
        ok = np.where(
            np.abs(model_metadata["MODEL_G-R"] -
                   model_metadata["DATA_G-R"]) < args.delta_color_cut)[0]
        nstars = model_flux.shape[0]
        nbad = nstars - ok.size
        if nbad > 0:
            log.warning(
                "discarding %d star(s) out of %d because |delta_color|>%f" %
                (nbad, nstars, args.delta_color_cut))
            model_flux = model_flux[ok]
            model_fibers = model_fibers[ok]
            model_metadata = model_metadata[:][ok]

    # automatically reject stars that ar chi2 outliers
    if args.chi2cut_nsig > 0:
        mchi2 = np.median(model_metadata["CHI2DOF"])
        rmschi2 = np.std(model_metadata["CHI2DOF"])
        maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
        ok = np.where(model_metadata["CHI2DOF"] <= maxchi2)[0]
        nstars = model_flux.shape[0]
        nbad = nstars - ok.size
        if nbad > 0:
            log.warning(
                "discarding %d star(s) out of %d because reduced chi2 outliers (at %d sigma, giving rchi2<%f )"
                % (nbad, nstars, args.chi2cut_nsig, maxchi2))
            model_flux = model_flux[ok]
            model_fibers = model_fibers[ok]
            model_metadata = model_metadata[:][ok]

    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap

    ## check whether star fibers from args.models are consistent with fibers from fibermap
    ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
    w = np.where(fibermap["OBJTYPE"][model_fibers % 500] != 'STD')[0]

    if len(w) > 0:
        for i in model_fibers % 500:
            log.error(
                "inconsistency with spectrum %d, OBJTYPE='%s' in fibermap" %
                (i, fibermap["OBJTYPE"][i]))
        sys.exit(12)

    fluxcalib = compute_flux_calibration(frame, model_wave, model_flux,
                                         model_fibers % 500)

    # QA
    if (args.qafile is not None):
        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile,
                                frame,
                                flavor=frame.meta['FLAVOR'])
        # Run
        #import pdb; pdb.set_trace()
        qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s" % args.outfile)
Example #32
0
    def run_pa(self, input_frame, fiberflat):

        from desispec.fiberflat import apply_fiberflat
        apply_fiberflat(input_frame, fiberflat)
        return input_frame
Example #33
0
def main(args=None, comm=None):
    if args is None:
        args = parse()
    # elif isinstance(args, (list, tuple)):
    #     args = parse(args)

    log = get_logger()

    start_mpi_connect = time.time()
    if comm is not None:
        #- Use the provided comm to determine rank and size
        rank = comm.rank
        size = comm.size
    else:
        #- Check MPI flags and determine the comm, rank, and size given the arguments
        comm, rank, size = assign_mpi(do_mpi=args.mpi,
                                      do_batch=args.batch,
                                      log=log)
    stop_mpi_connect = time.time()

    #- Start timer; only print log messages from rank 0 (others are silent)
    timer = desiutil.timer.Timer(silent=(rank > 0))

    #- Fill in timing information for steps before we had the timer created
    if args.starttime is not None:
        timer.start('startup', starttime=args.starttime)
        timer.stop('startup', stoptime=start_imports)

    timer.start('imports', starttime=start_imports)
    timer.stop('imports', stoptime=stop_imports)

    timer.start('mpi_connect', starttime=start_mpi_connect)
    timer.stop('mpi_connect', stoptime=stop_mpi_connect)

    #- Freeze IERS after parsing args so that it doesn't bother if only --help
    timer.start('freeze_iers')
    desiutil.iers.freeze_iers()
    timer.stop('freeze_iers')

    #- Preflight checks
    timer.start('preflight')
    if rank > 0:
        #- Let rank 0 fetch these, and then broadcast
        args, hdr, camhdr = None, None, None
    else:
        args, hdr, camhdr = update_args_with_headers(args)

    ## Make sure badamps is formatted properly
    if comm is not None and rank == 0 and args.badamps is not None:
        args.badamps = validate_badamps(args.badamps)

    if comm is not None:
        args = comm.bcast(args, root=0)
        hdr = comm.bcast(hdr, root=0)
        camhdr = comm.bcast(camhdr, root=0)

    known_obstype = [
        'SCIENCE', 'ARC', 'FLAT', 'ZERO', 'DARK', 'TESTARC', 'TESTFLAT',
        'PIXFLAT', 'SKY', 'TWILIGHT', 'OTHER'
    ]
    if args.obstype not in known_obstype:
        raise RuntimeError('obstype {} not in {}'.format(
            args.obstype, known_obstype))

    timer.stop('preflight')

    #-------------------------------------------------------------------------
    #- Create and submit a batch job if requested

    if args.batch:
        #exp_str = '{:08d}'.format(args.expid)
        jobdesc = args.obstype.lower()
        if args.obstype == 'SCIENCE':
            # if not doing pre-stdstar fitting or stdstar fitting and if there is
            # no flag stopping flux calibration, set job to poststdstar
            if args.noprestdstarfit and args.nostdstarfit and (
                    not args.nofluxcalib):
                jobdesc = 'poststdstar'
            # elif told not to do std or post stdstar but the flag for prestdstar isn't set,
            # then perform prestdstar
            elif (not args.noprestdstarfit
                  ) and args.nostdstarfit and args.nofluxcalib:
                jobdesc = 'prestdstar'
            #elif (not args.noprestdstarfit) and (not args.nostdstarfit) and (not args.nofluxcalib):
            #    jobdesc = 'science'
        scriptfile = create_desi_proc_batch_script(night=args.night, exp=args.expid, cameras=args.cameras,\
                                                jobdesc=jobdesc, queue=args.queue, runtime=args.runtime,\
                                                batch_opts=args.batch_opts, timingfile=args.timingfile,
                                                system_name=args.system_name)
        err = 0
        if not args.nosubmit:
            err = subprocess.call(['sbatch', scriptfile])
        sys.exit(err)

    #-------------------------------------------------------------------------
    #- Proceeding with running

    #- What are we going to do?
    if rank == 0:
        log.info('----------')
        log.info('Input {}'.format(args.input))
        log.info('Night {} expid {}'.format(args.night, args.expid))
        log.info('Obstype {}'.format(args.obstype))
        log.info('Cameras {}'.format(args.cameras))
        log.info('Output root {}'.format(desispec.io.specprod_root()))
        log.info('----------')

    #- Create output directories if needed
    if rank == 0:
        preprocdir = os.path.dirname(
            findfile('preproc', args.night, args.expid, 'b0'))
        expdir = os.path.dirname(
            findfile('frame', args.night, args.expid, 'b0'))
        os.makedirs(preprocdir, exist_ok=True)
        os.makedirs(expdir, exist_ok=True)

    #- Wait for rank 0 to make directories before proceeding
    if comm is not None:
        comm.barrier()

    #-------------------------------------------------------------------------
    #- Preproc
    #- All obstypes get preprocessed

    timer.start('fibermap')

    #- Assemble fibermap for science exposures
    fibermap = None
    fibermap_ok = None
    if rank == 0 and args.obstype == 'SCIENCE':
        fibermap = findfile('fibermap', args.night, args.expid)
        if not os.path.exists(fibermap):
            tmp = findfile('preproc', args.night, args.expid, 'b0')
            preprocdir = os.path.dirname(tmp)
            fibermap = os.path.join(preprocdir, os.path.basename(fibermap))

            log.info('Creating fibermap {}'.format(fibermap))
            cmd = 'assemble_fibermap -n {} -e {} -o {}'.format(
                args.night, args.expid, fibermap)
            if args.badamps is not None:
                cmd += ' --badamps={}'.format(args.badamps)
            runcmd(cmd, inputs=[], outputs=[fibermap])

        fibermap_ok = os.path.exists(fibermap)

        #- Some commissioning files didn't have coords* files that caused assemble_fibermap to fail
        #- these are well known failures with no other solution, so for those, just force creation
        #- of a fibermap with null coordinate information
        if not fibermap_ok and int(args.night) < 20200310:
            log.info(
                "Since night is before 20200310, trying to force fibermap creation without coords file"
            )
            cmd += ' --force'
            runcmd(cmd, inputs=[], outputs=[fibermap])
            fibermap_ok = os.path.exists(fibermap)

    #- If assemble_fibermap failed and obstype is SCIENCE, exit now
    if comm is not None:
        fibermap_ok = comm.bcast(fibermap_ok, root=0)

    if args.obstype == 'SCIENCE' and not fibermap_ok:
        sys.stdout.flush()
        if rank == 0:
            log.critical(
                'assemble_fibermap failed for science exposure; exiting now')

        sys.exit(13)

    #- Wait for rank 0 to make fibermap if needed
    if comm is not None:
        fibermap = comm.bcast(fibermap, root=0)

    timer.stop('fibermap')

    if not (args.obstype in ['SCIENCE'] and args.noprestdstarfit):
        timer.start('preproc')
        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            outfile = findfile('preproc', args.night, args.expid, camera)
            outdir = os.path.dirname(outfile)
            cmd = "desi_preproc -i {} -o {} --outdir {} --cameras {}".format(
                args.input, outfile, outdir, camera)
            if args.scattered_light:
                cmd += " --scattered-light"
            if fibermap is not None:
                cmd += " --fibermap {}".format(fibermap)
            if not args.obstype in ['ARC']:  # never model variance for arcs
                if not args.no_model_pixel_variance:
                    cmd += " --model-variance"
            runcmd(cmd, inputs=[args.input], outputs=[outfile])

        timer.stop('preproc')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Get input PSFs
    timer.start('findpsf')
    input_psf = dict()
    if rank == 0:
        for camera in args.cameras:
            if args.psf is not None:
                input_psf[camera] = args.psf
            elif args.calibnight is not None:
                # look for a psfnight psf for this calib night
                psfnightfile = findfile('psfnight', args.calibnight,
                                        args.expid, camera)
                if not os.path.isfile(psfnightfile):
                    log.error("no {}".format(psfnightfile))
                    raise IOError("no {}".format(psfnightfile))
                input_psf[camera] = psfnightfile
            else:
                # look for a psfnight psf
                psfnightfile = findfile('psfnight', args.night, args.expid,
                                        camera)
                if os.path.isfile(psfnightfile):
                    input_psf[camera] = psfnightfile
                elif args.most_recent_calib:
                    nightfile = find_most_recent(args.night,
                                                 file_type='psfnight')
                    if nightfile is None:
                        input_psf[camera] = findcalibfile(
                            [hdr, camhdr[camera]], 'PSF')
                    else:
                        input_psf[camera] = nightfile
                else:
                    input_psf[camera] = findcalibfile([hdr, camhdr[camera]],
                                                      'PSF')
            log.info("Will use input PSF : {}".format(input_psf[camera]))

    if comm is not None:
        input_psf = comm.bcast(input_psf, root=0)

    timer.stop('findpsf')

    #-------------------------------------------------------------------------
    #- Traceshift

    if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT']     )   or \
    ( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ):

        timer.start('traceshift')

        if rank == 0 and args.traceshift:
            log.info('Starting traceshift at {}'.format(time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            preprocfile = findfile('preproc', args.night, args.expid, camera)
            inpsf = input_psf[camera]
            outpsf = findfile('psf', args.night, args.expid, camera)
            if not os.path.isfile(outpsf):
                if args.traceshift:
                    cmd = "desi_compute_trace_shifts"
                    cmd += " -i {}".format(preprocfile)
                    cmd += " --psf {}".format(inpsf)
                    cmd += " --outpsf {}".format(outpsf)
                    cmd += " --degxx 2 --degxy 0"
                    if args.obstype in ['FLAT', 'TESTFLAT', 'TWILIGHT']:
                        cmd += " --continuum"
                    else:
                        cmd += " --degyx 2 --degyy 0"
                    if args.obstype in ['SCIENCE', 'SKY']:
                        cmd += ' --sky'
                else:
                    cmd = "ln -s {} {}".format(inpsf, outpsf)
                runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf])
            else:
                log.info("PSF {} exists".format(outpsf))

        timer.stop('traceshift')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- PSF
    #- MPI parallelize this step

    if args.obstype in ['ARC', 'TESTARC']:

        timer.start('arc_traceshift')

        if rank == 0:
            log.info('Starting traceshift before specex PSF fit at {}'.format(
                time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            preprocfile = findfile('preproc', args.night, args.expid, camera)
            inpsf = input_psf[camera]
            outpsf = findfile('psf', args.night, args.expid, camera)
            outpsf = replace_prefix(outpsf, "psf", "shifted-input-psf")
            if not os.path.isfile(outpsf):
                cmd = "desi_compute_trace_shifts"
                cmd += " -i {}".format(preprocfile)
                cmd += " --psf {}".format(inpsf)
                cmd += " --outpsf {}".format(outpsf)
                cmd += " --degxx 0 --degxy 0 --degyx 0 --degyy 0"
                cmd += ' --arc-lamps'
                runcmd(cmd, inputs=[preprocfile, inpsf], outputs=[outpsf])
            else:
                log.info("PSF {} exists".format(outpsf))

        timer.stop('arc_traceshift')
        if comm is not None:
            comm.barrier()

        timer.start('psf')

        if rank == 0:
            log.info('Starting specex PSF fitting at {}'.format(
                time.asctime()))

        if rank > 0:
            cmds = inputs = outputs = None
        else:
            cmds = dict()
            inputs = dict()
            outputs = dict()
            for camera in args.cameras:
                preprocfile = findfile('preproc', args.night, args.expid,
                                       camera)
                tmpname = findfile('psf', args.night, args.expid, camera)
                inpsf = replace_prefix(tmpname, "psf", "shifted-input-psf")
                outpsf = replace_prefix(tmpname, "psf", "fit-psf")

                log.info("now run specex psf fit")

                cmd = 'desi_compute_psf'
                cmd += ' --input-image {}'.format(preprocfile)
                cmd += ' --input-psf {}'.format(inpsf)
                cmd += ' --output-psf {}'.format(outpsf)

                # look for fiber blacklist
                cfinder = CalibFinder([hdr, camhdr[camera]])
                blacklistkey = "FIBERBLACKLIST"
                if not cfinder.haskey(blacklistkey) and cfinder.haskey(
                        "BROKENFIBERS"):
                    log.warning(
                        "BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST"
                    )
                    blacklistkey = "BROKENFIBERS"

                if cfinder.haskey(blacklistkey):
                    blacklist = cfinder.value(blacklistkey)
                    cmd += ' --broken-fibers {}'.format(blacklist)
                    if rank == 0:
                        log.warning('broken fibers: {}'.format(blacklist))

                if not os.path.exists(outpsf):
                    cmds[camera] = cmd
                    inputs[camera] = [preprocfile, inpsf]
                    outputs[camera] = [
                        outpsf,
                    ]

        if comm is not None:
            cmds = comm.bcast(cmds, root=0)
            inputs = comm.bcast(inputs, root=0)
            outputs = comm.bcast(outputs, root=0)
            #- split communicator by 20 (number of bundles)
            group_size = 20
            if (rank == 0) and (size % group_size != 0):
                log.warning(
                    'MPI size={} should be evenly divisible by {}'.format(
                        size, group_size))

            group = rank // group_size
            num_groups = (size + group_size - 1) // group_size
            comm_group = comm.Split(color=group)

            if rank == 0:
                log.info(
                    f'Fitting PSFs with {num_groups} sub-communicators of size {group_size}'
                )

            for i in range(group, len(args.cameras), num_groups):
                camera = args.cameras[i]
                if camera in cmds:
                    cmdargs = cmds[camera].split()[1:]
                    cmdargs = desispec.scripts.specex.parse(cmdargs)
                    if comm_group.rank == 0:
                        print('RUNNING: {}'.format(cmds[camera]))
                        t0 = time.time()
                        timestamp = time.asctime()
                        log.info(
                            f'MPI group {group} ranks {rank}-{rank+group_size-1} fitting PSF for {camera} at {timestamp}'
                        )
                    try:
                        desispec.scripts.specex.main(cmdargs, comm=comm_group)
                    except Exception as e:
                        if comm_group.rank == 0:
                            log.error(
                                f'FAILED: MPI group {group} ranks {rank}-{rank+group_size-1} camera {camera}'
                            )
                            log.error('FAILED: {}'.format(cmds[camera]))
                            log.error(e)

                    if comm_group.rank == 0:
                        specex_time = time.time() - t0
                        log.info(
                            f'specex fit for {camera} took {specex_time:.1f} seconds'
                        )

            comm.barrier()

        else:
            log.warning(
                'fitting PSFs without MPI parallelism; this will be SLOW')
            for camera in args.cameras:
                if camera in cmds:
                    runcmd(cmds[camera],
                           inputs=inputs[camera],
                           outputs=outputs[camera])

        if comm is not None:
            comm.barrier()

        # loop on all cameras and interpolate bad fibers
        for camera in args.cameras[rank::size]:
            t0 = time.time()
            log.info(f'Rank {rank} interpolating {camera} PSF over bad fibers')
            # look for fiber blacklist
            cfinder = CalibFinder([hdr, camhdr[camera]])
            blacklistkey = "FIBERBLACKLIST"
            if not cfinder.haskey(blacklistkey) and cfinder.haskey(
                    "BROKENFIBERS"):
                log.warning(
                    "BROKENFIBERS yaml keyword deprecated, please use FIBERBLACKLIST"
                )
                blacklistkey = "BROKENFIBERS"

            if cfinder.haskey(blacklistkey):
                fiberblacklist = cfinder.value(blacklistkey)
                tmpname = findfile('psf', args.night, args.expid, camera)
                inpsf = replace_prefix(tmpname, "psf", "fit-psf")
                outpsf = replace_prefix(tmpname, "psf",
                                        "fit-psf-fixed-blacklisted")
                if os.path.isfile(inpsf) and not os.path.isfile(outpsf):
                    cmd = 'desi_interpolate_fiber_psf'
                    cmd += ' --infile {}'.format(inpsf)
                    cmd += ' --outfile {}'.format(outpsf)
                    cmd += ' --fibers {}'.format(fiberblacklist)
                    log.info(
                        'For camera {} interpolating PSF for broken fibers: {}'
                        .format(camera, fiberblacklist))
                    runcmd(cmd, inputs=[inpsf], outputs=[outpsf])
                    if os.path.isfile(outpsf):
                        os.rename(
                            inpsf,
                            inpsf.replace("fit-psf",
                                          "fit-psf-before-blacklisted-fix"))
                        subprocess.call('cp {} {}'.format(outpsf, inpsf),
                                        shell=True)

            dt = time.time() - t0
            log.info(
                f'Rank {rank} {camera} PSF interpolation took {dt:.1f} sec')

        timer.stop('psf')

    #-------------------------------------------------------------------------
    #- Merge PSF of night if applicable

    #if args.obstype in ['ARC']:
    if False:
        if rank == 0:
            for camera in args.cameras:
                psfnightfile = findfile('psfnight', args.night, args.expid,
                                        camera)
                if not os.path.isfile(
                        psfnightfile
                ):  # we still don't have a psf night, see if we can compute it ...
                    psfs = glob.glob(
                        findfile('psf', args.night, args.expid,
                                 camera).replace("psf", "fit-psf").replace(
                                     str(args.expid), "*"))
                    log.info(
                        "Number of PSF for night={} camera={} = {}".format(
                            args.night, camera, len(psfs)))
                    if len(psfs) > 4:  # lets do it!
                        log.info("Computing psfnight ...")
                        dirname = os.path.dirname(psfnightfile)
                        if not os.path.isdir(dirname):
                            os.makedirs(dirname)
                        desispec.scripts.specex.mean_psf(psfs, psfnightfile)
                if os.path.isfile(psfnightfile):  # now use this one
                    input_psf[camera] = psfnightfile

    #-------------------------------------------------------------------------
    #- Extract
    #- This is MPI parallel so handle a bit differently

    # maybe add ARC and TESTARC too
    if ( args.obstype in ['FLAT', 'TESTFLAT', 'SKY', 'TWILIGHT']     )   or \
    ( args.obstype in ['SCIENCE'] and (not args.noprestdstarfit) ):

        timer.start('extract')
        if rank == 0:
            log.info('Starting extractions at {}'.format(time.asctime()))

        if rank > 0:
            cmds = inputs = outputs = None
        else:
            cmds = dict()
            inputs = dict()
            outputs = dict()
            for camera in args.cameras:
                cmd = 'desi_extract_spectra'

                #- Based on data from SM1-SM8, looking at central and edge fibers
                #- with in mind overlapping arc lamps lines
                if camera.startswith('b'):
                    cmd += ' -w 3600.0,5800.0,0.8'
                elif camera.startswith('r'):
                    cmd += ' -w 5760.0,7620.0,0.8'
                elif camera.startswith('z'):
                    cmd += ' -w 7520.0,9824.0,0.8'

                preprocfile = findfile('preproc', args.night, args.expid,
                                       camera)
                psffile = findfile('psf', args.night, args.expid, camera)
                framefile = findfile('frame', args.night, args.expid, camera)
                cmd += ' -i {}'.format(preprocfile)
                cmd += ' -p {}'.format(psffile)
                cmd += ' -o {}'.format(framefile)
                cmd += ' --psferr 0.1'

                if args.obstype == 'SCIENCE' or args.obstype == 'SKY':
                    if rank == 0:
                        log.info('Include barycentric correction')
                    cmd += ' --barycentric-correction'

                if not os.path.exists(framefile):
                    cmds[camera] = cmd
                    inputs[camera] = [preprocfile, psffile]
                    outputs[camera] = [
                        framefile,
                    ]

        #- TODO: refactor/combine this with PSF comm splitting logic
        if comm is not None:
            cmds = comm.bcast(cmds, root=0)
            inputs = comm.bcast(inputs, root=0)
            outputs = comm.bcast(outputs, root=0)

            #- split communicator by 20 (number of bundles)
            extract_size = 20
            if (rank == 0) and (size % extract_size != 0):
                log.warning(
                    'MPI size={} should be evenly divisible by {}'.format(
                        size, extract_size))

            extract_group = rank // extract_size
            num_extract_groups = (size + extract_size - 1) // extract_size
            comm_extract = comm.Split(color=extract_group)

            for i in range(extract_group, len(args.cameras),
                           num_extract_groups):
                camera = args.cameras[i]
                if camera in cmds:
                    cmdargs = cmds[camera].split()[1:]
                    extract_args = desispec.scripts.extract.parse(cmdargs)
                    if comm_extract.rank == 0:
                        print('RUNNING: {}'.format(cmds[camera]))

                    desispec.scripts.extract.main_mpi(extract_args,
                                                      comm=comm_extract)

            comm.barrier()

        else:
            log.warning(
                'running extractions without MPI parallelism; this will be SLOW'
            )
            for camera in args.cameras:
                if camera in cmds:
                    runcmd(cmds[camera],
                           inputs=inputs[camera],
                           outputs=outputs[camera])

        timer.stop('extract')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Fiberflat

    if args.obstype in ['FLAT', 'TESTFLAT']:
        timer.start('fiberflat')
        if rank == 0:
            log.info('Starting fiberflats at {}'.format(time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            framefile = findfile('frame', args.night, args.expid, camera)
            fiberflatfile = findfile('fiberflat', args.night, args.expid,
                                     camera)
            cmd = "desi_compute_fiberflat"
            cmd += " -i {}".format(framefile)
            cmd += " -o {}".format(fiberflatfile)
            runcmd(cmd, inputs=[
                framefile,
            ], outputs=[
                fiberflatfile,
            ])

        timer.stop('fiberflat')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Average and auto-calib fiberflats of night if applicable

    #if args.obstype in ['FLAT']:
    if False:
        if rank == 0:
            fiberflatnightfile = findfile('fiberflatnight', args.night,
                                          args.expid, args.cameras[0])
            fiberflatdirname = os.path.dirname(fiberflatnightfile)
            if not os.path.isfile(fiberflatnightfile) and len(
                    args.cameras
            ) >= 6:  # we still don't have them, see if we can compute them, but need at least 2 spectros ...
                flats = glob.glob(
                    findfile('fiberflat', args.night, args.expid,
                             "b0").replace(str(args.expid),
                                           "*").replace("b0", "*"))
                log.info("Number of fiberflat for night {} = {}".format(
                    args.night, len(flats)))
                if len(flats) >= 3 * 4 * len(
                        args.cameras
                ):  # lets do it! (3 exposures x 4 lamps x N cameras)
                    log.info(
                        "Computing fiberflatnight per lamp and camera ...")
                    tmpdir = os.path.join(fiberflatdirname, "tmp")
                    if not os.path.isdir(tmpdir):
                        os.makedirs(tmpdir)

                    log.info(
                        "First average measurements per camera and per lamp")
                    average_flats = dict()
                    for camera in args.cameras:
                        # list of flats for this camera
                        flats_for_this_camera = []
                        for flat in flats:
                            if flat.find(camera) >= 0:
                                flats_for_this_camera.append(flat)
                        #log.info("For camera {} , flats = {}".format(camera,flats_for_this_camera))
                        #sys.exit(12)

                        # average per lamp (and camera)
                        average_flats[camera] = list()
                        for lampbox in range(4):
                            ofile = os.path.join(
                                tmpdir,
                                "fiberflatnight-camera-{}-lamp-{}.fits".format(
                                    camera, lampbox))
                            if not os.path.isfile(ofile):
                                log.info(
                                    "Average flat for camera {} and lamp box #{}"
                                    .format(camera, lampbox))
                                pg = "CALIB DESI-CALIB-0{} LEDs only".format(
                                    lampbox)

                                cmd = "desi_average_fiberflat --program '{}' --outfile {} -i ".format(
                                    pg, ofile)
                                for flat in flats_for_this_camera:
                                    cmd += " {} ".format(flat)
                                runcmd(cmd,
                                       inputs=flats_for_this_camera,
                                       outputs=[
                                           ofile,
                                       ])
                                if os.path.isfile(ofile):
                                    average_flats[camera].append(ofile)
                            else:
                                log.info("Will use existing {}".format(ofile))
                                average_flats[camera].append(ofile)

                    log.info(
                        "Auto-calibration across lamps and spectro  per camera arm (b,r,z)"
                    )
                    for camera_arm in ["b", "r", "z"]:
                        cameras_for_this_arm = []
                        flats_for_this_arm = []
                        for camera in args.cameras:
                            if camera[0].lower() == camera_arm:
                                cameras_for_this_arm.append(camera)
                                if camera in average_flats:
                                    for flat in average_flats[camera]:
                                        flats_for_this_arm.append(flat)
                        cmd = "desi_autocalib_fiberflat --night {} --arm {} -i ".format(
                            args.night, camera_arm)
                        for flat in flats_for_this_arm:
                            cmd += " {} ".format(flat)
                        runcmd(cmd, inputs=flats_for_this_arm, outputs=[])
                    log.info("Done with fiber flats per night")

        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Get input fiberflat
    if args.obstype in ['SCIENCE', 'SKY'] and (not args.nofiberflat):
        timer.start('find_fiberflat')
        input_fiberflat = dict()
        if rank == 0:
            for camera in args.cameras:
                if args.fiberflat is not None:
                    input_fiberflat[camera] = args.fiberflat
                elif args.calibnight is not None:
                    # look for a fiberflatnight for this calib night
                    fiberflatnightfile = findfile('fiberflatnight',
                                                  args.calibnight, args.expid,
                                                  camera)
                    if not os.path.isfile(fiberflatnightfile):
                        log.error("no {}".format(fiberflatnightfile))
                        raise IOError("no {}".format(fiberflatnightfile))
                    input_fiberflat[camera] = fiberflatnightfile
                else:
                    # look for a fiberflatnight fiberflat
                    fiberflatnightfile = findfile('fiberflatnight', args.night,
                                                  args.expid, camera)
                    if os.path.isfile(fiberflatnightfile):
                        input_fiberflat[camera] = fiberflatnightfile
                    elif args.most_recent_calib:
                        nightfile = find_most_recent(
                            args.night, file_type='fiberflatnight')
                        if nightfile is None:
                            input_fiberflat[camera] = findcalibfile(
                                [hdr, camhdr[camera]], 'FIBERFLAT')
                        else:
                            input_fiberflat[camera] = nightfile
                    else:
                        input_fiberflat[camera] = findcalibfile(
                            [hdr, camhdr[camera]], 'FIBERFLAT')
                log.info("Will use input FIBERFLAT: {}".format(
                    input_fiberflat[camera]))

        if comm is not None:
            input_fiberflat = comm.bcast(input_fiberflat, root=0)

        timer.stop('find_fiberflat')

    #-------------------------------------------------------------------------
    #- Apply fiberflat and write fframe file

    if args.obstype in ['SCIENCE', 'SKY'] and args.fframe and \
    ( not args.nofiberflat ) and (not args.noprestdstarfit):
        timer.start('apply_fiberflat')
        if rank == 0:
            log.info('Applying fiberflat at {}'.format(time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            fframefile = findfile('fframe', args.night, args.expid, camera)
            if not os.path.exists(fframefile):
                framefile = findfile('frame', args.night, args.expid, camera)
                fr = desispec.io.read_frame(framefile)
                flatfilename = input_fiberflat[camera]
                if flatfilename is not None:
                    ff = desispec.io.read_fiberflat(flatfilename)
                    fr.meta['FIBERFLT'] = desispec.io.shorten_filename(
                        flatfilename)
                    apply_fiberflat(fr, ff)

                    fframefile = findfile('fframe', args.night, args.expid,
                                          camera)
                    desispec.io.write_frame(fframefile, fr)
                else:
                    log.warning(
                        "Missing fiberflat for camera {}".format(camera))

        timer.stop('apply_fiberflat')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Select random sky fibers (inplace update of frame file)
    #- TODO: move this to a function somewhere
    #- TODO: this assigns different sky fibers to each frame of same spectrograph

    if (args.obstype in [
            'SKY', 'SCIENCE'
    ]) and (not args.noskysub) and (not args.noprestdstarfit):
        timer.start('picksky')
        if rank == 0:
            log.info('Picking sky fibers at {}'.format(time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            framefile = findfile('frame', args.night, args.expid, camera)
            orig_frame = desispec.io.read_frame(framefile)

            #- Make a copy so that we can apply fiberflat
            fr = deepcopy(orig_frame)

            if np.any(fr.fibermap['OBJTYPE'] == 'SKY'):
                log.info('{} sky fibers already set; skipping'.format(
                    os.path.basename(framefile)))
                continue

            #- Apply fiberflat then select random fibers below a flux cut
            flatfilename = input_fiberflat[camera]
            if flatfilename is None:
                log.error("No fiberflat for {}".format(camera))
                continue
            ff = desispec.io.read_fiberflat(flatfilename)
            apply_fiberflat(fr, ff)
            sumflux = np.sum(fr.flux, axis=1)
            fluxcut = np.percentile(sumflux, 30)
            iisky = np.where(sumflux < fluxcut)[0]
            iisky = np.random.choice(iisky, size=100, replace=False)

            #- Update fibermap or original frame and write out
            orig_frame.fibermap['OBJTYPE'][iisky] = 'SKY'
            orig_frame.fibermap['DESI_TARGET'][iisky] |= desi_mask.SKY

            desispec.io.write_frame(framefile, orig_frame)

        timer.stop('picksky')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Sky subtraction
    if args.obstype in [
            'SCIENCE', 'SKY'
    ] and (not args.noskysub) and (not args.noprestdstarfit):
        timer.start('skysub')
        if rank == 0:
            log.info('Starting sky subtraction at {}'.format(time.asctime()))

        for i in range(rank, len(args.cameras), size):
            camera = args.cameras[i]
            framefile = findfile('frame', args.night, args.expid, camera)
            hdr = fitsio.read_header(framefile, 'FLUX')
            fiberflatfile = input_fiberflat[camera]
            if fiberflatfile is None:
                log.error("No fiberflat for {}".format(camera))
                continue
            skyfile = findfile('sky', args.night, args.expid, camera)

            cmd = "desi_compute_sky"
            cmd += " -i {}".format(framefile)
            cmd += " --fiberflat {}".format(fiberflatfile)
            cmd += " --o {}".format(skyfile)
            if args.no_extra_variance:
                cmd += " --no-extra-variance"
            if not args.no_sky_wavelength_adjustment:
                cmd += " --adjust-wavelength"
            if not args.no_sky_lsf_adjustment: cmd += " --adjust-lsf"

            runcmd(cmd, inputs=[framefile, fiberflatfile], outputs=[
                skyfile,
            ])

            #- sframe = flatfielded sky-subtracted but not flux calibrated frame
            #- Note: this re-reads and re-does steps previously done for picking
            #- sky fibers; desi_proc is about human efficiency,
            #- not I/O or CPU efficiency...
            sframefile = desispec.io.findfile('sframe', args.night, args.expid,
                                              camera)
            if not os.path.exists(sframefile):
                frame = desispec.io.read_frame(framefile)
                fiberflat = desispec.io.read_fiberflat(fiberflatfile)
                sky = desispec.io.read_sky(skyfile)
                apply_fiberflat(frame, fiberflat)
                subtract_sky(frame, sky, apply_throughput_correction=True)
                frame.meta['IN_SKY'] = shorten_filename(skyfile)
                frame.meta['FIBERFLT'] = shorten_filename(fiberflatfile)
                desispec.io.write_frame(sframefile, frame)

        timer.stop('skysub')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Standard Star Fitting

    if args.obstype in ['SCIENCE',] and \
            (not args.noskysub ) and \
            (not args.nostdstarfit) :

        timer.start('stdstarfit')
        if rank == 0:
            log.info('Starting flux calibration at {}'.format(time.asctime()))

        #- Group inputs by spectrograph
        framefiles = dict()
        skyfiles = dict()
        fiberflatfiles = dict()
        night, expid = args.night, args.expid  #- shorter
        for camera in args.cameras:
            sp = int(camera[1])
            if sp not in framefiles:
                framefiles[sp] = list()
                skyfiles[sp] = list()
                fiberflatfiles[sp] = list()

            framefiles[sp].append(findfile('frame', night, expid, camera))
            skyfiles[sp].append(findfile('sky', night, expid, camera))
            fiberflatfiles[sp].append(input_fiberflat[camera])

        #- Hardcoded stdstar model version
        starmodels = os.path.join(os.getenv('DESI_BASIS_TEMPLATES'),
                                  'stdstar_templates_v2.2.fits')

        #- Fit stdstars per spectrograph (not per-camera)
        spectro_nums = sorted(framefiles.keys())
        ## for sp in spectro_nums[rank::size]:
        for i in range(rank, len(spectro_nums), size):
            sp = spectro_nums[i]

            stdfile = findfile('stdstars', night, expid, spectrograph=sp)
            cmd = "desi_fit_stdstars"
            cmd += " --frames {}".format(' '.join(framefiles[sp]))
            cmd += " --skymodels {}".format(' '.join(skyfiles[sp]))
            cmd += " --fiberflats {}".format(' '.join(fiberflatfiles[sp]))
            cmd += " --starmodels {}".format(starmodels)
            cmd += " --outfile {}".format(stdfile)
            cmd += " --delta-color 0.1"
            if args.maxstdstars is not None:
                cmd += " --maxstdstars {}".format(args.maxstdstars)

            inputs = framefiles[sp] + skyfiles[sp] + fiberflatfiles[sp]
            runcmd(cmd, inputs=inputs, outputs=[stdfile])

        timer.stop('stdstarfit')
        if comm is not None:
            comm.barrier()

    # -------------------------------------------------------------------------
    # - Flux calibration

    if args.obstype in ['SCIENCE'] and \
                (not args.noskysub) and \
                (not args.nofluxcalib):
        timer.start('fluxcalib')

        night, expid = args.night, args.expid  #- shorter
        #- Compute flux calibration vectors per camera
        for camera in args.cameras[rank::size]:
            framefile = findfile('frame', night, expid, camera)
            skyfile = findfile('sky', night, expid, camera)
            spectrograph = int(camera[1])
            stdfile = findfile('stdstars',
                               night,
                               expid,
                               spectrograph=spectrograph)
            calibfile = findfile('fluxcalib', night, expid, camera)

            fiberflatfile = input_fiberflat[camera]

            cmd = "desi_compute_fluxcalibration"
            cmd += " --infile {}".format(framefile)
            cmd += " --sky {}".format(skyfile)
            cmd += " --fiberflat {}".format(fiberflatfile)
            cmd += " --models {}".format(stdfile)
            cmd += " --outfile {}".format(calibfile)
            cmd += " --delta-color-cut 0.1"

            inputs = [framefile, skyfile, fiberflatfile, stdfile]
            runcmd(cmd, inputs=inputs, outputs=[
                calibfile,
            ])

        timer.stop('fluxcalib')
        if comm is not None:
            comm.barrier()

    #-------------------------------------------------------------------------
    #- Applying flux calibration

    if args.obstype in [
            'SCIENCE',
    ] and (not args.noskysub) and (not args.nofluxcalib):

        night, expid = args.night, args.expid  #- shorter

        timer.start('applycalib')
        if rank == 0:
            log.info('Starting cframe file creation at {}'.format(
                time.asctime()))

        for camera in args.cameras[rank::size]:
            framefile = findfile('frame', night, expid, camera)
            skyfile = findfile('sky', night, expid, camera)
            spectrograph = int(camera[1])
            stdfile = findfile('stdstars',
                               night,
                               expid,
                               spectrograph=spectrograph)
            calibfile = findfile('fluxcalib', night, expid, camera)
            cframefile = findfile('cframe', night, expid, camera)

            fiberflatfile = input_fiberflat[camera]

            cmd = "desi_process_exposure"
            cmd += " --infile {}".format(framefile)
            cmd += " --fiberflat {}".format(fiberflatfile)
            cmd += " --sky {}".format(skyfile)
            cmd += " --calib {}".format(calibfile)
            cmd += " --outfile {}".format(cframefile)
            cmd += " --cosmics-nsig 6"
            if args.no_xtalk:
                cmd += " --no-xtalk"

            inputs = [framefile, fiberflatfile, skyfile, calibfile]
            runcmd(cmd, inputs=inputs, outputs=[
                cframefile,
            ])

        if comm is not None:
            comm.barrier()

        timer.stop('applycalib')

    #-------------------------------------------------------------------------
    #- Wrap up

    # if rank == 0:
    #     report = timer.report()
    #     log.info('Rank 0 timing report:\n' + report)

    if comm is not None:
        timers = comm.gather(timer, root=0)
    else:
        timers = [
            timer,
        ]

    if rank == 0:
        stats = desiutil.timer.compute_stats(timers)
        log.info('Timing summary statistics:\n' + json.dumps(stats, indent=2))

        if args.timingfile:
            if os.path.exists(args.timingfile):
                with open(args.timingfile) as fx:
                    previous_stats = json.load(fx)

                #- augment previous_stats with new entries, but don't overwrite old
                for name in stats:
                    if name not in previous_stats:
                        previous_stats[name] = stats[name]

                stats = previous_stats

            tmpfile = args.timingfile + '.tmp'
            with open(tmpfile, 'w') as fx:
                json.dump(stats, fx, indent=2)
            os.rename(tmpfile, args.timingfile)

    if rank == 0:
        log.info('All done at {}'.format(time.asctime()))
Example #34
0
def main(args):

    log = get_logger()

    cmd = [
        'desi_compute_fluxcalibration',
    ]
    for key, value in args.__dict__.items():
        if value is not None:
            cmd += ['--' + key, str(value)]
    cmd = ' '.join(cmd)
    log.info(cmd)

    log.info("read frame")
    # read frame
    frame = read_frame(args.infile)

    # Set fibermask flagged spectra to have 0 flux and variance
    frame = get_fiberbitmasked_frame(frame,
                                     bitmask='flux',
                                     ivar_framemask=True)

    log.info("apply fiberflat")
    # read fiberflat
    fiberflat = read_fiberflat(args.fiberflat)

    # apply fiberflat
    apply_fiberflat(frame, fiberflat)

    log.info("subtract sky")
    # read sky
    skymodel = read_sky(args.sky)

    # subtract sky
    subtract_sky(frame, skymodel)

    log.info("compute flux calibration")

    # read models
    model_flux, model_wave, model_fibers, model_metadata = read_stdstar_models(
        args.models)

    ok = np.ones(len(model_metadata), dtype=bool)

    if args.chi2cut > 0:
        log.info("Apply cut CHI2DOF<{}".format(args.chi2cut))
        ok &= (model_metadata["CHI2DOF"] < args.chi2cut)
    if args.delta_color_cut > 0:
        log.info("Apply cut |delta color|<{}".format(args.delta_color_cut))
        ok &= (np.abs(model_metadata["MODEL_G-R"] - model_metadata["DATA_G-R"])
               < args.delta_color_cut)
    if args.min_color is not None:
        log.info("Apply cut DATA_G-R>{}".format(args.min_color))
        ok &= (model_metadata["DATA_G-R"] > args.min_color)
    if args.chi2cut_nsig > 0:
        # automatically reject stars that ar chi2 outliers
        mchi2 = np.median(model_metadata["CHI2DOF"])
        rmschi2 = np.std(model_metadata["CHI2DOF"])
        maxchi2 = mchi2 + args.chi2cut_nsig * rmschi2
        log.info("Apply cut CHI2DOF<{} based on chi2cut_nsig={}".format(
            maxchi2, args.chi2cut_nsig))
        ok &= (model_metadata["CHI2DOF"] <= maxchi2)

    ok = np.where(ok)[0]
    if ok.size == 0:
        log.error("cuts discarded all stars")
        sys.exit(12)
    nstars = model_flux.shape[0]
    nbad = nstars - ok.size
    if nbad > 0:
        log.warning("discarding %d star(s) out of %d because of cuts" %
                    (nbad, nstars))
        model_flux = model_flux[ok]
        model_fibers = model_fibers[ok]
        model_metadata = model_metadata[:][ok]

    # check that the model_fibers are actually standard stars
    fibermap = frame.fibermap

    ## check whether star fibers from args.models are consistent with fibers from fibermap
    ## if not print the OBJTYPE from fibermap for the fibers numbers in args.models and exit
    fibermap_std_indices = np.where(isStdStar(fibermap))[0]
    if np.any(~np.in1d(model_fibers % 500, fibermap_std_indices)):
        target_colnames, target_masks, survey = main_cmx_or_sv(fibermap)
        colname = target_colnames[0]
        for i in model_fibers % 500:
            log.error(
                "inconsistency with spectrum {}, OBJTYPE={}, {}={} in fibermap"
                .format(i, fibermap["OBJTYPE"][i], colname,
                        fibermap[colname][i]))
        sys.exit(12)

    # Make sure the fibers of interest aren't entirely masked.
    if np.sum(
            np.sum(frame.ivar[model_fibers % 500, :] == 0, axis=1) ==
            frame.nwave) == len(model_fibers):
        log.warning('All standard-star spectra are masked!')
        return

    fluxcalib = compute_flux_calibration(
        frame,
        model_wave,
        model_flux,
        model_fibers % 500,
        highest_throughput_nstars=args.highest_throughput)

    # QA
    if (args.qafile is not None):
        log.info("performing fluxcalib QA")
        # Load
        qaframe = load_qa_frame(args.qafile,
                                frame_meta=frame.meta,
                                flavor=frame.meta['FLAVOR'])
        # Run
        #import pdb; pdb.set_trace()
        qaframe.run_qa('FLUXCALIB', (frame, fluxcalib))
        # Write
        if args.qafile is not None:
            write_qa_frame(args.qafile, qaframe)
            log.info("successfully wrote {:s}".format(args.qafile))
        # Figure(s)
        if args.qafig is not None:
            qa_plots.frame_fluxcalib(args.qafig, qaframe, frame, fluxcalib)

    # write result
    write_flux_calibration(args.outfile, fluxcalib, header=frame.meta)

    log.info("successfully wrote %s" % args.outfile)
Example #35
0
def main(args):
    log = get_logger()

    if (args.fiberflat is None) and (args.sky is None) and (args.calib is
                                                            None):
        log.critical('no --fiberflat, --sky, or --calib; nothing to do ?!?')
        sys.exit(12)

    if (not args.no_tsnr) and (args.calib is None):
        log.critical(
            'need --fiberflat --sky and --calib to compute template SNR')
        sys.exit(12)

    frame = read_frame(args.infile)

    if not args.no_tsnr:
        # tsnr alpha calc. requires uncalibrated + no substraction rame.
        uncalibrated_frame = copy.deepcopy(frame)

    #- Raw scores already added in extraction, but just in case they weren't
    #- it is harmless to rerun to make sure we have them.
    compute_and_append_frame_scores(frame, suffix="RAW")

    if args.cosmics_nsig > 0 and args.sky == None:  # Reject cosmics (otherwise do it after sky subtraction)
        log.info("cosmics ray 1D rejection")
        reject_cosmic_rays_1d(frame, args.cosmics_nsig)

    if args.fiberflat != None:
        log.info("apply fiberflat")
        # read fiberflat
        fiberflat = read_fiberflat(args.fiberflat)

        # apply fiberflat to all fibers
        apply_fiberflat(frame, fiberflat)
        compute_and_append_frame_scores(frame, suffix="FFLAT")
    else:
        fiberflat = None

    if args.no_xtalk:
        zero_ivar = (not args.no_zero_ivar)
    else:
        zero_ivar = False

    if args.sky != None:

        # read sky
        skymodel = read_sky(args.sky)

        if args.cosmics_nsig > 0:

            # use a copy the frame (not elegant but robust)
            copied_frame = copy.deepcopy(frame)

            # first subtract sky without throughput correction
            subtract_sky(copied_frame,
                         skymodel,
                         apply_throughput_correction=False,
                         zero_ivar=zero_ivar)

            # then find cosmics
            log.info("cosmics ray 1D rejection after sky subtraction")
            reject_cosmic_rays_1d(copied_frame, args.cosmics_nsig)

            # copy mask
            frame.mask = copied_frame.mask

            # and (re-)subtract sky, but just the correction term
            subtract_sky(frame,
                         skymodel,
                         apply_throughput_correction=(
                             not args.no_sky_throughput_correction),
                         zero_ivar=zero_ivar)

        else:
            # subtract sky
            subtract_sky(frame,
                         skymodel,
                         apply_throughput_correction=(
                             not args.no_sky_throughput_correction),
                         zero_ivar=zero_ivar)

        compute_and_append_frame_scores(frame, suffix="SKYSUB")

    if not args.no_xtalk:
        log.info("fiber crosstalk correction")
        correct_fiber_crosstalk(frame, fiberflat)

        if not args.no_zero_ivar:
            frame.ivar *= (frame.mask == 0)

    if args.calib != None:
        log.info("calibrate")
        # read calibration
        fluxcalib = read_flux_calibration(args.calib)
        # apply calibration
        apply_flux_calibration(frame, fluxcalib)

        # Ensure that ivars are set to 0 for all values if any designated
        # fibermask bit is set. Also flips a bits for each frame.mask value using specmask.BADFIBER
        frame = get_fiberbitmasked_frame(
            frame, bitmask="flux", ivar_framemask=(not args.no_zero_ivar))
        compute_and_append_frame_scores(frame, suffix="CALIB")

    if not args.no_tsnr:
        log.info("calculating tsnr")
        results, alpha = calc_tsnr2(uncalibrated_frame,
                                    fiberflat=fiberflat,
                                    skymodel=skymodel,
                                    fluxcalib=fluxcalib,
                                    alpha_only=args.alpha_only)

        frame.meta['TSNRALPH'] = alpha

        comments = {k: "from calc_frame_tsnr" for k in results.keys()}
        append_frame_scores(frame, results, comments, overwrite=True)

    # record inputs
    frame.meta['IN_FRAME'] = shorten_filename(args.infile)
    frame.meta['FIBERFLT'] = shorten_filename(args.fiberflat)
    frame.meta['IN_SKY'] = shorten_filename(args.sky)
    frame.meta['IN_CALIB'] = shorten_filename(args.calib)

    # save output
    write_frame(args.outfile, frame, units='10**-17 erg/(s cm2 Angstrom)')
    log.info("successfully wrote %s" % args.outfile)