示例#1
0
def agncalibrate(img, outfile, calfile, specformat='lcogt'):

    
    #set up some files that will be needed
    logfile='specext.log'

    hdu = fits.open(img)
    w1 = hdu[0].header['CRVAL1']
    p1 = hdu[0].header['CRPIX1']
    dw = hdu[0].header['CD1_1']
    f = hdu[0].data[0][0]
    e = hdu[0].data[3][0]
    xarr = np.arange(len(f))
    w = (xarr)*dw+w1


    cal_spectra=st.readspectrum(calfile, error=False, ftype='ascii')
    airmass=hdu[0].header['AIRMASS']
    exptime=hdu[0].header['EXPTIME']
    extfile=iraf.osfn("pysalt$data/site/suth_extinct.dat")
    ext_spectra=st.readspectrum(extfile, error=False, ftype='ascii')

    flux_spec=Spectrum.Spectrum(w, f, e, stype='continuum')
    flux_spec=calfunc(flux_spec, cal_spectra, ext_spectra, airmass, exptime, True)
    hdu[0].data[0][0] = flux_spec.flux
    hdu[0].data[3][0] = flux_spec.var
    hdu.writeto(outfile, clobber=True)
示例#2
0
def extract_spectra(hdu, yc, dy, outfile, ext=1, minsize=5, thresh=3, grow=0, smooth=False, maskzeros=False, 
                    convert=True,  cleanspectra=True, calfile=None, clobber=True, specformat='ascii'):
    """From an image, extract a spectra.   

    """
    data=hdu[ext].data

    #replace the zeros with the average from the frame
    if maskzeros:
       mean,std=iterstat(data[data>0])
       #rdata=mean  np.random.normal(mean, std, size=data.shape)
       data[data<=0]=mean #rdata[data<=0]

    y1=yc-dy
    y2=yc+dy
    ap_list=extract(hdu, method='normal', section=[(y1,y2)], minsize=minsize, thresh=thresh, convert=convert)
    sy1a=y2
    sy2a=sy1a+2.0*dy
    ska_list=extract(hdu, method='normal', section=[(sy1a,sy2a)], minsize=minsize, thresh=thresh, convert=convert)
    sy2b=y1-dy
    sy1b=sy2b-2.0*dy
    skb_list=extract(hdu, method='normal', section=[(sy1b,sy2b)], minsize=minsize, thresh=thresh, convert=convert)
    print sy1b, sy2b

    sdata = 0.5*(ska_list[0].ldata/(sy2a-sy1a) + skb_list[0].ldata/(sy2b-sy1b))
    #sdata = ska_list[0].ldata/(sy2a-sy1a)
    #sdata = skb_list[0].ldata/(sy2b-sy1b)
    raw = 1.0 * ap_list[0].ldata
    print 'extract:', ap_list[0].ldata[1124]
    ap_list[0].ldata=ap_list[0].ldata-float(y2-y1) * sdata
    print 'sky:', ap_list[0].ldata[1124]
 
    print ap_list[0].wave[10], ap_list[0].ldata[10], ap_list[0].lvar[10]
    flux_spec=Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
    print flux_spec.wavelength[10], flux_spec.flux[10], flux_spec.var[10]

    if cleanspectra:
       clean_spectra(ap_list[0], grow=grow)
    print 'clean:', ap_list[0].ldata[1124]

    if calfile:
           cal_spectra=st.readspectrum(calfile, error=False, ftype='ascii')
           airmass=hdu[0].header['AIRMASS']
           exptime=hdu[0].header['EXPTIME']
           extfile=os.path.dirname(st.__file__)+"/suth_extinct.dat"
           print extfile
           ext_spectra=st.readspectrum(extfile, error=False, ftype='ascii')

           flux_spec=Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
           print flux_spec.flux[10], flux_spec.var[10]
           flux_spec=calfunc(flux_spec, cal_spectra, ext_spectra, airmass, exptime, True)
           print flux_spec.flux[10], flux_spec.var[10]
    else:
        flux_spec = Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
    
    if specformat == 'ascii':
        write_ascii(outfile, flux_spec, clobber=clobber)
    elif specformat == 'lcogt':
        write_lcogt(outfile, flux_spec, hdu, sky=float(y2-y1) * sdata, raw = raw, clobber=clobber)
示例#3
0
文件: specsens.py 项目: dr-jpk/pysalt
def specsens(specfile, outfile, stdfile, extfile, airmass=None, exptime=None,
                stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5, 
                clobber=True, logfile='salt.log',verbose=True):

   with logging(logfile,debug) as log:

       #read in the specfile and create a spectrum object
       obs_spectra=st.readspectrum(specfile, error=True, ftype='ascii')

       #read in the std file and convert from magnitudes to fnu
       #then convert it to fwave (ergs/s/cm2/A)
       std_spectra=st.readspectrum(stdfile, error=False, ftype='ascii')
       std_spectra.flux=Spectrum.magtoflux(std_spectra.flux, stdzp)
       std_spectra.flux=Spectrum.fnutofwave(std_spectra.wavelength, std_spectra.flux)


       #read in the extinction file (leave in magnitudes)
       ext_spectra=st.readspectrum(extfile, error=False, ftype='ascii')

       #determine the airmass if not specified
       if saltio.checkfornone(airmass) is None:
           message='Airmass was not supplied'
           raise SALTSpecError(message)

       #determine the exptime if not specified
       if saltio.checkfornone(airmass) is None:
           message='Exposure Time was not supplied'
           raise SALTSpecError(message)
          
       #calculate the calibrated spectra
       log.message('Calculating the calibration curve for %s' % specfile)
       cal_spectra=sensfunc(obs_spectra, std_spectra, ext_spectra, airmass, exptime)

       #fit the spectra--first take a first cut of the spectra
       #using the median absolute deviation to throw away bad points
       cmed=np.median(cal_spectra.flux)
       cmad=saltstat.mad(cal_spectra.flux)
       mask=(abs(cal_spectra.flux-cmed)<thresh*cmad)
       mask=(cal_spectra.flux>0)

       #now fit the data
       fit=interfit(cal_spectra.wavelength[mask], cal_spectra.flux[mask], function=function, order=order, thresh=thresh, niter=niter)
       fit.interfit()

       #print 'plotting...'
       #figure()
       #plot(cal_spectra.wavelength, cal_spectra.flux)
       #plot(obs_spectra.wavelength, obs_spectra.flux*cal_spectra.flux.mean()/obs_spectra.flux.mean())
       #plot(std_spectra.wavelength, std_spectra.flux*cal_spectra.flux.mean()/std_spectra.flux.mean())
       #plot(cal_spectra.wavelength, fit(cal_spectra.wavelength))
       #show()

       #write the spectra out
       cal_spectra.flux=fit(cal_spectra.wavelength)
       st.writespectrum(cal_spectra, outfile, ftype='ascii')
示例#4
0
def speccal(specfile, outfile, calfile, extfile, airmass=None, exptime=None,
            clobber=True, logfile='salt.log', verbose=True):

    with logging(logfile, debug) as log:

        # read in the specfile and create a spectrum object
        obs_spectra = st.readspectrum(specfile, error=True, ftype='ascii')

        # read in the std file and convert from magnitudes to fnu
        # then convert it to fwave (ergs/s/cm2/A)
        cal_spectra = st.readspectrum(calfile, error=False, ftype='ascii')

        # read in the extinction file (leave in magnitudes)
        ext_spectra = st.readspectrum(extfile, error=False, ftype='ascii')

        # determine the airmass if not specified
        if saltio.checkfornone(airmass) is None:
            message = 'Airmass was not supplied'
            raise SALTSpecError(message)

        # determine the exptime if not specified
        if saltio.checkfornone(airmass) is None:
            message = 'Exposure Time was not supplied'
            raise SALTSpecError(message)

        # calculate the calibrated spectra
        log.message('Calculating the calibration curve for %s' % specfile)
        error = False
        try:
            if obs_spectra.var is not None:
                error = True
        except:
            error = False
        flux_spectra = calfunc(
            obs_spectra,
            cal_spectra,
            ext_spectra,
            airmass,
            exptime,
            error)

        # write the spectra out
        st.writespectrum(flux_spectra, outfile, ftype='ascii', error=error)
示例#5
0
def speccal(
    specfile, outfile, calfile, extfile, airmass=None, exptime=None, clobber=True, logfile="salt.log", verbose=True
):

    with logging(logfile, debug) as log:

        # read in the specfile and create a spectrum object
        obs_spectra = st.readspectrum(specfile, error=True, ftype="ascii")

        # read in the std file and convert from magnitudes to fnu
        # then convert it to fwave (ergs/s/cm2/A)
        cal_spectra = st.readspectrum(calfile, error=False, ftype="ascii")

        # read in the extinction file (leave in magnitudes)
        ext_spectra = st.readspectrum(extfile, error=False, ftype="ascii")

        # determine the airmass if not specified
        if saltio.checkfornone(airmass) is None:
            message = "Airmass was not supplied"
            raise SALTSpecError(message)

        # determine the exptime if not specified
        if saltio.checkfornone(airmass) is None:
            message = "Exposure Time was not supplied"
            raise SALTSpecError(message)

        # calculate the calibrated spectra
        log.message("Calculating the calibration curve for %s" % specfile)
        error = False
        try:
            if obs_spectra.var is not None:
                error = True
        except:
            error = False
        flux_spectra = calfunc(obs_spectra, cal_spectra, ext_spectra, airmass, exptime, error)

        # write the spectra out
        st.writespectrum(flux_spectra, outfile, ftype="ascii", error=error)
示例#6
0
def normalize_now(filenameblue,
                  filenamered,
                  redfile,
                  plotall=True,
                  extinct_correct=False):
    #Read in the observed spectrum
    obs_spectrablue, airmass, exptime, dispersion = st.readspectrum(
        filenameblue)
    datalistblue = fits.open(filenameblue)

    if redfile:
        obs_spectrared, airmassred, exptimered, dispersionred = st.readspectrum(
            filenamered)

    #Extinction correction
    if extinct_correct:
        print 'Extinction correcting spectra.'
        plt.clf()
        plt.plot(obs_spectrablue.warr, obs_spectrablue.opfarr)
        obs_spectrablue.opfarr = st.extinction_correction(
            obs_spectrablue.warr, obs_spectrablue.opfarr, airmass)
        obs_spectrablue.farr = st.extinction_correction(
            obs_spectrablue.warr, obs_spectrablue.farr, airmass)
        obs_spectrablue.sky = st.extinction_correction(obs_spectrablue.warr,
                                                       obs_spectrablue.sky,
                                                       airmass)
        obs_spectrablue.sigma = st.extinction_correction(
            obs_spectrablue.warr, obs_spectrablue.sigma, airmass)
        plt.plot(obs_spectrablue.warr, obs_spectrablue.opfarr)
        plt.show()

        if redfile:
            plt.clf()
            plt.plot(obs_spectrared.warr, obs_spectrared.opfarr)
            obs_spectrared.opfarr = st.extinction_correction(
                obs_spectrared.warr, obs_spectrared.opfarr, airmassred)
            obs_spectrared.farr = st.extinction_correction(
                obs_spectrared.warr, obs_spectrared.farr, airmassred)
            obs_spectrared.sky = st.extinction_correction(
                obs_spectrared.warr, obs_spectrared.sky, airmassred)
            obs_spectrared.sigma = st.extinction_correction(
                obs_spectrared.warr, obs_spectrared.sigma, airmassred)
            plt.plot(obs_spectrared.warr, obs_spectrared.opfarr)
            plt.show()

    #Read in measured FWHM from header. This is used to convolve the model spectrum.
    FWHMpix = datalistblue[0].header['specfwhm']
    FWHM = FWHMpix * (obs_spectrablue.warr[-1] -
                      obs_spectrablue.warr[0]) / len(obs_spectrablue.warr)

    #Read in DA model
    cwd = os.getcwd()
    os.chdir(
        '/afs/cas.unc.edu/depts/physics_astronomy/clemens/students/group/modelfitting/Koester_08'
    )
    dafile = 'da12500_800.dk'
    mod_wav, mod_spec = np.genfromtxt(dafile, unpack=True, skip_header=33)
    os.chdir(cwd)  #Move back to directory with observed spectra

    #Convolve the model to match the seeing of the spectrum
    intlambda = np.divide(range(31000), 10.) + 3660.0
    lowlambda = np.min(np.where(mod_wav > 3600.))
    highlambda = np.min(np.where(mod_wav > 6800.))
    shortlambdas = mod_wav[lowlambda:highlambda]
    shortinten = mod_spec[lowlambda:highlambda]
    interp = inter.InterpolatedUnivariateSpline(shortlambdas, shortinten, k=1)
    intflux = interp(intlambda)
    sig = FWHM / (2. * np.sqrt(2. * np.log(2.)))
    gx = np.divide(range(360), 10.)
    gauss = (1. / (sig * np.sqrt(2. * np.pi))) * np.exp(-(gx - 18.)**2. /
                                                        (2. * sig**2.))
    gf = np.divide(np.outer(intflux, gauss), 10.)
    length = len(intflux) - 360.
    cflux = np.zeros(length)
    clambda = intlambda[180:len(intlambda) - 180]
    x = 0
    while x < length:
        cflux[x] = np.sum(np.diagonal(gf, x, axis1=1, axis2=0), dtype='d')
        x += 1
    interp2 = inter.InterpolatedUnivariateSpline(clambda, cflux, k=1)
    cflux2blue = interp2(obs_spectrablue.warr)
    cflux2blue /= 10**13.  #Divide by 10**13 to scale
    if redfile:
        cflux2red = interp2(obs_spectrared.warr)
        cflux2red /= 10**13.  #Divide by 10**13 to scale

    #plt.clf()
    #plt.plot(obs_spectrablue.warr,obs_spectrablue.opfarr,'b')
    #plt.plot(obs_spectrablue.warr,cflux2blue,'r')
    #if redfile:
    #    plt.plot(obs_spectrared.warr,obs_spectrared.opfarr,'b')
    #    plt.plot(obs_spectrared.warr,cflux2red,'r')
    #plt.show()

    #The response function is the observed spectrum divided by the model spectrum.
    response_blue = obs_spectrablue.opfarr / cflux2blue
    if redfile:
        response_red = obs_spectrared.opfarr / cflux2red
    '''
    plt.clf()
    plt.plot(obs_spectrablue.warr,response_blue,'k')
    if redfile:
        plt.plot(obs_spectrared.warr,response_red,'k')
    plt.show()
    '''

    #We want to mask out the Balmer line features, and the telluric line in the red spectrum. Set up the wavelength ranges to mask here.
    #balmer_features_blue = [[3745,3757],[3760,3780],[3784,3812],[3816,3856],[3865,3921],[3935,4021],[4040,4191],[4223,4460],[4691,5010]] #Keeping ends
    balmer_features_blue = [[3400, 3700], [3745, 3757], [3760, 3780],
                            [3784, 3812], [3816, 3856], [3865, 3921],
                            [3935, 4021], [4040, 4191], [4223, 4460],
                            [4691, 5010], [5140, 5500]]  #Discarding ends
    balmer_features_red = [[6350, 6780], [6835, 6970]]

    balmer_mask_blue = obs_spectrablue.warr == obs_spectrablue.warr
    for wavrange in balmer_features_blue:
        inds = np.where((obs_spectrablue.warr > wavrange[0])
                        & (obs_spectrablue.warr < wavrange[1]))
        balmer_mask_blue[inds] = False

    if redfile:
        balmer_mask_red = obs_spectrared.warr == obs_spectrared.warr
        for wavrange in balmer_features_red:
            indxs = np.where((obs_spectrared.warr > wavrange[0])
                             & (obs_spectrared.warr < wavrange[1]))
            balmer_mask_red[indxs] = False

    spec_wav_masked_blue = obs_spectrablue.warr[balmer_mask_blue]
    response_masked_blue = response_blue[balmer_mask_blue]

    if redfile:
        spec_wav_masked_red = obs_spectrared.warr[balmer_mask_red]
        response_masked_red = response_red[balmer_mask_red]

    #Fit the response function with a polynomial. The order of polynomial is specified first.
    response_poly_order_blue = 7.
    response_fit_blue_poly = np.polyfit(spec_wav_masked_blue,
                                        response_masked_blue,
                                        response_poly_order_blue)
    response_fit_blue = np.poly1d(response_fit_blue_poly)

    if redfile:
        response_poly_order_red = 3.
        response_fit_red_poly = np.polyfit(spec_wav_masked_red,
                                           response_masked_red,
                                           response_poly_order_red)
        response_fit_red = np.poly1d(response_fit_red_poly)

    #Save response function
    #np.savetxt('response_model_no_extinction.txt',np.transpose([obs_spectrablue.warr,response_fit_blue(obs_spectrablue.warr),obs_spectrared.warr,response_fit_red(obs_spectrared.warr)]))
    #plt.clf()
    #plt.plot(obs_spectrablue.warr,response_fit_blue(obs_spectrablue.warr)/response_fit_blue(obs_spectrablue.warr)[1000])
    #plt.show()
    #exit()
    if plotall:
        plt.clf()
        plt.plot(obs_spectrablue.warr, response_blue, 'r')
        plt.plot(spec_wav_masked_blue, response_masked_blue, 'g.')
        plt.plot(obs_spectrablue.warr, response_fit_blue(obs_spectrablue.warr),
                 'k--')
        #plt.show()

        #plt.clf()
        if redfile:
            plt.plot(obs_spectrared.warr, response_red, 'r')
            plt.plot(spec_wav_masked_red, response_masked_red, 'g.')
            plt.plot(obs_spectrared.warr,
                     response_fit_red(obs_spectrared.warr), 'k--')
        plt.show()

    #Divide by the fit to the response function to get the continuum normalized spectra. Divide every extension by the same polynomial
    fcorr_wd_blue_opfarr = obs_spectrablue.opfarr / response_fit_blue(
        obs_spectrablue.warr)
    fcorr_wd_blue_farr = obs_spectrablue.farr / response_fit_blue(
        obs_spectrablue.warr)
    fcorr_wd_blue_sky = obs_spectrablue.sky / response_fit_blue(
        obs_spectrablue.warr)
    fcorr_wd_blue_sigma = obs_spectrablue.sigma / response_fit_blue(
        obs_spectrablue.warr)

    if redfile:
        fcorr_wd_red_opfarr = obs_spectrared.opfarr / response_fit_red(
            obs_spectrared.warr)
        fcorr_wd_red_farr = obs_spectrared.farr / response_fit_red(
            obs_spectrared.warr)
        fcorr_wd_red_sky = obs_spectrared.sky / response_fit_red(
            obs_spectrared.warr)
        fcorr_wd_red_sigma = obs_spectrared.sigma / response_fit_red(
            obs_spectrared.warr)

    if plotall:
        plt.clf()
        plt.plot(obs_spectrablue.warr, fcorr_wd_blue_opfarr, 'b')
        if redfile:
            plt.plot(obs_spectrared.warr, fcorr_wd_red_opfarr, 'r')
        plt.show()
    #exit()

    #Save parameters for diagnostics
    if redfile:
        bigarray = np.zeros([len(obs_spectrablue.warr), 12])
        bigarray[0:len(obs_spectrablue.warr), 0] = obs_spectrablue.warr
        bigarray[0:len(response_blue), 1] = response_blue
        bigarray[0:len(spec_wav_masked_blue), 2] = spec_wav_masked_blue
        bigarray[0:len(response_masked_blue), 3] = response_masked_blue
        bigarray[0:len(response_fit_blue(obs_spectrablue.warr)),
                 4] = response_fit_blue(obs_spectrablue.warr)
        bigarray[0:len(fcorr_wd_blue_opfarr), 5] = fcorr_wd_blue_opfarr
        bigarray[0:len(obs_spectrared.warr), 6] = obs_spectrared.warr
        bigarray[0:len(response_red), 7] = response_red
        bigarray[0:len(spec_wav_masked_red), 8] = spec_wav_masked_red
        bigarray[0:len(response_masked_red), 9] = response_masked_red
        bigarray[0:len(response_fit_red(obs_spectrared.warr)),
                 10] = response_fit_red(obs_spectrared.warr)
        bigarray[0:len(fcorr_wd_red_opfarr), 11] = fcorr_wd_red_opfarr
        now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
        endpoint = '_930'
        with open(
                'continuum_normalization_' +
                filenameblue[5:filenameblue.find(endpoint)] + '_' + now +
                '.txt', 'a') as handle:
            header = str(filenameblue) + ',' + str(
                filenamered
            ) + ',' + dafile + '\n Columns structured as blue then red. If no red file, only blue data given. Columns are: blue wavelengths, blue response all data, blue masked wavelengths, blue masked response data, blue response fit, blue continuum-normalize flux, red wavelengths, red response all data, red masked wavelengths, red masked response data, red response fit, red continuum-normalized flux'
            np.savetxt(handle, bigarray, fmt='%f', header=header)
    if not redfile:
        bigarray = np.zeros([len(obs_spectrablue.warr), 6])
        bigarray[0:len(obs_spectrablue.warr), 0] = obs_spectrablue.warr
        bigarray[0:len(response_blue), 1] = response_blue
        bigarray[0:len(spec_wav_masked_blue), 2] = spec_wav_masked_blue
        bigarray[0:len(response_masked_blue), 3] = response_masked_blue
        bigarray[0:len(response_fit_blue(obs_spectrablue.warr)),
                 4] = response_fit_blue(obs_spectrablue.warr)
        bigarray[0:len(fcorr_wd_blue_opfarr), 5] = fcorr_wd_blue_opfarr
        now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
        endpoint = '_930'
        with open(
                'continuum_normalization_' +
                filenameblue[5:filenameblue.find(endpoint)] + '_' + now +
                '.txt', 'a') as handle:
            header = str(
                filenameblue
            ) + ',' + ',' + dafile + '\n Columns structured as blue then red. If no red file, only blue data given. Columns are: blue wavelengths, blue response all data, blue masked wavelengths, blue masked response data, blue response fit, blue continuum-normalized flux'
            np.savetxt(handle, bigarray, fmt='%f', header=header)

    #Save the continuum normalized spectra here.
    Ni = 4.  #Number of extensions
    Nx1 = len(fcorr_wd_blue_opfarr)
    if redfile:
        Nx2 = len(fcorr_wd_red_opfarr)
    Ny = 1.  #All 1D spectra

    #Update header
    header1 = st.readheader(filenameblue)
    header1.set('STANDARD', dafile, 'DA Model for Continuum Calibration')
    header1.set('RESPPOLY', response_poly_order_blue,
                'Polynomial order for Response Function')
    header1.set('DATENORM',
                datetime.datetime.now().strftime("%Y-%m-%d"),
                'Date of Continuum Normalization')

    data1 = np.empty(shape=(Ni, Ny, Nx1))
    data1[0, :, :] = fcorr_wd_blue_opfarr
    data1[1, :, :] = fcorr_wd_blue_farr
    data1[2, :, :] = fcorr_wd_blue_sky
    data1[3, :, :] = fcorr_wd_blue_sigma

    #Check that filename does not already exist. Prompt user for input if it does.
    loc1 = filenameblue.find('.ms.fits')
    newname1 = filenameblue[0:loc1] + '_flux_model_short.ms.fits'
    clob = False
    mylist = [True for f in os.listdir('.') if f == newname1]
    exists = bool(mylist)

    if exists:
        print 'File %s already exists.' % newname1
        nextstep = raw_input(
            'Do you want to overwrite or designate a new name (overwrite/new)? '
        )
        if nextstep == 'overwrite':
            clob = True
            exists = False
        elif nextstep == 'new':
            newname1 = raw_input('New file name: ')
            exists = False
        else:
            exists = False
    print 'Writing ', newname1
    newim1 = fits.PrimaryHDU(data=data1, header=header1)
    newim1.writeto(newname1, clobber=clob)

    #Save the red file if it exists.
    if redfile:
        header2 = st.readheader(filenamered)
        header2.set('STANDARD', dafile, 'DA Model for Continuum Calibration')
        header2.set('RESPPOLY', response_poly_order_red,
                    'Polynomial order for Response Function')
        header2.set('DATENORM',
                    datetime.datetime.now().strftime("%Y-%m-%d"),
                    'Date of Continuum Normalization')
        data2 = np.empty(shape=(Ni, Ny, Nx2))
        data2[0, :, :] = fcorr_wd_red_opfarr
        data2[1, :, :] = fcorr_wd_red_farr
        data2[2, :, :] = fcorr_wd_red_sky
        data2[3, :, :] = fcorr_wd_red_sigma

        loc2 = filenamered.find('.ms.fits')
        newname2 = filenamered[0:loc2] + '_flux_model_short.ms.fits'
        clob = False
        mylist = [True for f in os.listdir('.') if f == newname2]
        exists = bool(mylist)

        if exists:
            print 'File %s already exists.' % newname2
            nextstep = raw_input(
                'Do you want to overwrite or designate a new name (overwrite/new)? '
            )
            if nextstep == 'overwrite':
                clob = True
                exists = False
            elif nextstep == 'new':
                newname2 = raw_input('New file name: ')
                exists = False
            else:
                exists = False
        print 'Writing ', newname2
        newim2 = fits.PrimaryHDU(data=data2, header=header2)
        newim2.writeto(newname2, clobber=clob)
示例#7
0
def extract_spectra(hdu, yc, dy, outfile, minsize=5, thresh=3, grow=0, smooth=False, maskzeros=False, 
                    convert=True,  cleanspectra=True, calfile=None, clobber=True, specformat='ascii'):
    """From an image, extract a spectra.   

    """
    data=hdu[1].data

    #replace the zeros with the average from the frame
    if maskzeros:
       mean,std=iterstat(data[data>0])
       #rdata=mean  np.random.normal(mean, std, size=data.shape)
       data[data<=0]=mean #rdata[data<=0]

    y1=yc-dy
    y2=yc+dy

    #sy1=y2-2*dy
    #sy2=y2+2*dy

    #sdata = 1.0 * data 
    #y,x = np.indices(sdata.shape)
    #for i in range(sdata.shape[1]):
    #   mask=(hdu[3].data[:,i]==0)
    #   mask[sy1:sy2] = 0
    #   if mask.sum()>0:
    #     sdata[y1:y2,i] = np.interp(y[y1:y2,i], y[:,i][mask], data[:,i][mask])  
    #hdu[1].data = sdata
    #sk_list=extract(hdu, method='normal', section=[(y1,y2)], minsize=minsize, thresh=thresh, convert=True)
    #ap_list[0].ldata=ap_list[0].ldata-sk_list[0].ldata
    #ap_list[0].ldata=ap_list[0].ldata-float(y2-y1)/(sy2-sy1)*sk_list[0].ldata

    ap_list=extract(hdu, method='normal', section=[(y1,y2)], minsize=minsize, thresh=thresh, convert=convert)
    sy1a=y2
    sy2a=sy1a+2.0*dy
    ska_list=extract(hdu, method='normal', section=[(sy1a,sy2a)], minsize=minsize, thresh=thresh, convert=convert)
    sy2b=y1-dy
    sy1b=sy2b-2.0*dy
    skb_list=extract(hdu, method='normal', section=[(sy1b,sy2b)], minsize=minsize, thresh=thresh, convert=convert)
    print sy1b, sy2b

    sdata = 0.5*(ska_list[0].ldata/(sy2a-sy1a) + skb_list[0].ldata/(sy2b-sy1b))
    #sdata = ska_list[0].ldata/(sy2a-sy1a)
    #sdata = skb_list[0].ldata/(sy2b-sy1b)
    raw = 1.0 * ap_list[0].ldata
    
    ap_list[0].ldata=ap_list[0].ldata-float(y2-y1) * sdata
 
    print ap_list[0].wave[10], ap_list[0].ldata[10], ap_list[0].lvar[10]
    flux_spec=Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
    print flux_spec.wavelength[10], flux_spec.flux[10], flux_spec.var[10]

    if cleanspectra:
       clean_spectra(ap_list[0], grow=grow)

    if calfile:
           cal_spectra=st.readspectrum(calfile, error=False, ftype='ascii')
           airmass=hdu[0].header['AIRMASS']
           exptime=hdu[0].header['EXPTIME']
           extfile=iraf.osfn("pysalt$data/site/suth_extinct.dat")
           ext_spectra=st.readspectrum(extfile, error=False, ftype='ascii')

           flux_spec=Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
           print flux_spec.flux[10], flux_spec.var[10]
           flux_spec=calfunc(flux_spec, cal_spectra, ext_spectra, airmass, exptime, True)
           print flux_spec.flux[10], flux_spec.var[10]
    else:
        flux_spec = Spectrum.Spectrum(ap_list[0].wave, ap_list[0].ldata, abs(ap_list[0].lvar)**0.5, stype='continuum')
    
    if specformat == 'ascii':
        write_ascii(outfile, flux_spec, clobber=clobber)
    elif specformat == 'lcogt':
        write_lcogt(outfile, flux_spec, hdu, sky=float(y2-y1) * sdata, raw = raw, clobber=clobber)
示例#8
0
def specsens(specfile, outfile, stdfile, extfile, airmass=None, exptime=None,
             stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5,
             fitter='gaussian', clobber=True, logfile='salt.log', verbose=True):

    with logging(logfile, debug) as log:

        # read in the specfile and create a spectrum object
        obs_spectra = st.readspectrum(specfile.strip(), error=True, ftype='ascii')

        # smooth the observed spectrum
        # read in the std file and convert from magnitudes to fnu
        # then convert it to fwave (ergs/s/cm2/A)
        std_spectra = st.readspectrum(stdfile.strip(), error=False, ftype='ascii')
        std_spectra.flux = Spectrum.magtoflux(std_spectra.flux, stdzp)
        std_spectra.flux = Spectrum.fnutofwave(
            std_spectra.wavelength, std_spectra.flux)

        # Get the typical bandpass of the standard star,
        std_bandpass = np.diff(std_spectra.wavelength).mean()
        # Smooth the observed spectrum to that bandpass
        obs_spectra.flux = st.boxcar_smooth(obs_spectra, std_bandpass)
        # read in the extinction file (leave in magnitudes)
        ext_spectra = st.readspectrum(extfile.strip(), error=False, ftype='ascii')

        # determine the airmass if not specified
        if saltio.checkfornone(airmass) is None:
            message = 'Airmass was not supplied'
            raise SALTSpecError(message)

        # determine the exptime if not specified
        if saltio.checkfornone(exptime) is None:
            message = 'Exposure Time was not supplied'
            raise SALTSpecError(message)

        # calculate the calibrated spectra
        log.message('Calculating the calibration curve for %s' % specfile)
        cal_spectra = sensfunc(
            obs_spectra, std_spectra, ext_spectra, airmass, exptime)

        # plot(cal_spectra.wavelength, cal_spectra.flux * std_spectra.flux)
        # fit the spectra--first take a first cut of the spectra
        # using the median absolute deviation to throw away bad points
        cmed = np.median(cal_spectra.flux)
        cmad = saltstat.mad(cal_spectra.flux)
        mask = (abs(cal_spectra.flux - cmed) < thresh * cmad)
        mask = np.logical_and(mask, (cal_spectra.flux > 0))

        # now fit the data
        # Fit using a gaussian process.
        if fitter=='gaussian':
            from sklearn.gaussian_process import GaussianProcess
            #Instanciate a Gaussian Process model

            dy = obs_spectra.var[mask] ** 0.5
            dy /= obs_spectra.flux[mask] / cal_spectra.flux[mask]
            y = cal_spectra.flux[mask]
            gp = GaussianProcess(corr='squared_exponential', theta0=1e-2,
                                 thetaL=1e-4, thetaU=0.1, nugget=(dy / y) ** 2.0)
            X = np.atleast_2d(cal_spectra.wavelength[mask]).T
            # Fit to data using Maximum Likelihood Estimation of the parameters
            gp.fit(X, y)
    
            x = np.atleast_2d(cal_spectra.wavelength).T
            # Make the prediction on the meshed x-axis (ask for MSE as well)
            y_pred = gp.predict(x)

            cal_spectra.flux = y_pred

        else:
            fit=interfit(cal_spectra.wavelength[mask], cal_spectra.flux[mask], function=function, order=order, thresh=thresh, niter=niter)
            fit.interfit()
            cal_spectra.flux=fit(cal_spectra.wavelength)

        # write the spectra out
        st.writespectrum(cal_spectra, outfile, ftype='ascii')
import os
import sys
import datetime


if len(sys.argv) == 3:
    script, filenameblue, filenamered = sys.argv
    redfile = True
elif len(sys.argv) == 2:
    script, filenameblue = sys.argv
    redfile = False
else:
    print '\n Incorrect number of arguments. \n'

#Read in the observed spectrum
obs_spectrablue,airmass,exptime,dispersion = st.readspectrum(filenameblue)
datalistblue = fits.open(filenameblue)


if redfile:
    obs_spectrared, airmassred,exptimered,dispersionred = st.readspectrum(filenamered)
    

#Read in measured FWHM from header. This is used to convolve the model spectrum.
FWHMpix = datalistblue[0].header['specfwhm']
FWHM = FWHMpix * (obs_spectrablue.warr[-1] - obs_spectrablue.warr[0])/len(obs_spectrablue.warr)



#Read in DA model
cwd = os.getcwd()
示例#10
0
def specsens(specfile,
             outfile,
             stdfile,
             extfile,
             airmass=None,
             exptime=None,
             stdzp=3.68e-20,
             function='polynomial',
             order=3,
             thresh=3,
             niter=5,
             fitter='gaussian',
             clobber=True,
             logfile='salt.log',
             verbose=True):

    with logging(logfile, debug) as log:

        # read in the specfile and create a spectrum object
        obs_spectra = st.readspectrum(specfile.strip(),
                                      error=True,
                                      ftype='ascii')

        # smooth the observed spectrum
        # read in the std file and convert from magnitudes to fnu
        # then convert it to fwave (ergs/s/cm2/A)
        std_spectra = st.readspectrum(stdfile.strip(),
                                      error=False,
                                      ftype='ascii')
        std_spectra.flux = Spectrum.magtoflux(std_spectra.flux, stdzp)
        std_spectra.flux = Spectrum.fnutofwave(std_spectra.wavelength,
                                               std_spectra.flux)

        # Get the typical bandpass of the standard star,
        std_bandpass = np.diff(std_spectra.wavelength).mean()
        # Smooth the observed spectrum to that bandpass
        obs_spectra.flux = st.boxcar_smooth(obs_spectra, std_bandpass)
        # read in the extinction file (leave in magnitudes)
        ext_spectra = st.readspectrum(extfile.strip(),
                                      error=False,
                                      ftype='ascii')

        # determine the airmass if not specified
        if saltio.checkfornone(airmass) is None:
            message = 'Airmass was not supplied'
            raise SALTSpecError(message)

        # determine the exptime if not specified
        if saltio.checkfornone(exptime) is None:
            message = 'Exposure Time was not supplied'
            raise SALTSpecError(message)

        # calculate the calibrated spectra
        log.message('Calculating the calibration curve for %s' % specfile)
        cal_spectra = sensfunc(obs_spectra, std_spectra, ext_spectra, airmass,
                               exptime)

        # plot(cal_spectra.wavelength, cal_spectra.flux * std_spectra.flux)
        # fit the spectra--first take a first cut of the spectra
        # using the median absolute deviation to throw away bad points
        cmed = np.median(cal_spectra.flux)
        cmad = saltstat.mad(cal_spectra.flux)
        mask = (abs(cal_spectra.flux - cmed) < thresh * cmad)
        mask = np.logical_and(mask, (cal_spectra.flux > 0))

        # now fit the data
        # Fit using a gaussian process.
        if fitter == 'gaussian':
            from sklearn.gaussian_process import GaussianProcess
            #Instanciate a Gaussian Process model

            dy = obs_spectra.var[mask]**0.5
            dy /= obs_spectra.flux[mask] / cal_spectra.flux[mask]
            y = cal_spectra.flux[mask]
            gp = GaussianProcess(corr='squared_exponential',
                                 theta0=1e-2,
                                 thetaL=1e-4,
                                 thetaU=0.1,
                                 nugget=(dy / y)**2.0)
            X = np.atleast_2d(cal_spectra.wavelength[mask]).T
            # Fit to data using Maximum Likelihood Estimation of the parameters
            gp.fit(X, y)

            x = np.atleast_2d(cal_spectra.wavelength).T
            # Make the prediction on the meshed x-axis (ask for MSE as well)
            y_pred = gp.predict(x)

            cal_spectra.flux = y_pred

        else:
            fit = interfit(cal_spectra.wavelength[mask],
                           cal_spectra.flux[mask],
                           function=function,
                           order=order,
                           thresh=thresh,
                           niter=niter)
            fit.interfit()
            cal_spectra.flux = fit(cal_spectra.wavelength)

        # write the spectra out
        st.writespectrum(cal_spectra, outfile, ftype='ascii')
    quickcheck = stdflux[onion//2].lower()[1:-4] in stanspec.lower()
    if not quickcheck:
        print 'Check your standard star and flux files. They are mixed up.'
        sys.exit()
    onion += 1
'''
orderused = np.zeros([len(standards)])
senspolys = []
airstd = np.zeros([len(standards)])
allexcluded = [[None] for i in range(len(standards))]

#Calculating the sensitivity function of each standard star
cucumber = 0
for stdspecfile in standards:
    #Read in the observed spectrum of the standard star
    obs_spectra,airmass,exptime,dispersion = st.readspectrum(stdspecfile) #This is an object containing var_farr,farr,sky,sigma,warr
    airstd[cucumber] = airmass
    #plt.clf()
    #plt.plot(obs_spectra.warr,obs_spectra.farr)
    #plt.show()
    #read in the standard file
    placeholder = cucumber // 2
    stdfile = stdflux[placeholder]
    std_spectra = st.readstandard(stdfile)

    #plt.clf()
    #plt.plot(std_spectra.warr,std_spectra.magarr)
    #plt.show()
    #Only keep the part of the standard file that overlaps with observation.
    lowwv = np.where(std_spectra.warr >= np.min(obs_spectra.warr))
    lowwv = np.asarray(lowwv)
示例#12
0
def flux_calibrate_now(stdlist,
                       fluxlist,
                       speclist,
                       extinct_correct=False,
                       masterresp=False):
    if extinct_correct:
        extinctflag = 0
    else:
        extinctflag = -1
    if masterresp:  #Use the master response function
        #Read in master response function and use that.
        cwd = os.getcwd()
        os.chdir(
            '/afs/cas.unc.edu/depts/physics_astronomy/clemens/students/group/standards/response_curves/'
        )
        standards = sorted(glob('*resp*.npy'))

        master_response_blue_in = np.load(standards[0])
        master_response_blue_in_pol = np.poly1d(master_response_blue_in)
        master_response_blue_out = np.load(standards[1])
        master_response_blue_out_pol = np.poly1d(master_response_blue_out)
        master_response_red_in = np.load(standards[2])
        master_response_red_in_pol = np.poly1d(master_response_red_in)
        master_response_red_out = np.load(standards[3])
        master_response_red_out_pol = np.poly1d(master_response_red_out)

        os.chdir(cwd)

        airstd = np.ones([4])
        #airstd[0] = 1.1

        #For saving files correctly
        stdflux = np.array(['mmaster.dat'])
        #standards = np.array([masterlist])
        allexcluded = [[None] for i in range(len(standards))]
        orderused = np.zeros([len(standards)])
        size = 0.

        #Find shift for each night
        #For blue setup: use mean of 4530-4590
        #for red setup: use mean of 6090-6190
        try:
            flux_tonight_list = np.genfromtxt('response_curves.txt', dtype=str)
            print 'Found response_curves.txt file.'
            print flux_tonight_list
            if flux_tonight_list.size == 1:
                flux_tonight_list = np.array([flux_tonight_list])
            for x in flux_tonight_list:
                #print x
                if 'blue' in x.lower():
                    wave_tonight, sens_tonight = np.genfromtxt(x, unpack=True)
                    blue_low_index = np.min(np.where(wave_tonight > 4530.))
                    blue_high_index = np.min(np.where(wave_tonight > 4590.))
                    blue_mean_tonight = np.mean(
                        sens_tonight[blue_low_index:blue_high_index])
                elif 'red' in x.lower():
                    wave_tonight, sens_tonight = np.genfromtxt(x, unpack=True)
                    red_low_index = np.min(np.where(wave_tonight > 6090.))
                    red_high_index = np.min(np.where(wave_tonight > 6190.))
                    red_mean_tonight = np.mean(
                        sens_tonight[red_low_index:red_high_index])
        except:
            print 'No response_curves.txt file found.'
            blue_mean_tonight = None
            red_mean_tonight = None
            flux_tonight_list = ['None', 'None']

    else:  #Use the standard star fluxes in the typical manner
        #Read in each standard star spectrum
        standards = np.genfromtxt(stdlist, dtype=str)
        if standards.size == 1:
            standards = np.array([standards])
        stdflux = np.genfromtxt(fluxlist, dtype=str)
        if stdflux.size == 1:
            stdflux = np.array(
                [stdflux]
            )  #save stdflux explicitly as an array so you can index if only 1 element
        #Check that the files are set up correctly to avoid mixing standards.
        #This checks that the files in liststandard have similar characters to those in listflux and the correct order. But might break if flux file doesn't match. E.G. mcd32d9927.dat is often called CD-32_9927 in our system.
        '''
        onion = 0
        for stanspec in standards:
            quickcheck = stdflux[onion//2].lower()[1:-4] in stanspec.lower()
            if not quickcheck:
                print 'Check your standard star and flux files. They are mixed up.'
                sys.exit()
            onion += 1
        '''
        orderused = np.zeros([len(standards)])
        senspolys = []
        airstd = np.zeros([len(standards)])
        allexcluded = [[None] for i in range(len(standards))]

        #Calculating the sensitivity function of each standard star
        cucumber = 0
        for stdspecfile in standards:
            print stdspecfile
            #Read in the observed spectrum of the standard star
            obs_spectra, airmass, exptime, dispersion = st.readspectrum(
                stdspecfile
            )  #obs_spectra is an object containing opfarr,farr,sky,sigma,warr
            airstd[cucumber] = airmass
            #plt.clf()
            #plt.plot(obs_spectra.warr,obs_spectra.opfarr)
            #plt.show()

            #Do the extinction correction
            if extinct_correct:
                print 'Extinction correcting spectra.'
                plt.clf()
                plt.plot(obs_spectra.warr, obs_spectra.opfarr)
                obs_spectra.opfarr = st.extinction_correction(
                    obs_spectra.warr, obs_spectra.opfarr, airmass)
                plt.plot(obs_spectra.warr, obs_spectra.opfarr)
                #plt.show()

            #Change to the standard star directory
            cwd = os.getcwd()
            os.chdir(
                '/afs/cas.unc.edu/depts/physics_astronomy/clemens/students/group/standards'
            )

            #read in the standard file
            placeholder = cucumber // 2
            stdfile = stdflux[placeholder]
            std_spectra = st.readstandard(stdfile)
            os.chdir(cwd)
            #plt.clf()
            #plt.plot(std_spectra.warr,std_spectra.magarr,'.')
            #plt.show()
            #Only keep the part of the standard file that overlaps with observation.
            lowwv = np.where(std_spectra.warr >= np.min(obs_spectra.warr))
            lowwv = np.asarray(lowwv)
            highwv = np.where(std_spectra.warr <= np.max(obs_spectra.warr))
            highwv = np.asarray(highwv)
            index = np.intersect1d(lowwv, highwv)

            std_spectra.warr = std_spectra.warr[index]
            std_spectra.magarr = std_spectra.magarr[index]
            std_spectra.wbin = std_spectra.wbin[index]

            #Convert from AB mag to fnu, then to fwave (ergs/s/cm2/A)
            stdzp = 3.68e-20  #The absolute flux per unit frequency at an AB mag of zero
            std_spectra.magarr = st.magtoflux(std_spectra.magarr, stdzp)
            std_spectra.magarr = st.fnutofwave(std_spectra.warr,
                                               std_spectra.magarr)

            #plt.clf()
            #plt.plot(std_spectra.warr,std_spectra.magarr,'.')
            #plt.show()
            #np.savetxt('hz4_stan.txt',np.transpose([std_spectra.warr,std_spectra.magarr]))
            #exit()

            #We want to rebin the observed spectrum to match with the bins in the standard file. This makes summing up counts significantly easier.
            #Set the new binning here.
            print 'Starting to rebin: ', stdspecfile
            low = np.rint(np.min(obs_spectra.warr))  #Rounds to nearest integer
            high = np.rint(np.max(obs_spectra.warr))
            size = 0.05  #size in Angstroms you want each bin

            num = (
                high - low
            ) / size + 1.  #number of bins. Must add one to get correct number.
            wavenew = np.linspace(low, high,
                                  num=num)  #wavelength of each new bin

            #Now do the rebinning using Ian Crossfield's rebinning package
            binflux = st.resamplespec(wavenew, obs_spectra.warr,
                                      obs_spectra.opfarr,
                                      200.)  #200 is the oversampling factor
            print 'Done rebinning. Now summing the spectrum into new bins to match', stdfile
            #plt.clf()
            #plt.plot(obs_spectra.warr,obs_spectra.opfarr)
            #plt.plot(wavenew,binflux)
            #plt.show()

            #Now sum the rebinned spectra into the same bins as the standard star file
            counts = st.sum_std(std_spectra.warr, std_spectra.wbin, wavenew,
                                binflux)
            #plt.clf()
            #plt.plot(std_spectra.warr,std_spectra.magarr)
            #plt.plot(obs_spectra.warr,obs_spectra.opfarr,'b')
            #plt.plot(std_spectra.warr,counts,'g+')
            #plt.show()

            #Calculate the sensitivity function
            sens_function = st.sensfunc(counts, std_spectra.magarr, exptime,
                                        std_spectra.wbin, airmass)
            #plt.clf()
            #plt.plot(std_spectra.warr,sens_function)
            #plt.show()
            #sys.exit()
            #Fit a low order polynomial to this function so that it is smooth.
            #The sensitivity function is in units of 2.5 * log10[counts/sec/Ang / ergs/cm2/sec/Ang]
            #Choose regions to not include in fit, first by checking if a mask file exists, and if not the prompt for user interaction.
            if 'blue' in stdspecfile.lower():
                std_mask = stdfile[0:-4] + '_blue_maskasdf.dat'
            if 'red' in stdspecfile.lower():
                std_mask = stdfile[0:-4] + '_red_maskasdf.dat'
            std_mask2 = glob(std_mask)
            if len(std_mask2) == 1.:
                print 'Found mask file.\n'
                mask = np.ones(len(std_spectra.warr))
                excluded_wave = np.genfromtxt(
                    std_mask)  #Read in wavelengths to exclude
                #print excluded_wave
                #print type(excluded_wave)
                #Find index of each wavelength
                excluded = []
                for x in excluded_wave:
                    #print x
                    #print np.where(std_spectra.warr == find_nearest(std_spectra.warr,x))
                    pix_val = np.where(
                        std_spectra.warr == find_nearest(std_spectra.warr, x))
                    excluded.append(pix_val[0][0])
                #print excluded
                lettuce = 0
                while lettuce < len(excluded):
                    mask[excluded[lettuce]:excluded[lettuce + 1] + 1] = 0
                    lettuce += 2
                excluded = np.array(excluded).tolist()
                allexcluded[cucumber] = excluded
                indices = np.where(mask != 0.)
                lambdasfit = std_spectra.warr[indices]
                fluxesfit = sens_function[indices]
            else:
                print 'No mask found. User interaction required.\n'

                global ax, fig, coords
                coords = []
                plt.clf()
                fig = plt.figure(1)
                ax = fig.add_subplot(111)
                ax.plot(std_spectra.warr, sens_function)
                cid = fig.canvas.mpl_connect('button_press_event', onclick)
                print 'Please click on both sides of regions you want to exclude. Then close the plot.'
                plt.title(
                    'Click both sides of regions you want to exclude. Then close the plot.'
                )
                plt.show(1)

                #Mask our the regions you don't want to fit
                #We need make sure left to right clicking and right to left clicking both work.
                mask = np.ones(len(std_spectra.warr))
                excluded = np.zeros(len(coords))
                lettuce = 0
                if len(coords) > 0:
                    while lettuce < len(coords):
                        x1 = np.where(std_spectra.warr == (find_nearest(
                            std_spectra.warr, coords[lettuce][0])))
                        excluded[lettuce] = np.asarray(x1)
                        lettuce += 1
                        x2 = np.where(std_spectra.warr == (find_nearest(
                            std_spectra.warr, coords[lettuce][0])))
                        if x2 < x1:
                            x1, x2 = x2, x1
                        mask[
                            x1[0][0]:x2[0][0] +
                            1] = 0  #have to add 1 here to the second index so that we exclude through that index. Most important for when we need to exclude the last point of the array.
                        excluded[lettuce - 1] = np.asarray(x1)
                        excluded[lettuce] = np.asarray(x2)
                        lettuce += 1

                excluded = np.array(excluded).tolist()
                allexcluded[cucumber] = excluded
                indices = np.where(mask != 0.)
                lambdasfit = std_spectra.warr[indices]
                fluxesfit = sens_function[indices]

                #Save masked wavelengths
                lambdasnotfit = std_spectra.warr[excluded]
                #print lambdasnotfit
                #print stdfile
                if 'blue' in stdspecfile.lower():
                    std_mask_name = stdfile[0:-4] + '_blue_mask.dat'
                if 'red' in stdspecfile.lower():
                    std_mask_name = stdfile[0:-4] + '_red_mask.dat'
                np.savetxt(std_mask_name,
                           np.transpose(np.array(lambdasnotfit)))
                #exit()

            ##Move back to directory with observed spectra
            #os.chdir(cwd)

            #Make sure they are finite
            ind1 = np.isfinite(lambdasfit) & np.isfinite(fluxesfit)
            lambdasfit = lambdasfit[ind1]
            fluxesfit = fluxesfit[ind1]

            print 'Fitting the sensitivity funtion now.'
            order = 4
            repeat = 'yes'
            while repeat == 'yes':
                p = np.polyfit(lambdasfit, fluxesfit, order)
                f = np.poly1d(p)
                smooth_sens = f(lambdasfit)
                residual = fluxesfit - smooth_sens
                plt.close()
                plt.ion()
                g, (ax1, ax2) = plt.subplots(2, sharex=True)
                ax1.plot(lambdasfit, fluxesfit, 'b+')
                ax1.plot(lambdasfit, smooth_sens, 'r', linewidth=2.0)
                ax1.set_ylabel('Sensitivity Function')
                ax2.plot(lambdasfit, residual, 'k+')
                ax2.set_ylabel('Residuals')
                ax1.set_title('Current polynomial order: %s' % order)
                g.subplots_adjust(hspace=0)
                plt.setp([a.get_xticklabels() for a in g.axes[:-1]],
                         visible=False)
                plt.show()
                plt.ioff()
                #Save this sensitivity curve
                '''
                try:
                    temp_file = fits.open(stdspecfile)
                    ADCstat = temp_file[0].header['ADCSTAT']
                except:
                    ADCstat = 'none'
                    pass
                if 'blue' in stdspecfile.lower():
                    resp_name = 'senscurve_' + stdfile[1:-4] + '_' + str(np.round(airstd[cucumber],decimals=3))  + '_' + ADCstat  + '_' + cwd[60:70] + '_blue.txt'
                elif 'red' in stdspecfile.lower():
                    resp_name = 'senscurve_' + stdfile[1:-4] + '_' + str(np.round(airstd[cucumber],decimals=3))  + '_' + ADCstat  + '_' + cwd[60:70] + '_red.txt'
                print resp_name
                #exit()
                np.savetxt(resp_name,np.transpose([lambdasfit,fluxesfit]))
                '''
                repeat = raw_input('Do you want to try again (yes/no)? ')
                if repeat == 'yes':
                    order = raw_input('New order for polynomial: ')

            orderused[cucumber] = order
            senspolys.append(f)

            #Save arrays for diagnostic plots
            if cucumber == 0:
                bigarray = np.zeros([len(lambdasfit), 4. * len(standards)])
                artichoke = 0
            bigarray[0:len(lambdasfit), artichoke] = lambdasfit
            bigarray[0:len(fluxesfit), artichoke + 1] = fluxesfit
            bigarray[0:len(smooth_sens), artichoke + 2] = smooth_sens
            bigarray[0:len(residual), artichoke + 3] = residual
            artichoke += 4

            cucumber += 1

        #Save fit and residuals into text file for diagnostic plotting later.
        #Need to save lambdasfit,fluxesfit,smooth_sens,residual for each standard
        #List of standards is found as standards
        now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
        with open('sens_fits_' + now + '.txt', 'a') as handle:
            header = str(
                standards
            ) + '\n Set of four columns correspond to wavelength, observed flux, polynomial fit, \n and residuals for each standard listed above. \n You will probably need to strip zeros from the bottoms of some columns.'
            np.savetxt(handle, bigarray, fmt='%f', header=header)

    #Outline for next steps:
    #Read in both red and blue files
    #compute airmass and compare to airstd
    #choose best standard and flux calibrate both blue and red
    #save files and write to sensitivity_params.txt

    if speclist[-4:] == 'fits':
        specfile = np.array([speclist])
    else:
        specfile = np.genfromtxt(speclist, dtype=str)
        if specfile.size == 1:
            specfile = np.array([specfile])

    length = len(specfile)
    airwd = np.zeros([length])
    bean = 0
    #if length == 1:
    #    redfile = False
    #else:
    #    redfile = True

    avocado = 0
    while avocado < length:
        #Read in the blue and red spectra we want to flux calibrate. Save the airmass
        WD_spectra1, airmass1, exptime1, dispersion1 = st.readspectrum(
            specfile[avocado])
        if (len(specfile) >= 1) and (avocado + 1 < length):
            if 'red' in specfile[avocado + 1]:
                redfile = True
            else:
                redfile = False
        else:
            redfile = False
        if redfile:
            WD_spectra2, airmass2, exptime2, dispersion2 = st.readspectrum(
                specfile[avocado + 1])

        #Extinction correct WD
        if extinct_correct:
            print 'Extinction correcting spectra.'
            #plt.clf()
            #plt.plot(WD_spectra1.warr,WD_spectra1.opfarr)
            WD_spectra1.opfarr = st.extinction_correction(
                WD_spectra1.warr, WD_spectra1.opfarr, airmass1)
            WD_spectra1.farr = st.extinction_correction(
                WD_spectra1.warr, WD_spectra1.farr, airmass1)
            WD_spectra1.sky = st.extinction_correction(WD_spectra1.warr,
                                                       WD_spectra1.sky,
                                                       airmass1)
            WD_spectra1.sigma = st.extinction_correction(
                WD_spectra1.warr, WD_spectra1.sigma, airmass1)
            #plt.plot(WD_spectra1.warr,WD_spectra1.opfarr)
            #plt.show()

            if redfile:
                #plt.clf()
                #plt.plot(WD_spectra2.warr,WD_spectra2.opfarr)
                WD_spectra2.opfarr = st.extinction_correction(
                    WD_spectra2.warr, WD_spectra2.opfarr, airmass2)
                WD_spectra2.farr = st.extinction_correction(
                    WD_spectra2.warr, WD_spectra2.farr, airmass2)
                WD_spectra2.sky = st.extinction_correction(
                    WD_spectra2.warr, WD_spectra2.sky, airmass2)
                WD_spectra2.sigma = st.extinction_correction(
                    WD_spectra2.warr, WD_spectra2.sigma, airmass2)

                #zaplt.plot(WD_spectra2.warr,WD_spectra2.opfarr)
                #plt.show()
        airwd[avocado] = airmass1
        if redfile:
            airwd[avocado + 1] = airmass2
        #Compare the airmasses to determine the best standard star
        tomato = 0
        while tomato < len(airstd):
            if redfile:
                diff = np.absolute(
                    np.mean([airwd[avocado], airwd[avocado + 1]]) -
                    np.mean([airstd[tomato], airstd[tomato + 1]]))
            else:
                diff = np.absolute(airwd[avocado] - airstd[tomato])
            if tomato == 0:
                difference = diff
                choice = tomato
            if diff < difference:
                difference = diff
                choice = tomato
            tomato += 2

        #To get the flux calibration, perform the following
        #Flux = counts / (Exptime * dispersion * 10**(sens/2.5))
        #Get the sensitivity function at the correct wavelength spacing
        if masterresp:
            header_temp = st.readheader(specfile[avocado])
            ADCstatus = header_temp['ADCSTAT']
            if ADCstatus == 'IN':
                sens_wave1_unscale = master_response_blue_in_pol(
                    WD_spectra1.warr)
                blue_low_index = np.min(np.where(WD_spectra1.warr > 4530.))
                blue_high_index = np.min(np.where(WD_spectra1.warr > 4590.))
                blue_mean_stan = np.mean(
                    sens_wave1_unscale[blue_low_index:blue_high_index])
                if blue_mean_tonight == None:
                    sens_wave1 = sens_wave1_unscale
                else:
                    sens_wave1 = sens_wave1_unscale + (blue_mean_tonight -
                                                       blue_mean_stan)
                choice = 0
            else:
                sens_wave1_unscale = master_response_blue_out_pol(
                    WD_spectra1.warr)
                blue_low_index = np.min(np.where(WD_spectra1.warr > 4530.))
                blue_high_index = np.min(np.where(WD_spectra1.warr > 4590.))
                blue_mean_stan = np.mean(
                    sens_wave1_unscale[blue_low_index:blue_high_index])
                if blue_mean_tonight == None:
                    sens_wave1 = sens_wave1_unscale
                else:
                    sens_wave1 = sens_wave1_unscale + (blue_mean_tonight -
                                                       blue_mean_stan)
                choice = 1
            if redfile:
                header_temp = st.readheader(specfile[avocado + 1])
                ADCstatus = header_temp['ADCSTAT']
                if ADCstatus == 'IN':
                    sens_wave2_unscale = master_response_red_in_pol(
                        WD_spectra2.warr)
                    red_low_index = np.min(np.where(WD_spectra2.warr > 6090.))
                    red_high_index = np.min(np.where(WD_spectra2.warr > 6190.))
                    red_mean_stan = np.mean(
                        sens_wave2_unscale[red_low_index:red_high_index])
                    if red_mean_tonight == None:
                        sens_wave2 = sens_wave2_unscale
                    else:
                        sens_wave2 = sens_wave2_unscale + (red_mean_tonight -
                                                           red_mean_stan)
                    choice2 = 2
                else:
                    sens_wave2_unscale = master_response_red_out_pol(
                        WD_spectra2.warr)
                    red_low_index = np.min(np.where(WD_spectra2.warr > 6090.))
                    red_high_index = np.min(np.where(WD_spectra2.warr > 6190.))
                    red_mean_stan = np.mean(
                        sens_wave2_unscale[red_low_index:red_high_index])
                    if red_mean_tonight == None:
                        sens_wave2 = sens_wave2_unscale
                    else:
                        sens_wave2 = sens_wave2_unscale + (red_mean_tonight -
                                                           red_mean_stan)
                    choice2 = 3
        else:
            sens_wave1 = senspolys[choice](WD_spectra1.warr)
            if redfile:
                sens_wave2 = senspolys[choice + 1](WD_spectra2.warr)

        #Perform the flux calibration. We do this on the optimal extraction, non-variance weighted aperture, the sky spectrum, and the sigma spectrum.
        print 'Doing the final flux calibration.'
        #np.savetxt('response_g60-54_extinction_2016-03-17.txt',np.transpose([WD_spectra1.warr,(exptime1 * dispersion1 * 10.**(sens_wave1/2.5))]))#,WD_spectra2.warr,(exptime2 * dispersion2 * 10.**(sens_wave2/2.5))]))
        #exit()
        star_opflux1 = st.cal_spec(WD_spectra1.opfarr, sens_wave1, exptime1,
                                   dispersion1)
        star_flux1 = st.cal_spec(WD_spectra1.farr, sens_wave1, exptime1,
                                 dispersion1)
        sky_flux1 = st.cal_spec(WD_spectra1.sky, sens_wave1, exptime1,
                                dispersion1)
        sigma_flux1 = st.cal_spec(WD_spectra1.sigma, sens_wave1, exptime1,
                                  dispersion1)

        if redfile:
            star_opflux2 = st.cal_spec(WD_spectra2.opfarr, sens_wave2,
                                       exptime2, dispersion2)
            star_flux2 = st.cal_spec(WD_spectra2.farr, sens_wave2, exptime2,
                                     dispersion2)
            sky_flux2 = st.cal_spec(WD_spectra2.sky, sens_wave2, exptime2,
                                    dispersion2)
            sigma_flux2 = st.cal_spec(WD_spectra2.sigma, sens_wave2, exptime2,
                                      dispersion2)

        #plt.clf()
        #plt.plot(WD_spectra.warr,star_opflux)
        #plt.show()

        #Save final spectra if using master response
        if masterresp:
            if avocado == 0:
                diagnostic_array = np.zeros(
                    [len(WD_spectra1.warr), 2 * length])
            diagnostic_array[0:len(WD_spectra1.warr), bean] = WD_spectra1.warr
            bean += 1
            diagnostic_array[0:len(star_opflux1), bean] = star_opflux1
            bean += 1
            if redfile:
                diagnostic_array[0:len(WD_spectra2.warr),
                                 bean] = WD_spectra2.warr
                bean += 1
                diagnostic_array[0:len(star_opflux2), bean] = star_opflux2
                bean += 1
        #if avocado == (length -1 ) or (redfile == True and avocado == (length-2)):
        #    print 'Saveing diagnostic file.'
        #    now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
        #    with open('flux_fits_' + now + '.txt','a') as handle:
        #        header = str(specfile) + '\n Each star is formatted as wavelength, flux'
        #        np.savetxt(handle,diagnostic_array,fmt='%.10e',header=header)

        print 'Saving the final spectrum.'

        #Save the flux-calibrated spectrum and update the header
        header1 = st.readheader(specfile[avocado])
        header1.set('EX-FLAG',
                    extinctflag)  #Extiction correction? 0=yes, -1=no
        header1.set('CA-FLAG', 0)  #Calibrated to flux scale? 0=yes, -1=no
        header1.set('BUNIT', 'erg/cm2/s/A')  #physical units of the array value
        header1.set(
            'STANDARD', str(standards[choice]),
            'Flux standard used')  #flux standard used for flux-calibration
        if masterresp:
            header1.set('STDOFF', str(flux_tonight_list[0]),
                        'Night offset used')

        if redfile:
            header2 = st.readheader(specfile[avocado + 1])
            header2.set('EX-FLAG',
                        extinctflag)  #Extiction correction? 0=yes, -1=no
            header2.set('CA-FLAG', 0)  #Calibrated to flux scale? 0=yes, -1=no
            header2.set('BUNIT',
                        'erg/cm2/s/A')  #physical units of the array value
            if masterresp:
                header2.set('STANDARD', str(standards[choice2]),
                            'Flux standard used'
                            )  #flux standard used for flux-calibration
                header1.set('STDOFF', str(flux_tonight_list[1]),
                            'Night offset used')
            else:
                header2.set('STANDARD', str(standards[choice + 1]),
                            'Flux standard used'
                            )  #flux standard used for flux-calibration

        #Set up size of new fits image
        Ni = 4.  #Number of extensions
        Nx1 = len(star_flux1)
        if redfile:
            Nx2 = len(star_flux2)
        Ny = 1.  #All 1D spectra

        data1 = np.empty(shape=(Ni, Ny, Nx1))
        data1[0, :, :] = star_opflux1
        data1[1, :, :] = star_flux1
        data1[2, :, :] = sky_flux1
        data1[3, :, :] = sigma_flux1

        if redfile:
            data2 = np.empty(shape=(Ni, Ny, Nx2))
            data2[0, :, :] = star_opflux2
            data2[1, :, :] = star_flux2
            data2[2, :, :] = sky_flux2
            data2[3, :, :] = sigma_flux2

        #Add '_flux' to the end of the filename
        loc1 = specfile[avocado].find('.ms.fits')
        if masterresp:
            newname1 = specfile[avocado][0:loc1] + '_flux_' + stdflux[0][
                1:-4] + '.ms.fits'
        else:
            newname1 = specfile[avocado][0:loc1] + '_flux_' + stdflux[
                choice // 2][1:-4] + '.ms.fits'
        clob = False
        mylist = [True for f in os.listdir('.') if f == newname1]
        exists = bool(mylist)

        if exists:
            print 'File %s already exists.' % newname1
            nextstep = raw_input(
                'Do you want to overwrite or designate a new name (overwrite/new)? '
            )
            if nextstep == 'overwrite':
                clob = True
                exists = False
            elif nextstep == 'new':
                newname1 = raw_input('New file name: ')
                exists = False
            else:
                exists = False
        print 'Saving: ', newname1
        newim1 = fits.PrimaryHDU(data=data1, header=header1)
        newim1.writeto(newname1, clobber=clob)

        if redfile:
            loc2 = specfile[avocado + 1].find('.ms.fits')
            if masterresp:
                newname2 = specfile[
                    avocado +
                    1][0:loc2] + '_flux_' + stdflux[0][1:-4] + '.ms.fits'
            else:
                newname2 = specfile[avocado + 1][0:loc2] + '_flux_' + stdflux[
                    choice // 2][1:-4] + '.ms.fits'
            clob = False
            mylist = [True for f in os.listdir('.') if f == newname2]
            exists = bool(mylist)

            if exists:
                print 'File %s already exists.' % newname2
                nextstep = raw_input(
                    'Do you want to overwrite or designate a new name (overwrite/new)? '
                )
                if nextstep == 'overwrite':
                    clob = True
                    exists = False
                elif nextstep == 'new':
                    newname2 = raw_input('New file name: ')
                    exists = False
                else:
                    exists = False

            newim2 = fits.PrimaryHDU(data=data2, header=header2)
            newim2.writeto(newname2, clobber=clob)
            print 'Saving: ', newname2

        #Finally, save all the used parameters into a file for future reference.
        # specfile,current date, stdspecfile,stdfile,order,size,newname
        f = open('sensitivity_params.txt', 'a')
        now = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M")
        if masterresp:
            newinfo1 = specfile[avocado] + '\t' + now + '\t' + standards[
                choice] + '\t' + stdflux[0] + '\t' + str(
                    allexcluded[choice]) + '\t' + str(
                        orderused[choice]) + '\t' + str(size) + '\t' + newname1
        else:
            newinfo1 = specfile[avocado] + '\t' + now + '\t' + standards[
                choice] + '\t' + stdflux[choice // 2] + '\t' + str(
                    allexcluded[choice]) + '\t' + str(
                        orderused[choice]) + '\t' + str(size) + '\t' + newname1
        if redfile:
            if masterresp:
                newinfo2 = specfile[
                    avocado + 1] + '\t' + now + '\t' + standards[
                        choice2] + '\t' + stdflux[0] + '\t' + str(
                            allexcluded[choice + 1]) + '\t' + str(
                                orderused[choice + 1]) + '\t' + str(
                                    size) + '\t' + newname2
            else:
                newinfo2 = specfile[
                    avocado + 1] + '\t' + now + '\t' + standards[
                        choice + 1] + '\t' + stdflux[choice // 2] + '\t' + str(
                            allexcluded[choice + 1]) + '\t' + str(
                                orderused[choice + 1]) + '\t' + str(
                                    size) + '\t' + newname2
            f.write(newinfo1 + "\n" + newinfo2 + "\n")
        else:
            f.write(newinfo1 + "\n")
        f.close()

        if redfile:
            avocado += 2
        else:
            avocado += 1

    print 'Done flux calibrating the spectra.'