Exemple #1
0
def mosred(infile_list, slitmask,propcode=None, dy=0, inter=True, guesstype='rss', guessfile='', rstep=100, automethod='Matchlines', preprocess=False):

    #set up the files
    infiles=','.join(['%s' % x for x in infile_list])
    obsdate=os.path.basename(infile_list[0])[7:15]

    #set up some files that will be needed
    logfile='spec'+obsdate+'.log'
    dbfile='spec%s.db' % obsdate

    #create the observation log
    obs_dict=obslog(infile_list)

    #check the value of dy


    #apply the mask to the data sets
    for i in range(len(infile_list)):
        specslit(image=infile_list[i], outimage='', outpref='s', exttype='rsmt', slitfile=slitmask,
                 outputslitfile='', regprefix='ds_', sections=3, width=25.0, sigma=2.2, thres=6.0, order=1, padding=5, yoffset=dy, 
                 inter=False, clobber=True, logfile=logfile, verbose=True)

    for i in range(len(infile_list)):
           if obs_dict['OBJECT'][i].upper().strip()=='ARC' and (obs_dict['PROPID'][i].upper().strip()==propcode or propcode is None):
               lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
               arcimage='s'+os.path.basename(infile_list[i])
               if lamp == 'NONE': lamp='CuAr'
               lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)

               specselfid(arcimage, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True)

               specidentify('a'+arcimage, lampfile, dbfile, guesstype=guesstype,
                  guessfile=guessfile, automethod=automethod,  function='legendre',  order=3,
                  rstep=rstep, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3,
                  inter=True, clobber=True, preprocess=True, logfile=logfile, verbose=True)

               #specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
               #    function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
               #    blank=0.0, clobber=True, logfile=logfile, verbose=True)

    objimages=''
    spec_list=[]
    for i in range(len(infile_list)):
       if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS')  and \
          (obs_dict['PROPID'][i].upper().strip()==propcode or propcode is None) and \
          obs_dict['OBSMODE'][i].count('SPECTROSCOPY'):
          img = infile_list[i]
          ##rectify it
          specselfid('s'+img, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True)
          specrectify('as'+img, outimages='', outpref='x', solfile=dbfile, caltype='line',
            function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
            blank=0.0, clobber=True, logfile=logfile, verbose=True)
Exemple #2
0
def mosred(infile_list, slitmask,propcode=None, dy=0, inter=True, guesstype='rss', guessfile='', rstep=100, automethod='Matchlines'):

    #set up the files
    infiles=','.join(['%s' % x for x in infile_list])
    obsdate=os.path.basename(infile_list[0])[7:15]

    #set up some files that will be needed
    logfile='spec'+obsdate+'.log'
    dbfile='spec%s.db' % obsdate

    #create the observation log
    obs_dict=obslog(infile_list)


    #apply the mask to the data sets
    for i in range(len(infile_list)):
        specslit(image=infile_list[i], outimage='', outpref='s', exttype='rsmt', slitfile='../../P001423N01.xml', 
                 outputslitfile='', regprefix='ds_', sections=3, width=25.0, sigma=2.2, thres=6.0, order=1, padding=5, yoffset=dy, 
                 inter=False, clobber=True, logfile=logfile, verbose=True)

    for i in range(len(infile_list)):
           if obs_dict['OBJECT'][i].upper().strip()=='ARC' and obs_dict['PROPID'][i].upper().strip()==propcode:
               lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
               arcimage='s'+os.path.basename(infile_list[i])
               if lamp == 'NONE': lamp='CuAr'
               lampfile=iraf.osfn("../../%s.salt" % lamp)

               specselfid(arcimage, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True)

               specidentify('a'+arcimage, lampfile, dbfile, guesstype=guesstype,
                  guessfile=guessfile, automethod=automethod,  function='legendre',  order=3,
                  rstep=rstep, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3,
                  inter=False, clobber=True, logfile=logfile, verbose=True)

               #specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
               #    function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
               #    blank=0.0, clobber=True, logfile=logfile, verbose=True)

    objimages=''
    spec_list=[]
    for i in range(len(infile_list)):
       if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS')  and \
          obs_dict['PROPID'][i].upper().strip()==propcode and \
          obs_dict['OBSMODE'][i].count('SPECTROSCOPY'):
          img = infile_list[i]
          ##rectify it
          specselfid('s'+img, '', 'a', arcimage, 'middlerow', 3, clobber=True, logfile=logfile, verbose=True)
          specrectify('as'+img, outimages='', outpref='x', solfile=dbfile, caltype='line',
            function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
            blank=0.0, clobber=True, logfile=logfile, verbose=True)
Exemple #3
0
def specred(infile_list, target, propcode, calfile=None, inter=True, automethod='Matchlines'):

    #set up the files
    infiles=','.join(['%s' % x for x in infile_list])
    obsdate=os.path.basename(infile_list[0])[7:15]

    #set up some files that will be needed
    logfile='spec'+obsdate+'.log'
    dbfile='spec%s.db' % obsdate

    #create the observation log
    obs_dict=obslog(infile_list)

    for i in range(len(infile_list)):
           if obs_dict['OBJECT'][i].upper().strip()=='ARC' and obs_dict['PROPID'][i].upper().strip()==propcode:
               lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
               arcimage=os.path.basename(infile_list[i])
               if lamp == 'NONE': lamp='CuAr'
               lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)

               specidentify(arcimage, lampfile, dbfile, guesstype='rss',
                  guessfile='', automethod=automethod,  function='legendre',  order=3,
                  rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3,
                  inter=False, clobber=True, logfile=logfile, verbose=True)

               specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
                   function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
                   blank=0.0, clobber=True, logfile=logfile, verbose=True)

    objimages=''
    spec_list=[]
    for i in range(len(infile_list)):
       if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS')  and obs_dict['PROPID'][i].upper().strip()==propcode:
          img = infile_list[i]
          ##rectify it
          specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line',
            function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
            blank=0.0, clobber=True, logfile=logfile, verbose=True)

          #extract the spectra
          spec_list.append(extract_spectra('x'+img, yc=1030, calfile=calfile, findobject=True, smooth=False, maskzeros=True, clobber=True))
          
    #combine the results
    w,f,e = speccombine(spec_list, obsdate)
    outfile = "%s_%s.spec" % (target, obsdate)
    write_spectra(outfile, w,f,e)
Exemple #4
0
def specred(infile_list, propcode=None, inter=True, automethod='Matchlines'):

    #set up the files
    infiles=','.join(['%s' % x for x in infile_list])
    obsdate=os.path.basename(infile_list[0])[7:15]

    #set up some files that will be needed
    logfile='spec'+obsdate+'.log'
    dbfile='spec%s.db' % obsdate

    #create the observation log
    obs_dict=obslog(infile_list)

    for i in range(len(infile_list)):
           if obs_dict['OBJECT'][i].upper().strip()=='ARC' and obs_dict['PROPID'][i].upper().strip()==propcode:
               lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
               arcimage=os.path.basename(infile_list[i])
               if lamp == 'NONE': lamp='CuAr'
               lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
               #lampfile='/Users/crawford/research/kepler/Xe.salt' 

               specidentify(arcimage, lampfile, dbfile, guesstype='rss',
                  guessfile='', automethod=automethod,  function='legendre',  order=3,
                  rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3,
                  inter=inter, clobber=True, logfile=logfile, verbose=True)

               #specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
               #    function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
               #    blank=0.0, clobber=True, logfile=logfile, verbose=True)

    objimages=''
    spec_list=[]
    for i in range(len(infile_list)):
       if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS')  and obs_dict['PROPID'][i].upper().strip()==propcode:
          img = infile_list[i]
          ##rectify it
          specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line',
            function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
            blank=0.0, nearest=True, clobber=True, logfile=logfile, verbose=True)
Exemple #5
0
def pol_wave_map(hduarc, image_no, drow_oc, rows, cols, lampfile, 
                 function='legendre', order=3, automethod="Matchlines",
                 log=None, logfile=None):
    """ Create a wave_map for an arc image

    For O,E arc straighten spectrum, find fov, identify for each, form (unstraightened) wavelength map   
    this corrects for the aberration introduced by the beam splitter
    this will be removed back out when creating the wave map
 
    Parameters 
    ----------
    hduarc: fits.HDUList
       Polarimetric arc data. This data should be split into O+E beams

    image_no: int
       File number of observations

    rows: int
       Nubmer of rows in original data

    cols: int
       Nubmer of columns in original data

    lampfile: str
       File name containing line list 

    function: str
       Function used for wavelength fitting

    order: int
       Order of fitting function

    automethod: str
       Method for automated line identification

    log: log
       Log for output

    Returns
    -------
    wavmap: numpy.ndarray
       Wave map of wavelengths correspond to pixels

    """

    arc_orc =  hduarc[1].data
    cbin, rbin = [int(x) for x in hduarc[0].header['CCDSUM'].split(" ")]
    axisrow_o = np.array([rows/4.0, rows/4.0]).astype(int)

    #set up some output arrays
    wavmap_orc = np.zeros((2,rows/2.0,cols))
    edgerow_od = np.zeros((2,2))
    cofrows_o = np.zeros(2)
    legy_od = np.zeros((2,2))
    guessfile=None

    for o in (0,1):

        #correct the shape of the arc for the distortions
        arc_yc = correct_wollaston(arc_orc[o], -drow_oc[o])

        # this is used to remove rows outside the slit
        maxoverlaprows = 34/rbin                        # beam overlap for 4' longslit in NIR
        arc_y = arc_yc.sum(axis=1)
        arc_y[[0,-1]] = 0.

        edgerow_od[o,0] = axisrow_o[o] - np.argmax(arc_y[axisrow_o[o]::-1] <  0.5*arc_y[axisrow_o[o]])
        edgerow_od[o,1] = axisrow_o[o] + np.argmax(arc_y[axisrow_o[o]:] <  0.5*arc_y[axisrow_o[o]])
        axisrow_o[o] = edgerow_od[o].mean()
        if np.abs(edgerow_od[o] - np.array([0,rows/2-1])).min() < maxoverlaprows:
            edgerow_od[o] += maxoverlaprows*np.array([+1,-1])

        #wrtite out temporary image to run specidentify
        hduarc['SCI'].data = arc_yc
        arcimage = "arc_"+str(image_no)+"_"+str(o)+".fits"
        dbfilename = "arcdb_"+str(image_no)+"_"+str(o)+".txt"
        ystart = axisrow_o[o]

        if (not os.path.exists(dbfilename)):
            if guessfile is not None:
                guesstype = 'file'
            else:
                guessfile=dbfilename
                guesstype = 'rss'
            hduarc.writeto(arcimage,clobber=True)
                 
            specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype,
                guessfile=guessfile, automethod=automethod,  function=function,  order=order,
                rstep=20, rstart=ystart, mdiff=20, thresh=3, niter=5, smooth=3,
                inter=True, clobber=True, logfile=logfile, verbose=True)
            if (not debug): os.remove(arcimage)

                
        wavmap_yc, cofrows_o[o], legy_od[o], edgerow_od[o] = \
                wave_map(dbfilename, edgerow_od[o], rows, cols, ystart, order, log=log)
        #TODO: Once rest is working, try to switch to pysalt wavemap
        #soldict = sr.entersolution(dbfilename)
        #wavmap_yc = wavemap(hduarc, soldict, caltype='line', function=function, 
        #          order=order,blank=0, nearest=True, array_only=True,
        #          clobber=True, log=log, verbose=True)
                  
        # put curvature back in, zero out areas beyond slit and wavelength range (will be flagged in bpm)
        if debug: np.savetxt("drow_wmap_oc.txt",drow_oc.T,fmt="%8.3f %8.3f")
        wavmap_orc[o] = correct_wollaston(wavmap_yc,drow_oc[o])

        y, x = np.indices(wavmap_orc[o].shape)

# fixing problem in 0312 sc wavmap
        notwav_c = np.isnan(drow_oc[o])
        drow_oc[o,notwav_c] = 0.
        mask = (y < edgerow_od[o,0] + drow_oc[o]) | (y > edgerow_od[o,1] + drow_oc[o])

        wavmap_orc[o,mask] = 0.
        wavmap_orc[o][:,notwav_c] = 0.
#

    if log is not None:
        log.message('\n  Wavl coeff rows:  O    %4i     E    %4i' % tuple(cofrows_o), with_header=False)
        log.message('  Bottom, top row:  O %4i %4i   E %4i %4i' \
            % tuple(legy_od.flatten()), with_header=False)
        log.message('\n  Slit axis row:    O    %4i     E    %4i' % tuple(axisrow_o), with_header=False)
        log.message('  Bottom, top row:  O %4i %4i   E %4i %4i \n' \
            % tuple(edgerow_od.flatten()), with_header=False)

    return wavmap_orc
Exemple #6
0
def specred(infile_list, propcode=None, inter=True, automethod='Matchlines'):

    #set up the files
    infiles = ','.join(['%s' % x for x in infile_list])
    obsdate = os.path.basename(infile_list[0])[7:15]

    #set up some files that will be needed
    logfile = 'spec' + obsdate + '.log'
    dbfile = 'spec%s.db' % obsdate

    #create the observation log
    obs_dict = obslog(infile_list)

    for i in range(len(infile_list)):
        if obs_dict['OBJECT'][i].upper().strip(
        ) == 'ARC' and obs_dict['PROPID'][i].upper().strip() == propcode:
            lamp = obs_dict['LAMPID'][i].strip().replace(' ', '')
            arcimage = os.path.basename(infile_list[i])
            if lamp == 'NONE': lamp = 'CuAr'
            lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
            #lampfile='/Users/crawford/research/kepler/Xe.salt'

            specidentify(arcimage,
                         lampfile,
                         dbfile,
                         guesstype='rss',
                         guessfile='',
                         automethod=automethod,
                         function='legendre',
                         order=3,
                         rstep=100,
                         rstart='middlerow',
                         mdiff=20,
                         thresh=3,
                         niter=5,
                         smooth=3,
                         inter=inter,
                         clobber=True,
                         logfile=logfile,
                         verbose=True)

            #specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
            #    function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
            #    blank=0.0, clobber=True, logfile=logfile, verbose=True)

    objimages = ''
    spec_list = []
    for i in range(len(infile_list)):
        if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][
                i].count('RSS') and obs_dict['PROPID'][i].upper().strip(
                ) == propcode:
            img = infile_list[i]
            ##rectify it
            specrectify(img,
                        outimages='',
                        outpref='x',
                        solfile=dbfile,
                        caltype='line',
                        function='legendre',
                        order=3,
                        inttype='interp',
                        w1=None,
                        w2=None,
                        dw=None,
                        nw=None,
                        blank=0.0,
                        nearest=True,
                        clobber=True,
                        logfile=logfile,
                        verbose=True)
Exemple #7
0
#/usr/bin/env python
import sys
from pyraf import iraf
from pyraf.iraf import pysalt
from specidentify import specidentify

specidentify(images=sys.argv[1], linelist=sys.argv[2], outfile='db.sol', guesstype='rss', guessfile='', automethod='Matchlines', function='polynomial', order=3,rstep=100 , rstart='middlerow', mdiff=20, thresh=3,niter=5,inter=True, clobber=False, logfile='salt.log', verbose=True)
Exemple #8
0
def pol_wave_map(hduarc, image_no, drow_oc, rows, cols, lampfile, 
                 function='legendre', order=3, automethod="Matchlines",
                 log=None, logfile=None):
    """ Create a wave_map for an arc image

    For O,E arc straighten spectrum, find fov, identify for each, form (unstraightened) wavelength map   
    this corrects for the aberration introduced by the beam splitter
    this will be removed back out when creating the wave map
 
    Parameters 
    ----------
    hduarc: fits.HDUList
       Polarimetric arc data. This data should be split into O+E beams

    image_no: int
       File number of observations

    rows: int
       Nubmer of rows in original data

    cols: int
       Nubmer of columns in original data

    lampfile: str
       File name containing line list 

    function: str
       Function used for wavelength fitting

    order: int
       Order of fitting function

    automethod: str
       Method for automated line identification

    log: log
       Log for output

    Returns
    -------
    wavmap: numpy.ndarray
       Wave map of wavelengths correspond to pixels

    """

    arc_orc =  hduarc[1].data
    cbin, rbin = [int(x) for x in hduarc[0].header['CCDSUM'].split(" ")]
    axisrow_o = np.array([rows/4.0, rows/4.0]).astype(int)
    grating = hduarc[0].header['GRATING'].strip()
    grang = hduarc[0].header['GR-ANGLE']
    artic = hduarc[0].header['CAMANG']
    trkrho = hduarc[0].header['TRKRHO']  
    date =  hduarc[0].header['DATE-OBS'].replace('-','')  

    #set up some output arrays
    wavmap_orc = np.zeros((2,rows/2,cols))
    edgerow_od = np.zeros((2,2))
    cofrows_o = np.zeros(2)
    legy_od = np.zeros((2,2))

    lam_X = rssmodelwave(grating,grang,artic,trkrho,cbin,cols,date)
#    np.savetxt("lam_X_"+str(image_no)+".txt",lam_X,fmt="%8.3f")
    C_f = np.polynomial.legendre.legfit(np.arange(cols),lam_X,3)[::-1]

    for o in (0,1):
        #correct the shape of the arc for the distortions
        arc_yc = correct_wollaston(arc_orc[o], -drow_oc[o])

        # this is used to remove rows outside the slit
        maxoverlaprows = 34/rbin                        # beam overlap for 4' longslit in NIR
        arc_y = arc_yc.sum(axis=1)
        arc_y[[0,-1]] = 0.

        edgerow_od[o,0] = axisrow_o[o] - np.argmax(arc_y[axisrow_o[o]::-1] <  0.5*arc_y[axisrow_o[o]])
        edgerow_od[o,1] = axisrow_o[o] + np.argmax(arc_y[axisrow_o[o]:] <  0.5*arc_y[axisrow_o[o]])
        axisrow_o[o] = edgerow_od[o].mean()
        if np.abs(edgerow_od[o] - np.array([0,rows/2-1])).min() < maxoverlaprows:
            edgerow_od[o] += maxoverlaprows*np.array([+1,-1])

        #write out temporary image to run specidentify
        hduarc['SCI'].data = arc_yc
        arcimage = "arc_"+str(image_no)+"_"+str(o)+".fits"
        dbfilename = "arcdb_"+str(image_no)+"_"+str(o)+".txt"

     #  use for guessfile dbfile for other beam, or, if none, "arcdb_(img)_guess.txt"
        otherdbfilename = "arcdb_"+str(image_no)+"_"+str(int(not(o==1)))+".txt"
        if (not os.path.exists(otherdbfilename)):
            otherdbfilename = "arcdb_"+str(image_no)+"_guess.txt"
        guessfilename = ""
        ystart = axisrow_o[o]

        if (not os.path.exists(dbfilename)):
            if (os.path.exists(otherdbfilename)):
                row_Y = np.loadtxt(otherdbfilename,dtype=float,usecols=(0,),ndmin=1)
                closestrow = row_Y[np.argmin(np.abs(row_Y - ystart))]
                guessfilename="wavguess_"+str(image_no)+"_"+str(o)+".txt"  
                guessfile=open(guessfilename, 'w')            
                for line in open(otherdbfilename):
                    if (len(line.split())==0): continue             # ignore naughty dbfile extra lines
                    if (line[0] == "#"):
                        guessfile.write(line)
                    elif (float(line.split()[0]) == closestrow):
                        guessfile.write(line)
                guessfile.close()
                guesstype = 'file'
#                guesstype = 'rss'
            else:
                guesstype = 'rss'            
            hduarc.writeto(arcimage,overwrite=True)
                 
            specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype,
                guessfile=guessfilename, automethod=automethod,  function=function,  order=order,
                rstep=20, rstart=ystart, mdiff=20, thresh=3, niter=5, smooth=3,
                inter=True, clobber=True, logfile=logfile, verbose=True)
            if (not debug): os.remove(arcimage)
                
        wavmap_yc, cofrows_o[o], legy_od[o], edgerow_od[o] = \
                wave_map(dbfilename, edgerow_od[o], rows, cols, ystart, order, log=log)
        #TODO: Once rest is working, try to switch to pysalt wavemap
        #soldict = sr.entersolution(dbfilename)
        #wavmap_yc = wavemap(hduarc, soldict, caltype='line', function=function, 
        #          order=order,blank=0, nearest=True, array_only=True,
        #          overwrite=True, log=log, verbose=True)
                  
        # put curvature back in, zero out areas beyond slit and wavelength range (will be flagged in bpm)
        if debug: np.savetxt("drow_wmap_oc.txt",drow_oc.T,fmt="%8.3f %8.3f")
        wavmap_orc[o] = correct_wollaston(wavmap_yc,drow_oc[o])

        y, x = np.indices(wavmap_orc[o].shape)

# fixing problem in 0312 sc wavmap
        notwav_c = np.isnan(drow_oc[o])
        drow_oc[o,notwav_c] = 0.
        mask = (y < edgerow_od[o,0] + drow_oc[o]) | (y > edgerow_od[o,1] + drow_oc[o])

        wavmap_orc[o,mask] = 0.
        wavmap_orc[o][:,notwav_c] = 0.
#

    if log is not None:
        log.message('\n  Wavl coeff rows:  O    %4i     E    %4i' % tuple(cofrows_o), with_header=False)
        log.message('  Bottom, top row:  O %4i %4i   E %4i %4i' \
            % tuple(legy_od.flatten()), with_header=False)
        log.message('\n  Slit axis row:    O    %4i     E    %4i' % tuple(axisrow_o), with_header=False)
        log.message('  Bottom, top row:  O %4i %4i   E %4i %4i \n' \
            % tuple(edgerow_od.flatten()), with_header=False)

    return wavmap_orc
Exemple #9
0
def specred(rawdir, prodir, imreduce=True, specreduce=True, calfile=None, lamp='Ar', automethod='Matchlines', skysection=[800,1000], cleanup=True):
    print rawdir
    print prodir

    #get the name of the files
    infile_list=glob.glob(rawdir+'*.fits')
    infiles=','.join(['%s' % x for x in infile_list])
    

    #get the current date for the files
    obsdate=os.path.basename(infile_list[0])[1:9]
    print obsdate

    #set up some files that will be needed
    logfile='spec'+obsdate+'.log'
    flatimage='FLAT%s.fits' % (obsdate)
    dbfile='spec%s.db' % obsdate

    #create the observation log
    obs_dict=obslog(infile_list)

 
    if imreduce:   
      #prepare the data
      saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)

      #bias subtract the data
      saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',  
              median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0, 
              niter=10, plotover=False, turbo=False, 
              clobber=True, logfile=logfile, verbose=True)

      #gain correct the data
      saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True)

      #cross talk correct the data
      saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)

      #cosmic ray clean the data
      #only clean the object data
      for i in range(len(infile_list)):
        if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
          img='xgbp'+os.path.basename(infile_list[i])
          saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
                flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
                gbox=3, maxiter=5, multithread=True,  clobber=True, logfile=logfile, verbose=True)
 
      #flat field correct the data
      flat_imgs=''
      for i in range(len(infile_list)):
        if obs_dict['CCDTYPE'][i].count('FLAT'):
           if flat_imgs: flat_imgs += ','
           flat_imgs += 'xgbp'+os.path.basename(infile_list[i])

      if len(flat_imgs)!=0:
         saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False,    \
                weight=True, blank=0, scale='average', statsec='[200:300, 600:800]', lthresh=3,    \
                hthresh=3, clobber=True, logfile=logfile, verbose=True)
         saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)

         saltflat('xgbpP*fits', '', 'f', flatimage, minflat=500, clobber=True, logfile=logfile, verbose=True)
      else:
         flats=None
         imfiles=glob.glob('cxgbpP*fits')
         for f in imfiles:
             shutil.copy(f, 'f'+f)

      #mosaic the data
      geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
      saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)

      #clean up the images
      if cleanup:
           for f in glob.glob('p*fits'): os.remove(f)
           for f in glob.glob('bp*fits'): os.remove(f)
           for f in glob.glob('gbp*fits'): os.remove(f)
           for f in glob.glob('xgbp*fits'): os.remove(f)
           for f in glob.glob('fxgbp*fits'): os.remove(f)


    #set up the name of the images
    if specreduce:
       for i in range(len(infile_list)):
           if obs_dict['OBJECT'][i].upper().strip()=='ARC':
               lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
               arcimage='mfxgbp'+os.path.basename(infile_list[i])
               lampfile=iraf.osfn("pysalt$data/linelists/%s.txt" % lamp)

               specidentify(arcimage, lampfile, dbfile, guesstype='rss', 
                  guessfile='', automethod=automethod,  function='legendre',  order=5, 
                  rstep=100, rstart='middlerow', mdiff=10, thresh=3, niter=5, 
                  inter=True, clobber=True, logfile=logfile, verbose=True)

               specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', 
                   function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
                   blank=0.0, clobber=True, logfile=logfile, verbose=True)
     

    objimages=''
    for i in range(len(infile_list)):
       if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
          if objimages: objimages += ','
          objimages+='mfxgbp'+os.path.basename(infile_list[i])

    if specreduce:
      #run specidentify on the arc files

      specrectify(objimages, outimages='', outpref='x', solfile=dbfile, caltype='line', 
           function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
           blank=0.0, clobber=True, logfile=logfile, verbose=True)


    #create the spectra text files for all of our objects
    spec_list=[]
    for img in objimages.split(','):
       spec_list.extend(createspectra('x'+img, obsdate, smooth=False, skysection=skysection, clobber=True))
    print spec_list
 
    #determine the spectrophotometric standard
    extfile=iraf.osfn('pysalt$data/site/suth_extinct.dat')

    for spec, am, et, pc in spec_list:
        if pc=='CAL_SPST':
           stdstar=spec.split('.')[0]
           print stdstar, am, et
           stdfile=iraf.osfn('pysalt$data/standards/spectroscopic/m%s.dat' % stdstar.lower().replace('-', '_'))
           print stdfile
           ofile=spec.replace('txt', 'sens')
           calfile=ofile #assumes only one observations of a SP standard
           specsens(spec, ofile, stdfile, extfile, airmass=am, exptime=et,
                stdzp=3.68e-20, function='polynomial', order=3, thresh=3, niter=5,
                clobber=True, logfile='salt.log',verbose=True)
    

    for spec, am, et, pc in spec_list:
        if pc!='CAL_SPST':
           ofile=spec.replace('txt', 'spec')
           speccal(spec, ofile, calfile, extfile, airmass=am, exptime=et, 
                  clobber=True, logfile='salt.log',verbose=True)
           #clean up the spectra for bad pixels
           cleanspectra(ofile)
Exemple #10
0
def auto_arc_lens(arc_image, dbfile='wav.db', ndstep=20, logfile='salt.log'):
    """Automatically process an arc image for the SALT Lens project

    """
    hdu = fits.open(arc_image)
    hdr = hdu[0].header
    if hdr['LAMPID'] == 'Xe' and hdr['GRATING'] == 'PG0900' and hdr['GRTILT'] == 15.875:
        print('Automatically processing arc image')
        data = hdu[1].data
        ystart = int(0.5 * len(data))
        xarr = np.arange(len(data[ystart]), dtype='int64')
        farr = data[ystart]

        lampfile = os.path.dirname(__file__)+'/data/Xe.lens'
        guessfile = os.path.dirname(__file__)+'/data/lens.db'

        slines, sfluxes = st.readlinelist(lampfile)
        spectrum = Spectrum.Spectrum(
            slines, sfluxes, dw=0.1, stype='line', sigma=6)
        swarr = spectrum.wavelength
        sfarr = spectrum.flux * farr.max() / spectrum.flux.max()

        soldict = readsolascii(guessfile, {})
        soldict = (soldict[soldict.keys()[0]])
        ws = WavelengthSolution.WavelengthSolution(
             xarr, xarr, function=soldict[7], order=soldict[8])
        ws.func.func.domain = soldict[11]
        ws.set_coef(soldict[10][2])
 
        # start the pre processing
        dcoef = ws.coef * 0.0
        dcoef[0] = 0.05 * ndstep
        dcoef[1] = 0.01 * ws.coef[1]
        ws = st.findxcor(xarr, farr, swarr, sfarr, ws,
                         dcoef=dcoef, ndstep=ndstep, best=False, inttype='interp')
        xp, wp = st.crosslinematch(xarr, farr, slines, sfluxes, ws,
                                   res=6, mdiff=20, wdiff=10,
                                   sections=3, sigma=5, niter=5)
        ws = st.findfit(np.array(xp), np.array(wp), ws=ws, thresh=ws.thresh)
        print(ws)
        with logging(logfile, True) as log:
            iws = ai.AutoIdentify(xarr, data, slines, sfluxes, ws, farr=farr,
                      method='Matchlines', rstep=100, istart=ystart, nrows=1,
                      res=6, dres=0.25, mdiff=20, sigma=5,
                      smooth=3, niter=5, dc=5, ndstep=ndstep, 
                      oneline=False, log=log, verbose=True)

            # get the basic information about the spectrograph
            dateobs = saltkey.get('DATE-OBS', hdu[0])
            try:
                utctime = saltkey.get('UTC-OBS', hdu[0])
            except SaltError:
                utctime = saltkey.get('TIME-OBS', hdu[0])

            instrume = saltkey.get('INSTRUME', hdu[0]).strip()
            grating = saltkey.get('GRATING', hdu[0]).strip()
            grang = saltkey.get('GR-ANGLE', hdu[0])
            grasteps = saltkey.get('GRTILT', hdu[0])
            arang = saltkey.get('AR-ANGLE', hdu[0])
            arsteps = saltkey.get('CAMANG', hdu[0])
            rssfilter = saltkey.get('FILTER', hdu[0])
            specmode = saltkey.get('OBSMODE', hdu[0])
            masktype = saltkey.get('MASKTYP', hdu[0]).strip().upper()
            slitname = saltkey.get('MASKID', hdu[0])
            slit = st.getslitsize(slitname)
            xbin, ybin = saltkey.ccdbin(hdu[0], arc_image)
            writeIS(iws, dbfile, dateobs=dateobs, utctime=utctime, instrume=instrume,
                    grating=grating, grang=grang, grasteps=grasteps, arsteps=arsteps,
                    arang=arang, rfilter=rssfilter, slit=slit, xbin=xbin,
                    ybin=ybin, objid=None, filename=arc_image, log=log, verbose=True)
        print(iws)

        #self.findfit()

 
    else:
        lamp = hdr['LAMPID']
        lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
        lampfile = 'Xe.lens'
        specidentify(arc_image, lampfile, dbfile, guesstype='rss', 
                  guessfile=None, automethod='Matchlines',  function='legendre',  order=3,
                  rstep=100, rstart='middlerow', mdiff=20, thresh=5, niter=5, smooth=3,
                  inter=True, clobber=True,  preprocess=True, logfile=logfile, verbose=True)
        print("Running specidenity in interactive mode")
Exemple #11
0
def specpolwavmap(infilelist,
                  linelistlib="",
                  inter=True,
                  automethod='Matchlines',
                  logfile='salt.log'):
    obsdate = os.path.basename(infilelist[0])[7:15]

    with logging(logfile, debug) as log:
        # create the observation log
        obs_dict = obslog(infilelist)
        log.message('Pysalt Version: ' + pysalt.verno, with_header=False)

        if len(linelistlib) == 0: linelistlib = datadir + "linelistlib.txt"
        with open(linelistlib) as fd:
            linelistdict = dict(line.strip().split(None, 1) for line in fd)

    # eliminate inapplicable images
        for i in range(len(infilelist)):
            if int(obs_dict['BS-STATE'][i][1]) != 2: del infilelist[i]
        obs_dict = obslog(infilelist)

        # Map out which arc goes with which image.  Use arc in closest wavcal block of the config.
        # wavcal block: neither spectrograph config nor track changes, and no gap in data files
        infiles = len(infilelist)
        newtrk = 5.  # new track when rotator changes by more (deg)
        trkrho_i = np.array(map(float, obs_dict['TRKRHO']))
        trkno_i = np.zeros((infiles), dtype=int)
        trkno_i[1:] = ((np.abs(trkrho_i[1:] - trkrho_i[:-1])) >
                       newtrk).cumsum()

        confno_i, confdatlist = configmap(infilelist)
        configs = len(confdatlist)

        imageno_i = np.array([int(os.path.basename(infilelist[i]).split('.')[0][-4:]) \
                for i in range(infiles)])
        filegrp_i = np.zeros((infiles), dtype=int)
        filegrp_i[1:] = ((imageno_i[1:] - imageno_i[:-1]) > 1).cumsum()
        isarc_i = np.array([(obs_dict['OBJECT'][i].upper().strip() == 'ARC')
                            for i in range(infiles)])

        wavblk_i = np.zeros((infiles), dtype=int)
        wavblk_i[1:] = ((filegrp_i[1:] != filegrp_i[:-1]) \
                    | (trkno_i[1:] != trkno_i[:-1]) \
                    | (confno_i[1:] != confno_i[:-1])).cumsum()
        wavblks = wavblk_i.max() + 1

        arcs_c = (isarc_i[:, None] &
                  (confno_i[:, None] == range(configs))).sum(axis=0)
        np.savetxt("wavblktbl.txt",np.vstack((trkrho_i,imageno_i,filegrp_i,trkno_i, \
                confno_i,wavblk_i,isarc_i)).T,fmt="%7.2f "+6*"%3i ",header=" rho img grp trk conf wblk arc")
        for c in range(configs):  # worst: no arc for config, remove images
            if arcs_c[c] == 0:
                lostimages = imageno_i[confno_i == c]
                log.message('No Arc for this configuration: ' \
                    +("Grating %s Grang %6.2f Artic %6.2f" % confdatlist[c])  \
                    +("\n Images: "+lostimages.shape[0]*"%i " % tuple(lostimages)), with_header=False)
                wavblk_i[confno_i == c] = -1
            if arcs_c.sum() == 0:
                log.message("Cannot calibrate any images", with_header=False)
                exit()
        iarc_i = -np.zeros((infiles), dtype=int)
        for w in range(wavblks):
            blkimages = imageno_i[wavblk_i == w]
            if blkimages.shape[0] == 0: continue
            iarc_I = np.where((wavblk_i == w) & (isarc_i))[0]
            if iarc_I.shape[0] > 0:
                iarc = iarc_I[0]  # best: arc is in wavblk, take first
            else:
                conf = confno_i[wavblk_i == w][
                    0]  # fallback: take closest arc of this config
                iarc_I = np.where((confno_i == conf) & (isarc_i))[0]
                blkimagepos = blkimages.mean()
                iarc = iarc_I[np.argmin(imageno_i[iarc_I] - blkimagepos)]
            iarc_i[wavblk_i == w] = iarc
            log.message(("For images: "+blkimages.shape[0]*"%i " % tuple(blkimages)) \
                +("\n  Use Arc %5i" % imageno_i[iarc]), with_header=False)

        iarc_a = np.unique(iarc_i[iarc_i != -1])
        arcs = iarc_a.shape[0]
        lam_m = np.loadtxt(datadir + "wollaston.txt",
                           dtype=float,
                           usecols=(0, ))
        rpix_om = np.loadtxt(datadir + "wollaston.txt",
                             dtype=float,
                             unpack=True,
                             usecols=(1, 2))

        for a in range(arcs):
            iarc = iarc_a[a]
            conf = confno_i[iarc]
            # use arc to make first-guess wavecal from model, locate beamsplitter split point
            cbin, rbin = np.array(
                obs_dict["CCDSUM"][iarc].split(" ")).astype(int)
            grating, grang, artic = confdatlist[confno_i[iarc]]
            hduarc = pyfits.open(infilelist[iarc])
            arc_rc = hduarc['SCI'].data
            rows, cols = arc_rc.shape
            lam_c = rssmodelwave(grating, grang, artic, cbin, cols)
            arc_r = arc_rc.sum(axis=1)
            expectrow_o = ((2052 + interp1d(lam_m,rpix_om,kind='cubic') \
                        (lam_c[cols/2-cols/16:cols/2+cols/16])).mean(axis=1)/rbin).astype(int)
            foundrow_o = np.zeros((2), dtype=int)
            for o in (0, 1):
                foundrow_o[o] = expectrow_o[o]-100/rbin \
                    + np.argmax(arc_r[expectrow_o[o]-100/rbin:expectrow_o[o]+100/rbin])
                arcsignal = arc_r[foundrow_o[o]]
                topedge = foundrow_o[o] + np.argmax(
                    arc_r[foundrow_o[o]:] < 0.75 * arcsignal)
                botedge = foundrow_o[o] - np.argmax(
                    arc_r[foundrow_o[o]::-1] < 0.75 * arcsignal)
                foundrow_o[o] = (botedge + topedge) / 2.
            splitrow = foundrow_o.mean()
            offset = int(splitrow - rows / 2)

            # split arc into o/e images
            padbins = (np.indices((rows, cols))[0] < offset) | (np.indices(
                (rows, cols))[0] > rows + offset)
            arc_rc = np.roll(arc_rc, -offset, axis=0)
            arc_rc[padbins] = 0.
            arc_orc = arc_rc.reshape((2, rows / 2, cols))

            # for O,E arc straighten spectrum, identify for each, form (unstraightened) wavelength map
            lamp = obs_dict['LAMPID'][iarc].strip().replace(' ', '')
            if lamp == 'NONE': lamp = 'CuAr'
            hduarc[0].header.update('MASKTYP', 'LONGSLIT')
            hduarc[0].header.update('MASKID', 'PL0100N001')
            del hduarc['VAR']
            del hduarc['BPM']
            #           lampfile=iraf.osfn("pysalt$data/linelists/%s.wav" % lamp)      zsalt version: not good
            lampfile = iraf.osfn("pysalt$data/linelists/" + linelistdict[lamp])
            rpix_oc = interp1d(lam_m, rpix_om, kind='cubic')(lam_c)

            log.message('\nARC: image '+str(imageno_i[iarc])+' GRATING '+grating\
                +' GRANG '+("%8.3f" % grang)+' ARTIC '+("%8.3f" % artic)+' LAMP '+lamp, with_header=False)
            log.message('  Split Row: ' + ("%4i " % splitrow),
                        with_header=False)

            wavmap_orc = np.zeros_like(arc_orc)
            for o in (0, 1):
                foundrow_o[o] += -offset - o * rows / 2
                arc_Rc = np.zeros((rows / 2, cols), dtype='float32')
                for c in range(cols):
                    shift(arc_orc[o,:,c],  \
                        -(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin,arc_Rc[:,c])
                hduarc['SCI'].data = arc_Rc
                arcimage = "arc_" + str(
                    imageno_i[iarc]) + "_" + str(o) + ".fits"
                dbfilename = "arcdb_" + str(
                    imageno_i[iarc]) + "_" + str(o) + ".txt"
                order = 3
                if os.path.isfile(dbfilename):
                    #                    guessfile = "guess_"+str(o)+".txt"
                    #                    os.rename(dbfilename,guessfile)
                    guesstype = 'file'
                else:
                    hduarc.writeto(arcimage, clobber=True)
                    guesstype = 'rss'
                    guessfile = ''
                    Rstart = foundrow_o[o]
                    specidentify(arcimage,
                                 lampfile,
                                 dbfilename,
                                 guesstype=guesstype,
                                 guessfile=guessfile,
                                 automethod=automethod,
                                 function='legendre',
                                 order=order,
                                 rstep=20,
                                 rstart=Rstart,
                                 mdiff=20,
                                 thresh=3,
                                 niter=5,
                                 smooth=3,
                                 inter=inter,
                                 clobber=True,
                                 logfile=logfile,
                                 verbose=True)

                wavlegR_y = np.loadtxt(dbfilename,
                                       dtype=float,
                                       usecols=(0, ),
                                       ndmin=1)
                wavlegcof_ly = np.loadtxt(dbfilename,
                                          unpack=True,
                                          dtype=float,
                                          usecols=range(1, order + 2),
                                          ndmin=2)
                if wavlegR_y.min() < 0:
                    wavlegcof_ly = np.delete(wavlegcof_ly,
                                             np.where(wavlegR_y < 0)[0],
                                             axis=1)
                    wavlegR_y = np.delete(wavlegR_y,
                                          np.where(wavlegR_y < 0)[0],
                                          axis=0)
                wavmap_yc = np.polynomial.legendre.legval(
                    np.arange(cols), wavlegcof_ly)

                mediancof_l = np.median(wavlegcof_ly, axis=1)
                rms_l = np.sqrt(
                    np.median((wavlegcof_ly - mediancof_l[:, None])**2,
                              axis=1))
                sigma_ly = (wavlegcof_ly - mediancof_l[:, None]) / rms_l[:,
                                                                         None]
                usey = (sigma_ly[0] < 3) & (sigma_ly[1] < 3)
                wavmap_Rc = np.zeros((rows / 2, cols), dtype='float32')
                R_y = wavlegR_y[usey]
                if usey.shape[0] < 5:
                    # for future: if few lines in db, use model shifted to agree, for now just use first
                    wavmap_orc[o] = wavmap_yc[0]
                else:
                    if usey.shape[0] > 9:
                        aa = np.vstack(
                            (R_y**3, R_y**2, R_y, np.ones_like(R_y))).T
                    else:
                        aa = np.vstack((R_y**2, R_y, np.ones_like(R_y))).T
                    for c in range(cols):
                        polycofs = la.lstsq(aa, wavmap_yc[usey, c])[0]
                        wavmap_orc[o,:,c]  \
                            = np.polyval(polycofs,range(rows/2)+(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin)


#                os.remove(arcimage)

# for images using this arc,save split data along third fits axis,
# add wavmap extension, save as 'w' file

            hduwav = pyfits.ImageHDU(data=wavmap_orc,
                                     header=hduarc['SCI'].header,
                                     name='WAV')
            for i in np.where(iarc_i == iarc_a[a])[0]:
                hdu = pyfits.open(infilelist[i])
                image_rc = np.roll(hdu['SCI'].data, -offset, axis=0)
                image_rc[padbins] = 0.
                hdu['SCI'].data = image_rc.reshape((2, rows / 2, cols))
                var_rc = np.roll(hdu['VAR'].data, -offset, axis=0)
                var_rc[padbins] = 0.
                hdu['VAR'].data = var_rc.reshape((2, rows / 2, cols))
                bpm_rc = np.roll(hdu['BPM'].data, -offset, axis=0)
                bpm_rc[padbins] = 1
                hdu['BPM'].data = bpm_rc.reshape((2, rows / 2, cols))
                hdu.append(hduwav)
                for f in ('SCI', 'VAR', 'BPM', 'WAV'):
                    hdu[f].header.update('CTYPE3', 'O,E')
                hdu.writeto('w' + infilelist[i], clobber='True')
                log.message('Output file ' + 'w' + infilelist[i],
                            with_header=False)

    return
Exemple #12
0
def specred(infile_list, propcode=None, inter=True, guessfile = None, automethod='Matchlines', preprocess=False):

    #set up the files
    infiles=','.join(['%s' % x for x in infile_list])
    obsdate=os.path.basename(infile_list[0])[7:15]

    #set up some files that will be needed
    logfile='spec'+obsdate+'.log'
    dbfile='spec%s.db' % obsdate
    sdbfile='spec_straight_%s.db' % obsdate
    straight_function = 'legendre'
    straight_order = 2
    dcoef = [0.50, 1.0, 0.0] 

    #create the observation log
    obs_dict=obslog(infile_list)
    for i in range(len(infile_list)):
           print infile_list[i], obs_dict['OBJECT'][i].upper().strip(), obs_dict['PROPID'][i].upper().strip()
           if obs_dict['OBJECT'][i].upper().strip()=='ARC' and (propcode is None or obs_dict['PROPID'][i].upper().strip()==propcode):
               lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
               arcimage=os.path.basename(infile_list[i])
               if lamp == 'NONE': lamp='CuAr'
               lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
               #lampfile='/Users/crawford/research/kepler/Xe.salt' 

               #straighten the arc
               #specarcstraighten(arcimage, sdbfile , function=straight_function, order=straight_order, rstep=20,
               #       rstart='middlerow', nrows=1, dcoef=dcoef, ndstep=10,
               #       startext=0, clobber=False, logfile='salt.log', verbose=True)

               #rectify it
               #specrectify(arcimage, outimages='', outpref='s', solfile=sdbfile, caltype='line',
               #    function=straight_function,  order=straight_order, inttype='interp', w1=None, w2=None, dw=None, nw=None,
               #    nearest=True, blank=0.0, clobber=True, logfile=logfile, verbose=True)

               #idnetify the line
               if guessfile is None:
                   guesstype='rss'
                   guessfile=None
               else:
                   guesstype='file'
                   
               specidentify(arcimage, lampfile, dbfile, guesstype=guesstype,
                  guessfile=guessfile, automethod=automethod,  function='legendre',  order=3,
                  rstep=100, rstart='middlerow', mdiff=20, thresh=5, niter=5, smooth=3,
                  inter=inter, clobber=True,  preprocess=preprocess, logfile=logfile, verbose=True)
 
               #apply the final rectification
               specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line',
                   function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
                   blank=0.0, clobber=True, logfile=logfile, verbose=True)

    objimages=''
    spec_list=[]
    for i in range(len(infile_list)):
       if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS')  and (propcode is None or obs_dict['PROPID'][i].upper().strip()==propcode):
          img = infile_list[i]
          # rectify it
          #specrectify(img, outimages='', outpref='s', solfile=sdbfile, caltype='line',
          #    function='legendre',  order=2, inttype='interp', w1=None, w2=None, dw=None, nw=None,
          #    blank=0.0, clobber=True, logfile=logfile, verbose=True)

          specrectify(img, outimages='', outpref='x', solfile=dbfile, caltype='line',
            function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
            blank=0.0, nearest=True, clobber=True, logfile=logfile, verbose=True)
Exemple #13
0
def pol_wave_map(hduarc,
                 image_id,
                 drow_oc,
                 rows,
                 cols,
                 lampfile,
                 function='legendre',
                 order=3,
                 automethod="Matchlines",
                 log=None,
                 logfile=None):
    """ Create a wave_map for an arc image

    For O,E arc straighten spectrum, find fov, identify for each, form (unstraightened) wavelength map   
    this corrects for the aberration introduced by the beam splitter
    this will be removed back out when creating the wave map
 
    Parameters 
    ----------
    hduarc: fits.HDUList
       Polarimetric arc data. This data should be split into O+E beams

    image_id: str
       Arc identifier

    rows: int
       Nubmer of rows in original data

    cols: int
       Nubmer of columns in original data

    lampfile: str
       File name containing line list 

    function: str
       Function used for wavelength fitting

    order: int
       Order of fitting function

    automethod: str
       Method for automated line identification

    log: log
       Log for output

    Returns
    -------
    wavmap: numpy.ndarray
       Wave map of wavelengths correspond to pixels

    """

    arc_orc = hduarc[1].data
    cbin, rbin = [int(x) for x in hduarc[0].header['CCDSUM'].split(" ")]
    axisrow_o = np.array([rows / 4.0, rows / 4.0]).astype(int)
    grating = hduarc[0].header['GRATING'].strip()
    grang = hduarc[0].header['GR-ANGLE']
    artic = hduarc[0].header['CAMANG']
    trkrho = hduarc[0].header['TRKRHO']
    date = hduarc[0].header['DATE-OBS'].replace('-', '')

    #set up some output arrays
    wavmap_orc = np.zeros((2, rows / 2, cols))
    edgerow_od = np.zeros((2, 2))
    cofrows_o = np.zeros(2)
    legy_od = np.zeros((2, 2))

    lam_X = rssmodelwave(grating, grang, artic, trkrho, cbin, cols, date)
    #    np.savetxt("lam_X_"+image_id+".txt",lam_X,fmt="%8.3f")
    C_f = np.polynomial.legendre.legfit(np.arange(cols), lam_X, 3)[::-1]

    for o in (0, 1):
        #correct the shape of the arc for the distortions
        arc_yc = correct_wollaston(arc_orc[o], -drow_oc[o])

        # this is used to remove rows outside the slit
        maxoverlaprows = 34 / rbin  # beam overlap for 4' longslit in NIR
        arc_y = arc_yc.sum(axis=1)
        arc_y[[0, -1]] = 0.

        edgerow_od[o, 0] = axisrow_o[o] - np.argmax(
            arc_y[axisrow_o[o]::-1] < 0.5 * arc_y[axisrow_o[o]])
        edgerow_od[o, 1] = axisrow_o[o] + np.argmax(
            arc_y[axisrow_o[o]:] < 0.5 * arc_y[axisrow_o[o]])
        axisrow_o[o] = edgerow_od[o].mean()
        if np.abs(edgerow_od[o] -
                  np.array([0, rows / 2 - 1])).min() < maxoverlaprows:
            edgerow_od[o] += maxoverlaprows * np.array([+1, -1])

        #write out temporary image to run specidentify
        hduarc['SCI'].data = arc_yc
        arcimage = "arc_" + image_id + "_" + str(o) + ".fits"
        dbfilename = "arcdb_" + image_id + "_" + str(o) + ".txt"

        #  use for guessfile dbfile for other beam, or, if none, "arcdb_(img)_guess.txt"
        otherdbfilename = "arcdb_" + image_id + "_" + str(
            int(not (o == 1))) + ".txt"
        if (not os.path.exists(otherdbfilename)):
            otherdbfilename = "arcdb_" + image_id + "_guess.txt"
        guessfilename = ""
        ystart = axisrow_o[o]

        if (not os.path.exists(dbfilename)):
            if (os.path.exists(otherdbfilename)):
                row_Y = np.loadtxt(otherdbfilename,
                                   dtype=float,
                                   usecols=(0, ),
                                   ndmin=1)
                closestrow = row_Y[np.argmin(np.abs(row_Y - ystart))]
                guessfilename = "wavguess_" + image_id + "_" + str(o) + ".txt"
                guessfile = open(guessfilename, 'w')
                for line in open(otherdbfilename):
                    if (len(line.split()) == 0):
                        continue  # ignore naughty dbfile extra lines
                    if (line[0] == "#"):
                        guessfile.write(line)
                    elif (float(line.split()[0]) == closestrow):
                        guessfile.write(line)
                guessfile.close()
                guesstype = 'file'
#                guesstype = 'rss'
            else:
                guesstype = 'rss'
            hduarc.writeto(arcimage, overwrite=True)

            specidentify(arcimage,
                         lampfile,
                         dbfilename,
                         guesstype=guesstype,
                         guessfile=guessfilename,
                         automethod=automethod,
                         function=function,
                         order=order,
                         rstep=20,
                         rstart=ystart,
                         mdiff=20,
                         thresh=3,
                         niter=5,
                         smooth=3,
                         inter=True,
                         clobber=True,
                         logfile=logfile,
                         verbose=True)
            if (not debug): os.remove(arcimage)

        wavmap_yc, cofrows_o[o], legy_od[o], edgerow_od[o] = \
                wave_map(dbfilename, edgerow_od[o], rows, cols, ystart, order, log=log)
        #TODO: Once rest is working, try to switch to pysalt wavemap
        #soldict = sr.entersolution(dbfilename)
        #wavmap_yc = wavemap(hduarc, soldict, caltype='line', function=function,
        #          order=order,blank=0, nearest=True, array_only=True,
        #          overwrite=True, log=log, verbose=True)

        # put curvature back in, zero out areas beyond slit and wavelength range (will be flagged in bpm)
        if debug: np.savetxt("drow_wmap_oc.txt", drow_oc.T, fmt="%8.3f %8.3f")
        wavmap_orc[o] = correct_wollaston(wavmap_yc, drow_oc[o])

        y, x = np.indices(wavmap_orc[o].shape)

        # fixing problem in 0312 sc wavmap
        notwav_c = np.isnan(drow_oc[o])
        drow_oc[o, notwav_c] = 0.
        mask = (y < edgerow_od[o, 0] + drow_oc[o]) | (
            y > edgerow_od[o, 1] + drow_oc[o])

        wavmap_orc[o, mask] = 0.
        wavmap_orc[o][:, notwav_c] = 0.


#

    if log is not None:
        log.message('\n  Wavl coeff rows:  O    %4i     E    %4i' %
                    tuple(cofrows_o),
                    with_header=False)
        log.message('  Bottom, top row:  O %4i %4i   E %4i %4i' \
            % tuple(legy_od.flatten()), with_header=False)
        log.message('\n  Slit axis row:    O    %4i     E    %4i' %
                    tuple(axisrow_o),
                    with_header=False)
        log.message('  Bottom, top row:  O %4i %4i   E %4i %4i \n' \
            % tuple(edgerow_od.flatten()), with_header=False)

    return wavmap_orc
Exemple #14
0
def specpolmap(infile_list, propcode=None, inter=True, automethod='Matchlines',logfile='salt.log'):
    obsdate=os.path.basename(infile_list[0])[7:15]

    with logging(logfile, debug) as log:
    #create the observation log
        obs_dict=obslog(infile_list)

    # eliminate inapplicable images
        for i in range(len(infile_list)):
            if obs_dict['PROPID'][i].upper().strip()!=propcode or int(obs_dict['BS-STATE'][i][1])!=2: 
                del infile_list[i]
        obs_dict=obslog(infile_list)

    # use arc to make first-guess wavecal from model, locate beamsplitter split point
        for i in range(len(infile_list)):
            if obs_dict['OBJECT'][i].upper().strip()=='ARC' : arcidx = i
        rbin,cbin = np.array(obs_dict["CCDSUM"][arcidx].split(" ")).astype(int)
        grating = obs_dict['GRATING'][arcidx].strip()
        grang = float(obs_dict['GR-ANGLE'][arcidx])
        artic = float(obs_dict['CAMANG'][arcidx])
        hduarc = pyfits.open(infile_list[arcidx])
        arc_rc = hduarc['SCI'].data
        rows,cols = arc_rc.shape
        lam_c = rssmodelwave(grating,grang,artic,cbin,cols)
        arc_r = arc_rc.sum(axis=1)

        lam_m = np.loadtxt(datadir+"wollaston.txt",dtype=float,usecols=(0,))
        rpix_om = np.loadtxt(datadir+"wollaston.txt",dtype=float,unpack=True,usecols=(1,2))
        expectrow_o = ((2052 + interp1d(lam_m,rpix_om,kind='cubic') \
                        (lam_c[cols/2-cols/16:cols/2+cols/16])).mean(axis=1)/rbin).astype(int)
        foundrow_o = np.zeros((2),dtype=int)
        for o in (0,1):
            foundrow_o[o] = expectrow_o[o]-100/rbin \
                    + np.argmax(arc_r[expectrow_o[o]-100/rbin:expectrow_o[o]+100/rbin])
            arcsignal = arc_r[foundrow_o[o]]
            botedge = foundrow_o[o] + np.argmax(arc_r[foundrow_o[o]:] <  arcsignal/2.) 
            topedge = foundrow_o[o] - np.argmax(arc_r[foundrow_o[o]::-1] <  arcsignal/2.)
            foundrow_o[o] = (botedge+topedge)/2.
        splitrow = foundrow_o.mean()
        offset = int(splitrow - rows/2)
        log.message('Split Row: '+("%4i " % splitrow), with_header=False)

    # split arc into o/e images
        padbins = (np.indices((rows,cols))[0]<offset) | (np.indices((rows,cols))[0]>rows+offset)
        arc_rc = np.roll(arc_rc,-offset,axis=0)
        arc_rc[padbins] = 0.
        arc_orc =  arc_rc.reshape((2,rows/2,cols))
    
    # for O,E arc straighten spectrum, identify for each, form (unstraightened) wavelength map   

        lamp=obs_dict['LAMPID'][arcidx].strip().replace(' ', '')
        if lamp == 'NONE': lamp='CuAr'
        hduarc[0].header.update('MASKTYP','LONGSLIT')
        hduarc[0].header.update('MASKID','PL0100N001')
        del hduarc['VAR']
        del hduarc['BPM']
        lampfile=iraf.osfn("pysalt$data/linelists/%s.wav" % lamp)
        rpix_oc = interp1d(lam_m, rpix_om,kind ='cubic')(lam_c)

        log.message('\nARC: image '+infile_list[i].split('.')[0][-4:]+' GRATING '+grating\
            +' GRANG '+("%8.3f" % grang)+' ARTIC '+("%8.3f" % artic)+' LAMP '+lamp, with_header=False)

        wavmap_orc = np.zeros_like(arc_orc)
        for o in (0,1):
            foundrow_o[o] += -offset + o*rows/2
            arc_Rc = np.zeros((rows/2,cols),dtype='float32')
            for c in range(cols): 
                shift(arc_orc[o,:,c],  \
                    -(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin,arc_Rc[:,c])
            hduarc['SCI'].data = arc_Rc
            arcimage = "arc_"+str(o)+".fits"
            hduarc.writeto(arcimage,clobber=True)
            Rstart = foundrow_o[o]
            order = 3
            dbfilename = "arcdb_"+str(o)+".txt"
            if os.path.isfile(dbfilename):
#                    guessfile = "guess_"+str(o)+".txt"
#                    os.rename(dbfilename,guessfile)
                guesstype = 'file'
            else:
                guessfile = ''
                guesstype = 'rss'
                specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype,
                    guessfile=guessfile, automethod=automethod,  function='legendre',  order=order,
                    rstep=10, rstart=Rstart, mdiff=20, thresh=3, niter=5, smooth=3,
                    inter=inter, clobber=True, logfile=logfile, verbose=True)

            wavlegR_y = np.loadtxt(dbfilename,dtype=float,usecols=(0,))
            wavlegcof_ly = np.loadtxt(dbfilename,unpack=True,dtype=float,usecols=range(1,order+2))
            wavlegcof_ly = np.delete(wavlegcof_ly,np.where(wavlegR_y<0)[0],axis=1)
            wavlegR_y = np.delete(wavlegR_y,np.where(wavlegR_y<0)[0],axis=0)
            wavmap_yc = np.polynomial.legendre.legval(np.arange(cols),wavlegcof_ly)

            mediancof_l = np.median(wavlegcof_ly,axis=1)
            rms_l = np.sqrt(np.median((wavlegcof_ly - mediancof_l[:,None])**2,axis=1))
            sigma_ly = (wavlegcof_ly - mediancof_l[:,None])/rms_l[:,None]
            usey = (sigma_ly[0]<3) & (sigma_ly[1]<3)
            wavmap_Rc = np.zeros((rows/2,cols),dtype='float32')
            R_y = wavlegR_y[usey]
            a = np.vstack((R_y**3,R_y**2,R_y,np.ones_like(R_y))).T
            for c in range(cols):
                polycofs = la.lstsq(a,wavmap_yc[usey,c])[0]
                wavmap_orc[o,:,c]  \
                    = np.polyval(polycofs,range(rows/2)+(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin)

    # save split data along third fits axis, add wavmap extension, save as 'w' file

        hduwav = pyfits.ImageHDU(data=wavmap_orc, header=hduarc['SCI'].header, name='WAV')          
        for i in range(len(infile_list)):  
            hdu = pyfits.open(infile_list[i])
            image_rc = np.roll(hdu['SCI'].data,-offset,axis=0)
            image_rc[padbins] = 0.
            hdu['SCI'].data = image_rc.reshape((2,rows/2,cols))
            var_rc = np.roll(hdu['VAR'].data,-offset,axis=0)
            var_rc[padbins] = 0.
            hdu['VAR'].data = var_rc.reshape((2,rows/2,cols))
            bpm_rc = np.roll(hdu['BPM'].data,-offset,axis=0)
            bpm_rc[padbins] = 1
            hdu['BPM'].data = bpm_rc.reshape((2,rows/2,cols))
            hdu.append(hduwav)
            for f in ('SCI','VAR','BPM','WAV'): hdu[f].header.update('CTYPE3','O,E')
            hdu.writeto('w'+infile_list[i],clobber='True')
            log.message('Output file '+'w'+infile_list[i] , with_header=False)
Exemple #15
0
def specpolmap(infile_list,
               propcode=None,
               inter=True,
               automethod='Matchlines',
               logfile='salt.log'):
    obsdate = os.path.basename(infile_list[0])[7:15]

    with logging(logfile, debug) as log:
        #create the observation log
        obs_dict = obslog(infile_list)

        # eliminate inapplicable images
        for i in range(len(infile_list)):
            if obs_dict['PROPID'][i].upper().strip() != propcode or int(
                    obs_dict['BS-STATE'][i][1]) != 2:
                del infile_list[i]
        obs_dict = obslog(infile_list)

        # use arc to make first-guess wavecal from model, locate beamsplitter split point
        for i in range(len(infile_list)):
            if obs_dict['OBJECT'][i].upper().strip() == 'ARC': arcidx = i
        rbin, cbin = np.array(
            obs_dict["CCDSUM"][arcidx].split(" ")).astype(int)
        grating = obs_dict['GRATING'][arcidx].strip()
        grang = float(obs_dict['GR-ANGLE'][arcidx])
        artic = float(obs_dict['CAMANG'][arcidx])
        hduarc = pyfits.open(infile_list[arcidx])
        arc_rc = hduarc['SCI'].data
        rows, cols = arc_rc.shape
        lam_c = rssmodelwave(grating, grang, artic, cbin, cols)
        arc_r = arc_rc.sum(axis=1)

        lam_m = np.loadtxt(datadir + "wollaston.txt",
                           dtype=float,
                           usecols=(0, ))
        rpix_om = np.loadtxt(datadir + "wollaston.txt",
                             dtype=float,
                             unpack=True,
                             usecols=(1, 2))
        expectrow_o = ((2052 + interp1d(lam_m,rpix_om,kind='cubic') \
                        (lam_c[cols/2-cols/16:cols/2+cols/16])).mean(axis=1)/rbin).astype(int)
        foundrow_o = np.zeros((2), dtype=int)
        for o in (0, 1):
            foundrow_o[o] = expectrow_o[o]-100/rbin \
                    + np.argmax(arc_r[expectrow_o[o]-100/rbin:expectrow_o[o]+100/rbin])
            arcsignal = arc_r[foundrow_o[o]]
            botedge = foundrow_o[o] + np.argmax(
                arc_r[foundrow_o[o]:] < arcsignal / 2.)
            topedge = foundrow_o[o] - np.argmax(
                arc_r[foundrow_o[o]::-1] < arcsignal / 2.)
            foundrow_o[o] = (botedge + topedge) / 2.
        splitrow = foundrow_o.mean()
        offset = int(splitrow - rows / 2)
        log.message('Split Row: ' + ("%4i " % splitrow), with_header=False)

        # split arc into o/e images
        padbins = (np.indices((rows, cols))[0] < offset) | (np.indices(
            (rows, cols))[0] > rows + offset)
        arc_rc = np.roll(arc_rc, -offset, axis=0)
        arc_rc[padbins] = 0.
        arc_orc = arc_rc.reshape((2, rows / 2, cols))

        # for O,E arc straighten spectrum, identify for each, form (unstraightened) wavelength map

        lamp = obs_dict['LAMPID'][arcidx].strip().replace(' ', '')
        if lamp == 'NONE': lamp = 'CuAr'
        hduarc[0].header.update('MASKTYP', 'LONGSLIT')
        hduarc[0].header.update('MASKID', 'PL0100N001')
        del hduarc['VAR']
        del hduarc['BPM']
        lampfile = iraf.osfn("pysalt$data/linelists/%s.wav" % lamp)
        rpix_oc = interp1d(lam_m, rpix_om, kind='cubic')(lam_c)

        log.message('\nARC: image '+infile_list[i].split('.')[0][-4:]+' GRATING '+grating\
            +' GRANG '+("%8.3f" % grang)+' ARTIC '+("%8.3f" % artic)+' LAMP '+lamp, with_header=False)

        wavmap_orc = np.zeros_like(arc_orc)
        for o in (0, 1):
            foundrow_o[o] += -offset + o * rows / 2
            arc_Rc = np.zeros((rows / 2, cols), dtype='float32')
            for c in range(cols):
                shift(arc_orc[o,:,c],  \
                    -(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin,arc_Rc[:,c])
            hduarc['SCI'].data = arc_Rc
            arcimage = "arc_" + str(o) + ".fits"
            hduarc.writeto(arcimage, clobber=True)
            Rstart = foundrow_o[o]
            order = 3
            dbfilename = "arcdb_" + str(o) + ".txt"
            if os.path.isfile(dbfilename):
                #                    guessfile = "guess_"+str(o)+".txt"
                #                    os.rename(dbfilename,guessfile)
                guesstype = 'file'
            else:
                guessfile = ''
                guesstype = 'rss'
                specidentify(arcimage,
                             lampfile,
                             dbfilename,
                             guesstype=guesstype,
                             guessfile=guessfile,
                             automethod=automethod,
                             function='legendre',
                             order=order,
                             rstep=10,
                             rstart=Rstart,
                             mdiff=20,
                             thresh=3,
                             niter=5,
                             smooth=3,
                             inter=inter,
                             clobber=True,
                             logfile=logfile,
                             verbose=True)

            wavlegR_y = np.loadtxt(dbfilename, dtype=float, usecols=(0, ))
            wavlegcof_ly = np.loadtxt(dbfilename,
                                      unpack=True,
                                      dtype=float,
                                      usecols=range(1, order + 2))
            wavlegcof_ly = np.delete(wavlegcof_ly,
                                     np.where(wavlegR_y < 0)[0],
                                     axis=1)
            wavlegR_y = np.delete(wavlegR_y,
                                  np.where(wavlegR_y < 0)[0],
                                  axis=0)
            wavmap_yc = np.polynomial.legendre.legval(np.arange(cols),
                                                      wavlegcof_ly)

            mediancof_l = np.median(wavlegcof_ly, axis=1)
            rms_l = np.sqrt(
                np.median((wavlegcof_ly - mediancof_l[:, None])**2, axis=1))
            sigma_ly = (wavlegcof_ly - mediancof_l[:, None]) / rms_l[:, None]
            usey = (sigma_ly[0] < 3) & (sigma_ly[1] < 3)
            wavmap_Rc = np.zeros((rows / 2, cols), dtype='float32')
            R_y = wavlegR_y[usey]
            a = np.vstack((R_y**3, R_y**2, R_y, np.ones_like(R_y))).T
            for c in range(cols):
                polycofs = la.lstsq(a, wavmap_yc[usey, c])[0]
                wavmap_orc[o,:,c]  \
                    = np.polyval(polycofs,range(rows/2)+(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin)

    # save split data along third fits axis, add wavmap extension, save as 'w' file

        hduwav = pyfits.ImageHDU(data=wavmap_orc,
                                 header=hduarc['SCI'].header,
                                 name='WAV')
        for i in range(len(infile_list)):
            hdu = pyfits.open(infile_list[i])
            image_rc = np.roll(hdu['SCI'].data, -offset, axis=0)
            image_rc[padbins] = 0.
            hdu['SCI'].data = image_rc.reshape((2, rows / 2, cols))
            var_rc = np.roll(hdu['VAR'].data, -offset, axis=0)
            var_rc[padbins] = 0.
            hdu['VAR'].data = var_rc.reshape((2, rows / 2, cols))
            bpm_rc = np.roll(hdu['BPM'].data, -offset, axis=0)
            bpm_rc[padbins] = 1
            hdu['BPM'].data = bpm_rc.reshape((2, rows / 2, cols))
            hdu.append(hduwav)
            for f in ('SCI', 'VAR', 'BPM', 'WAV'):
                hdu[f].header.update('CTYPE3', 'O,E')
            hdu.writeto('w' + infile_list[i], clobber='True')
            log.message('Output file ' + 'w' + infile_list[i],
                        with_header=False)
Exemple #16
0
def science_red(rawdir, prodir, imreduce=True, specreduce=True, bpmfile=None, calfile=None, lampfile='Ar', automethod='Matchlines', skysection=[800,1000], cleanup=True):
    print rawdir
    print prodir

    #get the name of the files
    infile_list=glob.glob(rawdir+'P*.fits')
    infiles=','.join(['%s' % x for x in infile_list])
    

    #get the current date for the files
    obsdate=os.path.basename(infile_list[0])[1:9]
    print obsdate

    #set up some files that will be needed
    logfile='spec'+obsdate+'.log'
    flatimage='FLAT%s.fits' % (obsdate)
    dbfile='spec%s.db' % obsdate

    #create the observation log
    obs_dict=obslog(infile_list)

 
    if imreduce:   
      #prepare the data
      saltprepare(infiles, '', 'p', createvar=False, badpixelimage='', clobber=True, logfile=logfile, verbose=True)

      #bias subtract the data
      saltbias('pP*fits', '', 'b', subover=True, trim=True, subbias=False, masterbias='',  
              median=False, function='polynomial', order=5, rej_lo=3.0, rej_hi=5.0, 
              niter=10, plotover=False, turbo=False, 
              clobber=True, logfile=logfile, verbose=True)

      add_variance('bpP*fits', bpmfile)

      #gain correct the data
      saltgain('bpP*fits', '', 'g', usedb=False, mult=True, clobber=True, logfile=logfile, verbose=True)

      #cross talk correct the data
      saltxtalk('gbpP*fits', '', 'x', xtalkfile = "", usedb=False, clobber=True, logfile=logfile, verbose=True)

 
      #flat field correct the data
      flat_imgs=''
      for i in range(len(infile_list)):
        if obs_dict['CCDTYPE'][i].count('FLAT'):
           if flat_imgs: flat_imgs += ','
           flat_imgs += 'xgbp'+os.path.basename(infile_list[i])

      if 0: #len(flat_imgs)!=0:
         saltcombine(flat_imgs,flatimage, method='median', reject=None, mask=False,    \
                weight=False, blank=0, scale=None, statsec='[200:300, 600:800]', lthresh=3,    \
                hthresh=3, clobber=True, logfile=logfile, verbose=True)
         saltillum(flatimage, flatimage, '', mbox=11, clobber=True, logfile=logfile, verbose=True)

         saltflat('xgbpP*fits', '', 'f', flatimage, minflat=0.8, allext=False, clobber=True, logfile=logfile, verbose=True)
      else:
         flats=None
         imfiles=glob.glob('xgbpP*fits')
         for f in imfiles:
             shutil.copy(f, 'f'+f)

      #cosmic ray clean the data
      #only clean the object data
      for i in range(len(infile_list)):
        if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
          img='fxgbp'+os.path.basename(infile_list[i])
          saltcrclean(img, img, '', crtype='edge', thresh=5, mbox=11, bthresh=5.0,
                flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
                gbox=3, maxiter=5, multithread=True,  clobber=True, logfile=logfile, verbose=True)

      #mosaic the data
      geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
      saltmosaic('fxgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)

      #clean up the images
      if cleanup:
           for f in glob.glob('p*fits'): os.remove(f)
           for f in glob.glob('bp*fits'): os.remove(f)
           for f in glob.glob('gbp*fits'): os.remove(f)
           for f in glob.glob('xgbp*fits'): os.remove(f)
           for f in glob.glob('fxgbp*fits'): os.remove(f)


    #set up the name of the images
    if specreduce:
       for i in range(len(infile_list)):
           if obs_dict['OBJECT'][i].upper().strip()=='ARC':
               lamp=obs_dict['LAMPID'][i].strip().replace(' ', '')
               arcimage='mfxgbp'+os.path.basename(infile_list[i])
               #lampfile=iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)

               specidentify(arcimage, lampfile, dbfile, guesstype='rss', 
                  guessfile='', automethod=automethod,  function='legendre',  order=3, 
                  rstep=100, rstart='middlerow', mdiff=50, thresh=2, niter=5, smooth=3,
                  inter=True, clobber=True, logfile=logfile, verbose=True)

               specrectify(arcimage, outimages='', outpref='x', solfile=dbfile, caltype='line', 
                   function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
                   blank=0.0, clobber=True, logfile=logfile, verbose=True)
     

    objimages=''
    for i in range(len(infile_list)):
       if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][i].count('RSS'):
          if objimages: objimages += ','
          objimages+='mfxgbp'+os.path.basename(infile_list[i])

    if specreduce:
      #run specidentify on the arc files

      specrectify(objimages, outimages='', outpref='x', solfile=dbfile, caltype='line', 
           function='legendre',  order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None,
           blank=0.0, clobber=True, logfile=logfile, verbose=True)


    return
def specpolwavmap(infilelist, linelistlib="", automethod='Matchlines',logfile='salt.log',debug=False):
    obsdate=os.path.basename(infilelist[0])[7:15]

    with logging(logfile, debug) as log:
    # create the observation log
        obs_dict=obslog(infilelist)
        log.message('Pysalt Version: '+pysalt.verno, with_header=False)
      
        
    # eliminate inapplicable images
        for i in reversed(range(len(infilelist))):
            if int(obs_dict['BS-STATE'][i][1])!=2: del infilelist[i]
        obs_dict=obslog(infilelist)

    # Map out which arc goes with which image.  Use arc in closest wavcal block of the config.
    # wavcal block: neither spectrograph config nor track changes, and no gap in data files
        infiles = len(infilelist)
        newtrk = 5.                                     # new track when rotator changes by more (deg)
        trkrho_i = np.array(map(float,obs_dict['TRKRHO']))
        trkno_i = np.zeros((infiles),dtype=int)
        trkno_i[1:] = ((np.abs(trkrho_i[1:]-trkrho_i[:-1]))>newtrk).cumsum()

        confno_i,confdatlist = configmap(infilelist)
        configs = len(confdatlist)

        imageno_i = np.array([int(os.path.basename(infilelist[i]).split('.')[0][-4:]) \
                for i in range(infiles)])
        filegrp_i = np.zeros((infiles),dtype=int)
        filegrp_i[1:] = ((imageno_i[1:]-imageno_i[:-1])>1).cumsum()
        isarc_i = np.array([(obs_dict['OBJECT'][i].upper().strip()=='ARC') for i in range(infiles)])

        wavblk_i = np.zeros((infiles),dtype=int)
        wavblk_i[1:] = ((filegrp_i[1:] != filegrp_i[:-1]) \
                    | (trkno_i[1:] != trkno_i[:-1]) \
                    | (confno_i[1:] != confno_i[:-1])).cumsum()
        wavblks = wavblk_i.max() +1

        arcs_c = (isarc_i[:,None] & (confno_i[:,None]==range(configs))).sum(axis=0)
        np.savetxt("wavblktbl.txt",np.vstack((trkrho_i,imageno_i,filegrp_i,trkno_i, \
                confno_i,wavblk_i,isarc_i)).T,fmt="%7.2f "+6*"%3i ",header=" rho img grp trk conf wblk arc")
        for c in range(configs):                               # worst: no arc for config, remove images
            if arcs_c[c] == 0:
                lostimages = imageno_i[confno_i==c]
                log.message('No Arc for this configuration: ' \
                    +("Grating %s Grang %6.2f Artic %6.2f" % confdatlist[c])  \
                    +("\n Images: "+lostimages.shape[0]*"%i " % tuple(lostimages)), with_header=False)
                wavblk_i[confno_i==c] = -1
            if arcs_c.sum() ==0: 
                log.message("Cannot calibrate any images", with_header=False)
                exit()
        iarc_i = -np.zeros((infiles),dtype=int)
        for w in range(wavblks):
            blkimages =  imageno_i[wavblk_i==w]
            if blkimages.shape[0]==0: continue 
            iarc_I = np.where((wavblk_i==w) & (isarc_i))[0]
            if iarc_I.shape[0] >0: 
                iarc = iarc_I[0]                        # best: arc is in wavblk, take first
            else:
                conf = confno_i[wavblk_i==w][0]       # fallback: take closest arc of this config
                iarc_I = np.where((confno_i==conf) & (isarc_i))[0]
                blkimagepos = blkimages.mean()
                iarc = iarc_I[np.argmin(imageno_i[iarc_I] - blkimagepos)]
            iarc_i[wavblk_i==w] = iarc                    
            log.message(("\nFor images: "+blkimages.shape[0]*"%i " % tuple(blkimages)) \
                +("\n  Use Arc %5i" % imageno_i[iarc]), with_header=False)               

        iarc_a = np.unique(iarc_i[iarc_i != -1])
        arcs = iarc_a.shape[0]
        lam_m = np.loadtxt(datadir+"wollaston.txt",dtype=float,usecols=(0,))
        rpix_om = np.loadtxt(datadir+"wollaston.txt",dtype=float,unpack=True,usecols=(1,2))

        for a in range(arcs):
            iarc = iarc_a[a]
            conf = confno_i[iarc]
            grating,grang,artic = confdatlist[confno_i[iarc]]
            
            if len(linelistlib) ==0: 
                linelistlib=datadir+"linelistlib.txt"   
                if grating=="PG0300": 
                    linelistlib=datadir+"linelistlib_300.txt"
            with open(linelistlib) as fd:
                linelistdict = dict(line.strip().split(None, 1) for line in fd)  

        # use arc to make first-guess wavecal from model
            cbin,rbin = np.array(obs_dict["CCDSUM"][iarc].split(" ")).astype(int)

            hduarc = pyfits.open(infilelist[iarc])
            arc_rc = hduarc['SCI'].data
            rows,cols = arc_rc.shape
            lam_c = rssmodelwave(grating,grang,artic,cbin,cols)
            arc_r = arc_rc.sum(axis=1)
#            if debug: np.savetxt("arc_r_"+str(imageno_i[iarc]+".txt"),arc_r,fmt="%8.2f")
        # locate beamsplitter split point
            axisrow_o = ((2052 + interp1d(lam_m,rpix_om,kind='cubic',bounds_error=False) \
                        (lam_c[cols/2]))/rbin).astype(int)
            top = axisrow_o[1] + np.argmax(arc_r[axisrow_o[1]:] <  0.5*arc_r[axisrow_o[1]])
            bot = axisrow_o[0] - np.argmax(arc_r[axisrow_o[0]::-1] <  0.5*arc_r[axisrow_o[0]])
            splitrow = 0.5*(bot + top)
            offset = int(splitrow - rows/2)                 # how far split is from center of detector

        # split arc into o/e images
            padbins = (np.indices((rows,cols))[0]<offset) | (np.indices((rows,cols))[0]>rows+offset)
            arc_rc = np.roll(arc_rc,-offset,axis=0)
            arc_rc[padbins] = 0.
            arc_orc =  arc_rc.reshape((2,rows/2,cols))
    
        # for O,E arc straighten spectrum, find fov, identify for each, form (unstraightened) wavelength map   
            lamp=obs_dict['LAMPID'][iarc].strip().replace(' ', '')
            if lamp == 'NONE': lamp='CuAr'
            hduarc[0].header.update('MASKTYP','LONGSLIT')
            del hduarc['VAR']
            del hduarc['BPM']
            lampfile=iraf.osfn("pysalt$data/linelists/"+linelistdict[lamp])    
            rpix_oc = interp1d(lam_m, rpix_om,kind ='cubic',bounds_error=False,fill_value=0.)(lam_c)
            drow_oc = (rpix_oc-rpix_oc[:,cols/2][:,None])/rbin

            log.message('\nARC: image '+str(imageno_i[iarc])+' GRATING '+grating\
                +' GRANG '+("%8.3f" % grang)+' ARTIC '+("%8.3f" % artic)+' LAMP '+lamp, with_header=False)
            log.message('  Split Row: '+("%4i " % splitrow), with_header=False)

            wavmap_orc = np.zeros((2,rows/2,cols))
            edgerow_od = np.zeros((2,2))
            cofrows_o = np.zeros(2)
            legy_od = np.zeros((2,2))
            guessfile=None
            for o in (0,1):
                axisrow_o[o] += -offset - o*rows/2
                arc_yc = np.zeros((rows/2,cols),dtype='float32')
                for c in range(cols): 
                    shift(arc_orc[o,:,c],-drow_oc[o,c],arc_yc[:,c])

                maxoverlaprows = 34/rbin                        # beam overlap for 4' longslit in NIR
                arc_yc[(0,rows/2-1)] = 0.
                arc_y = arc_yc.sum(axis=1)
                edgerow_od[o,0] = axisrow_o[o] - np.argmax(arc_y[axisrow_o[o]::-1] <  0.5*arc_y[axisrow_o[o]])
                edgerow_od[o,1] = axisrow_o[o] + np.argmax(arc_y[axisrow_o[o]:] <  0.5*arc_y[axisrow_o[o]])
                axisrow_o[o] = edgerow_od[o].mean()
                if np.abs(edgerow_od[o] - np.array([0,rows/2-1])).min() < maxoverlaprows:
                    edgerow_od[o] += maxoverlaprows*np.array([+1,-1])
                hduarc['SCI'].data = arc_yc
                order = 3
                arcimage = "arc_"+str(imageno_i[iarc])+"_"+str(o)+".fits"
                dbfilename = "arcdb_"+str(imageno_i[iarc])+"_"+str(o)+".txt"

                if (not os.path.exists(dbfilename)):
                    if guessfile is not None:
                        guesstype = 'file'
                    else:
                        guessfile=dbfilename
                        guesstype = 'rss'
                    hduarc.writeto(arcimage,clobber=True)
                    ystart = axisrow_o[o]
                    specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype,
                        guessfile=guessfile, automethod=automethod,  function='legendre',  order=order,
                        rstep=20, rstart=ystart, mdiff=20, thresh=3, niter=5, smooth=3,
                        inter=True, clobber=True, logfile=logfile, verbose=True)
                    if (not debug): os.remove(arcimage)

            # process dbfile legendre coefs within FOV into wavmap (_Y = line in dbfile)
                legy_Y = np.loadtxt(dbfilename,dtype=float,usecols=(0,),ndmin=1)
                dblegcof_lY = np.loadtxt(dbfilename,unpack=True,dtype=float,usecols=range(1,order+2),ndmin=2)

            # first convert to centered legendre coefficients to remove crosscoupling
                xcenter = cols/2.
                legcof_lY = np.zeros_like(dblegcof_lY)
                legcof_lY[2] = dblegcof_lY[2] + 5.*dblegcof_lY[3]*xcenter
                legcof_lY[3] = dblegcof_lY[3]
                legcof_lY[0] = 0.5*legcof_lY[2] + (dblegcof_lY[0]-dblegcof_lY[2]) + \
                    (dblegcof_lY[1]-1.5*dblegcof_lY[3])*xcenter + 1.5*dblegcof_lY[2]*xcenter**2 + \
                    2.5*dblegcof_lY[3]*xcenter**3
                legcof_lY[1] = 1.5*legcof_lY[3] + (dblegcof_lY[1]-1.5*dblegcof_lY[3]) + \
                    3.*dblegcof_lY[2]*xcenter + 7.5*dblegcof_lY[3]*xcenter**2

            # remove rows outside slit and outlier fits
                argYbad = np.where((legy_Y<edgerow_od[o,0]) | (legy_Y>edgerow_od[o,1]))[0]
                legy_Y = np.delete(legy_Y, argYbad,axis=0)
                legcof_lY = np.delete(legcof_lY, argYbad,axis=1)
                mediancof_l = np.median(legcof_lY,axis=1)
                rms_l = np.sqrt(np.median((legcof_lY - mediancof_l[:,None])**2,axis=1))
                sigma_lY = np.abs((legcof_lY - mediancof_l[:,None]))/rms_l[:,None]
                argYbad = np.where((sigma_lY>4).any(axis=0))[0]
                legy_Y = np.delete(legy_Y, argYbad,axis=0)
                legcof_lY = np.delete(legcof_lY, argYbad,axis=1)
                cofrows_o[o] = legy_Y.shape[0]
                legy_od[o] = legy_Y.min(),legy_Y.max()

                if cofrows_o[o] < 5:
            # for future: if few lines in db, use model shifted to agree, for now just use mean
                    legcof_l = legcof_lY.mean(axis=1)
                    wavmap_yc = np.polynomial.legendre.legval(np.arange(cols),legcof_l)
                else:
            # smooth wavmap along rows by fitting L_0 to quadratic, others to linear fn of row
                    ycenter = rows/4.
                    Y_y = np.arange(-ycenter,ycenter)
                    aa = np.vstack(((legy_Y-ycenter)**2,(legy_Y-ycenter),np.ones(cofrows_o[o]))).T
                    polycofs = la.lstsq(aa,legcof_lY[0])[0]
                    legcof_ly = np.zeros((order+1,rows/2))
                    legcof_ly[0] = np.polyval(polycofs,Y_y)
                    for l in range(1,order+1):
                        polycofs = la.lstsq(aa[:,1:],legcof_lY[l])[0]
                        legcof_ly[l] = np.polyval(polycofs,Y_y)
                    wavmap_yc = np.zeros((rows/2,cols))
                    for y in range(rows/2):
                        wavmap_yc[y] = np.polynomial.legendre.legval(np.arange(-cols/2,cols/2),legcof_ly[:,y])

            # put curvature back in, zero out areas beyond slit and wavelength range (will be flagged in bpm)
                if debug: np.savetxt("drow_wmap_oc.txt",drow_oc.T,fmt="%8.3f %8.3f")
                for c in range(cols): 
                    shift(wavmap_yc[:,c],drow_oc[o,c],wavmap_orc[o,:,c],order=1)

                isoffslit_rc = ((np.arange(rows/2)[:,None] < (edgerow_od[o,0]+(rpix_oc[o]-rpix_oc[o,cols/2])/rbin)[None,:]) \
                           | (np.arange(rows/2)[:,None] > (edgerow_od[o,1]+(rpix_oc[o]-rpix_oc[o,cols/2])/rbin)[None,:]))
                notwav_rc = (rpix_oc[o]==0.)[None,:]
                wavmap_orc[o,(isoffslit_rc | notwav_rc)] = 0.

            log.message('\n  Wavl coeff rows:  O    %4i     E    %4i' % tuple(cofrows_o), with_header=False)
            log.message('  Bottom, top row:  O %4i %4i   E %4i %4i' \
                % tuple(legy_od.flatten()), with_header=False)
            log.message('\n  Slit axis row:    O    %4i     E    %4i' % tuple(axisrow_o), with_header=False)
            log.message('  Bottom, top row:  O %4i %4i   E %4i %4i \n' \
                % tuple(edgerow_od.flatten()), with_header=False)

        # for images using this arc,save split data along third fits axis, 
        # add wavmap extension, save as 'w' file
            hduwav = pyfits.ImageHDU(data=wavmap_orc.astype('float32'), header=hduarc['SCI'].header, name='WAV')                 
            for i in np.where(iarc_i==iarc_a[a])[0]:  
                hdu = pyfits.open(infilelist[i])
                image_rc = np.roll(hdu['SCI'].data,-offset,axis=0)
                image_rc[padbins] = 0.
                hdu['SCI'].data = image_rc.reshape((2,rows/2,cols))
                var_rc = np.roll(hdu['VAR'].data,-offset,axis=0)
                var_rc[padbins] = 0.
                hdu['VAR'].data = var_rc.reshape((2,rows/2,cols))
                bpm_rc = np.roll(hdu['BPM'].data,-offset,axis=0)
                bpm_rc[padbins] = 1
                bpm_orc = bpm_rc.reshape((2,rows/2,cols))
                bpm_orc[wavmap_orc==0.] = 1
                hdu['BPM'].data = bpm_orc
                hdu.append(hduwav)
                for f in ('SCI','VAR','BPM','WAV'): hdu[f].header.update('CTYPE3','O,E')
                hdu.writeto('w'+infilelist[i],clobber='True')
                log.message('Output file '+'w'+infilelist[i] , with_header=False)

    return
def specpolwavmap(infilelist, linelistlib="", inter=True, automethod='Matchlines',logfile='salt.log'):
    obsdate=os.path.basename(infilelist[0])[7:15]

    with logging(logfile, debug) as log:
    # create the observation log
        obs_dict=obslog(infilelist)
        log.message('Pysalt Version: '+pysalt.verno, with_header=False)

        if len(linelistlib) ==0:    linelistlib=datadir+"linelistlib.txt"
        with open(linelistlib) as fd:
            linelistdict = dict(line.strip().split(None, 1) for line in fd)        
        
    # eliminate inapplicable images
        for i in range(len(infilelist)):
            if int(obs_dict['BS-STATE'][i][1])!=2: del infilelist[i]
        obs_dict=obslog(infilelist)

    # Map out which arc goes with which image.  Use arc in closest wavcal block of the config.
    # wavcal block: neither spectrograph config nor track changes, and no gap in data files
        infiles = len(infilelist)
        newtrk = 5.                                     # new track when rotator changes by more (deg)
        trkrho_i = np.array(map(float,obs_dict['TRKRHO']))
        trkno_i = np.zeros((infiles),dtype=int)
        trkno_i[1:] = ((np.abs(trkrho_i[1:]-trkrho_i[:-1]))>newtrk).cumsum()

        confno_i,confdatlist = configmap(infilelist)
        configs = len(confdatlist)

        imageno_i = np.array([int(os.path.basename(infilelist[i]).split('.')[0][-4:]) \
                for i in range(infiles)])
        filegrp_i = np.zeros((infiles),dtype=int)
        filegrp_i[1:] = ((imageno_i[1:]-imageno_i[:-1])>1).cumsum()
        isarc_i = np.array([(obs_dict['OBJECT'][i].upper().strip()=='ARC') for i in range(infiles)])

        wavblk_i = np.zeros((infiles),dtype=int)
        wavblk_i[1:] = ((filegrp_i[1:] != filegrp_i[:-1]) \
                    | (trkno_i[1:] != trkno_i[:-1]) \
                    | (confno_i[1:] != confno_i[:-1])).cumsum()
        wavblks = wavblk_i.max() +1

        arcs_c = (isarc_i[:,None] & (confno_i[:,None]==range(configs))).sum(axis=0)
        np.savetxt("wavblktbl.txt",np.vstack((trkrho_i,imageno_i,filegrp_i,trkno_i, \
                confno_i,wavblk_i,isarc_i)).T,fmt="%7.2f "+6*"%3i ",header=" rho img grp trk conf wblk arc")
        for c in range(configs):                               # worst: no arc for config, remove images
            if arcs_c[c] == 0:
                lostimages = imageno_i[confno_i==c]
                log.message('No Arc for this configuration: ' \
                    +("Grating %s Grang %6.2f Artic %6.2f" % confdatlist[c])  \
                    +("\n Images: "+lostimages.shape[0]*"%i " % tuple(lostimages)), with_header=False)
                wavblk_i[confno_i==c] = -1
            if arcs_c.sum() ==0: 
                log.message("Cannot calibrate any images", with_header=False)
                exit()
        iarc_i = -np.zeros((infiles),dtype=int)
        for w in range(wavblks):
            blkimages =  imageno_i[wavblk_i==w]
            if blkimages.shape[0]==0: continue 
            iarc_I = np.where((wavblk_i==w) & (isarc_i))[0]
            if iarc_I.shape[0] >0: 
                iarc = iarc_I[0]                        # best: arc is in wavblk, take first
            else:
                conf = confno_i[wavblk_i==w][0]       # fallback: take closest arc of this config
                iarc_I = np.where((confno_i==conf) & (isarc_i))[0]
                blkimagepos = blkimages.mean()
                iarc = iarc_I[np.argmin(imageno_i[iarc_I] - blkimagepos)]
            iarc_i[wavblk_i==w] = iarc                    
            log.message(("For images: "+blkimages.shape[0]*"%i " % tuple(blkimages)) \
                +("\n  Use Arc %5i" % imageno_i[iarc]), with_header=False)               

        iarc_a = np.unique(iarc_i[iarc_i != -1])
        arcs = iarc_a.shape[0]
        lam_m = np.loadtxt(datadir+"wollaston.txt",dtype=float,usecols=(0,))
        rpix_om = np.loadtxt(datadir+"wollaston.txt",dtype=float,unpack=True,usecols=(1,2))

        for a in range(arcs):
            iarc = iarc_a[a]
            conf = confno_i[iarc]
        # use arc to make first-guess wavecal from model, locate beamsplitter split point
            cbin,rbin = np.array(obs_dict["CCDSUM"][iarc].split(" ")).astype(int)
            grating,grang,artic = confdatlist[confno_i[iarc]]
            hduarc = pyfits.open(infilelist[iarc])
            arc_rc = hduarc['SCI'].data
            rows,cols = arc_rc.shape
            lam_c = rssmodelwave(grating,grang,artic,cbin,cols)
            arc_r = arc_rc.sum(axis=1)
            expectrow_o = ((2052 + interp1d(lam_m,rpix_om,kind='cubic') \
                        (lam_c[cols/2-cols/16:cols/2+cols/16])).mean(axis=1)/rbin).astype(int)
            foundrow_o = np.zeros((2),dtype=int)
            for o in (0,1):
                foundrow_o[o] = expectrow_o[o]-100/rbin \
                    + np.argmax(arc_r[expectrow_o[o]-100/rbin:expectrow_o[o]+100/rbin])
                arcsignal = arc_r[foundrow_o[o]]
                topedge = foundrow_o[o] + np.argmax(arc_r[foundrow_o[o]:] <  0.75*arcsignal) 
                botedge = foundrow_o[o] - np.argmax(arc_r[foundrow_o[o]::-1] <  0.75*arcsignal)
                foundrow_o[o] = (botedge+topedge)/2.
            splitrow = foundrow_o.mean()
            offset = int(splitrow - rows/2)

        # split arc into o/e images
            padbins = (np.indices((rows,cols))[0]<offset) | (np.indices((rows,cols))[0]>rows+offset)
            arc_rc = np.roll(arc_rc,-offset,axis=0)
            arc_rc[padbins] = 0.
            arc_orc =  arc_rc.reshape((2,rows/2,cols))
    
        # for O,E arc straighten spectrum, identify for each, form (unstraightened) wavelength map   
            lamp=obs_dict['LAMPID'][iarc].strip().replace(' ', '')
            if lamp == 'NONE': lamp='CuAr'
            hduarc[0].header.update('MASKTYP','LONGSLIT')
            hduarc[0].header.update('MASKID','PL0100N001')
            del hduarc['VAR']
            del hduarc['BPM']
#           lampfile=iraf.osfn("pysalt$data/linelists/%s.wav" % lamp)      zsalt version: not good
            lampfile=iraf.osfn("pysalt$data/linelists/"+linelistdict[lamp])    
            rpix_oc = interp1d(lam_m, rpix_om,kind ='cubic')(lam_c)

            log.message('\nARC: image '+str(imageno_i[iarc])+' GRATING '+grating\
                +' GRANG '+("%8.3f" % grang)+' ARTIC '+("%8.3f" % artic)+' LAMP '+lamp, with_header=False)
            log.message('  Split Row: '+("%4i " % splitrow), with_header=False)

            wavmap_orc = np.zeros_like(arc_orc)
            for o in (0,1):
                foundrow_o[o] += -offset - o*rows/2
                arc_Rc = np.zeros((rows/2,cols),dtype='float32')
                for c in range(cols): 
                    shift(arc_orc[o,:,c],  \
                        -(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin,arc_Rc[:,c])
                hduarc['SCI'].data = arc_Rc
                arcimage = "arc_"+str(imageno_i[iarc])+"_"+str(o)+".fits"
                dbfilename = "arcdb_"+str(imageno_i[iarc])+"_"+str(o)+".txt"
                order = 3
                if os.path.isfile(dbfilename):
#                    guessfile = "guess_"+str(o)+".txt"
#                    os.rename(dbfilename,guessfile)
                    guesstype = 'file'
                else:
                    hduarc.writeto(arcimage,clobber=True)
                    guesstype = 'rss'
                    guessfile = ''
                    Rstart = foundrow_o[o]
                    specidentify(arcimage, lampfile, dbfilename, guesstype=guesstype,
                        guessfile=guessfile, automethod=automethod,  function='legendre',  order=order,
                        rstep=20, rstart=Rstart, mdiff=20, thresh=3, niter=5, smooth=3,
                        inter=inter, clobber=True, logfile=logfile, verbose=True)

                wavlegR_y = np.loadtxt(dbfilename,dtype=float,usecols=(0,),ndmin=1)
                wavlegcof_ly = np.loadtxt(dbfilename,unpack=True,dtype=float,usecols=range(1,order+2),ndmin=2)
                if wavlegR_y.min()<0:
                    wavlegcof_ly = np.delete(wavlegcof_ly,np.where(wavlegR_y<0)[0],axis=1)
                    wavlegR_y = np.delete(wavlegR_y,np.where(wavlegR_y<0)[0],axis=0)
                wavmap_yc = np.polynomial.legendre.legval(np.arange(cols),wavlegcof_ly)

                mediancof_l = np.median(wavlegcof_ly,axis=1)
                rms_l = np.sqrt(np.median((wavlegcof_ly - mediancof_l[:,None])**2,axis=1))
                sigma_ly = (wavlegcof_ly - mediancof_l[:,None])/rms_l[:,None]
                usey = (sigma_ly[0]<3) & (sigma_ly[1]<3)
                wavmap_Rc = np.zeros((rows/2,cols),dtype='float32')
                R_y = wavlegR_y[usey]
                if usey.shape[0] < 5:
            # for future: if few lines in db, use model shifted to agree, for now just use first
                    wavmap_orc[o] = wavmap_yc[0]
                else:
                    if usey.shape[0] > 9:
                        aa = np.vstack((R_y**3,R_y**2,R_y,np.ones_like(R_y))).T
                    else:
                        aa = np.vstack((R_y**2,R_y,np.ones_like(R_y))).T
                    for c in range(cols):
                        polycofs = la.lstsq(aa,wavmap_yc[usey,c])[0]
                        wavmap_orc[o,:,c]  \
                            = np.polyval(polycofs,range(rows/2)+(rpix_oc[o,c]-rpix_oc[o,cols/2])/rbin)
#                os.remove(arcimage)

        # for images using this arc,save split data along third fits axis, 
        # add wavmap extension, save as 'w' file

            hduwav = pyfits.ImageHDU(data=wavmap_orc, header=hduarc['SCI'].header, name='WAV')                 
            for i in np.where(iarc_i==iarc_a[a])[0]:  
                hdu = pyfits.open(infilelist[i])
                image_rc = np.roll(hdu['SCI'].data,-offset,axis=0)
                image_rc[padbins] = 0.
                hdu['SCI'].data = image_rc.reshape((2,rows/2,cols))
                var_rc = np.roll(hdu['VAR'].data,-offset,axis=0)
                var_rc[padbins] = 0.
                hdu['VAR'].data = var_rc.reshape((2,rows/2,cols))
                bpm_rc = np.roll(hdu['BPM'].data,-offset,axis=0)
                bpm_rc[padbins] = 1
                hdu['BPM'].data = bpm_rc.reshape((2,rows/2,cols))
                hdu.append(hduwav)
                for f in ('SCI','VAR','BPM','WAV'): hdu[f].header.update('CTYPE3','O,E')
                hdu.writeto('w'+infilelist[i],clobber='True')
                log.message('Output file '+'w'+infilelist[i] , with_header=False)

    return
Exemple #19
0
import sys

from pyraf import iraf
from iraf import pysalt
from specidentify import specidentify

specidentify(images=sys.argv[1], linelist=sys.argv[2], outfile='wav.db', guesstype='rss', guessfile='', automethod='Matchlines', function='poly', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=3, inter=True, startext=0 ,clobber=True, textcolor='black', logfile='salt.log',
verbose=True)
Exemple #20
0
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None, 
	geomfile=None,subover=True,trim=True,masbias=None, 
        subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
        rej_hi=3,niter=5,interp='linear',  sdbhost='',sdbname='',sdbuser='', password='',
        clobber=False, cleanup=True, logfile='salt.log', verbose=True):
   """SALTADVANCE provides advanced data reductions for a set of data.  It will 
      sort the data, and first process the biases, flats, and then the science 
      frames.  It will record basic quality control information about each of 
      the steps.
   """
   plotover=False

   #start logging
   with logging(logfile,debug) as log:

       # Check the input images 
       infiles = saltio.argunpack ('Input',images)
       infiles.sort()

       # create list of output files 
       outpath=saltio.abspath(outpath)

       #log into the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

       #does the gain database file exist
       if gaindb:
           dblist= saltio.readgaindb(gaindb)
       else:
           dblist=[]

       # does crosstalk coefficient data exist
       if xtalkfile:
           xtalkfile = xtalkfile.strip()
           xdict = saltio.readxtalkcoeff(xtalkfile)
       else:
           xdict=None
       #does the mosaic file exist--raise error if no
       saltio.fileexists(geomfile)


       # Delete the obslog file if it already exists
       if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)

       #read in the obsveration log or create it
       if os.path.isfile(obslogfile):
           msg='The observing log already exists.  Please either delete it or run saltclean with clobber=yes'
           raise SaltError(msg)
       else:
           headerDict=obslog(infiles, log)
           obsstruct=createobslogfits(headerDict)
           saltio.writefits(obsstruct, obslogfile)

       #create the list of bias frames and process them
       filename=obsstruct.data.field('FILENAME')
       detmode=obsstruct.data.field('DETMODE')
       obsmode=obsstruct.data.field('OBSMODE')
       ccdtype=obsstruct.data.field('CCDTYPE')
       propcode=obsstruct.data.field('PROPID')
       masktype=obsstruct.data.field('MASKTYP')

       #set the bias list of objects
       biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')]
       masterbias_dict={}
       for img in infiles:
           if os.path.basename(img) in biaslist:
               #open the image
               struct=fits.open(img)
               bimg=outpath+'bxgp'+os.path.basename(img)

               #print the message
               if log:
                   message='Processing Zero frame %s' % img
                   log.message(message, with_stdout=verbose)

               #process the image
               struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
                            bstruct=None, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
                            verbose=verbose)
 
               #update the database
               updatedq(os.path.basename(img), struct, sdb)

               #write the file out
               # housekeeping keywords
               fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
               saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
               saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
               saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
               saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])

               # write FITS file
               saltio.writefits(struct,bimg, clobber=clobber)
               saltio.closefits(struct)

               #add files to the master bias list
               masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)

       #create the master bias frame
       for i in masterbias_dict.keys():
           bkeys=masterbias_dict[i][0]
           blist=masterbias_dict[i][1:]
           mbiasname=outpath+createmasterbiasname(blist, bkeys)
           bfiles=','.join(blist)
           saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, 
                       weight=False, blank=0, scale=None, statsec=None, lthresh=3,    \
                       hthresh=3, clobber=False, logfile=logfile,verbose=verbose)

           

       #create the list of flatfields and process them
       flatlist=filename[ccdtype=='FLAT']
       masterflat_dict={}
       for img in infiles:
           if os.path.basename(img) in flatlist:
               #open the image
               struct=fits.open(img)
               fimg=outpath+'bxgp'+os.path.basename(img)

               #print the message
               if log:
                   message='Processing Flat frame %s' % img
                   log.message(message, with_stdout=verbose)

               #process the image
               struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
                            bstruct=None, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
                            verbose=verbose)

               #update the database
               updatedq(os.path.basename(img), struct, sdb)

               #write the file out
               # housekeeping keywords
               fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
               saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
               saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
               saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
               saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])

               # write FITS file
               saltio.writefits(struct,fimg, clobber=clobber)
               saltio.closefits(struct)

               #add files to the master bias list
               masterflat_dict=compareimages(struct, fimg, masterflat_dict,  keylist=flatheader_list)

       #create the master flat frame
       for i in masterflat_dict.keys():
           fkeys=masterflat_dict[i][0]
           flist=masterflat_dict[i][1:]
           mflatname=outpath+createmasterflatname(flist, fkeys)
           ffiles=','.join(flist)
           saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, 
                       weight=False, blank=0, scale=None, statsec=None, lthresh=3,    \
                       hthresh=3, clobber=False, logfile=logfile,verbose=verbose)

       #process the arc data
       arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')]
       for i, img in enumerate(infiles):
           nimg=os.path.basename(img)
           if nimg in arclist:
               #open the image
               struct=fits.open(img)
               simg=outpath+'bxgp'+os.path.basename(img)
               obsdate=os.path.basename(img)[1:9]

               #print the message
               if log:
                   message='Processing ARC frame %s' % img
                   log.message(message, with_stdout=verbose)


               struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
                            bstruct=None, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, 
                            log=log, verbose=verbose)

               # write FITS file
               saltio.writefits(struct,simg, clobber=clobber)
               saltio.closefits(struct)

               #mosaic the images
               mimg=outpath+'mbxgp'+os.path.basename(img)
               saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
                    interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
                    verbose=verbose)

               #remove the intermediate steps
               saltio.delete(simg)


               #measure the arcdata
               arcimage=outpath+'mbxgp'+nimg
               dbfile=outpath+obsdate+'_specid.db'
               lamp = obsstruct.data.field('LAMPID')[i]
               lamp = lamp.replace(' ', '')
               lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
               print arcimage, lampfile, os.getcwd()
               specidentify(arcimage, lampfile, dbfile, guesstype='rss', 
                                guessfile='', automethod='Matchlines', function='legendre',
                                order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3,
                                startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile, 
                                verbose=verbose)
               try:
                   ximg = outpath+'xmbxgp'+os.path.basename(arcimage)
                   specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
                              function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
                              nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
                              logfile=logfile, verbose=verbose)
               except:
                   pass


              
       #process the science data
       for i, img in enumerate(infiles):
           nimg=os.path.basename(img)
           if not (nimg in flatlist or nimg in biaslist or nimg in arclist):
     
               #open the image
               struct=fits.open(img)
               if struct[0].header['PROPID'].count('CAL_GAIN'): continue
               simg=outpath+'bxgp'+os.path.basename(img)
   

               #print the message
               if log:
                   message='Processing science frame %s' % img
                   log.message(message, with_stdout=verbose)


               #Check to see if it is RSS 2x2 and add bias subtraction
               instrume=saltkey.get('INSTRUME', struct[0]).strip()
               gainset = saltkey.get('GAINSET', struct[0])    
               rospeed = saltkey.get('ROSPEED', struct[0])    
               target = saltkey.get('OBJECT', struct[0]).strip()
               exptime = saltkey.get('EXPTIME', struct[0])
               obsmode = saltkey.get('OBSMODE', struct[0]).strip()
               detmode = saltkey.get('DETMODE', struct[0]).strip()
               masktype = saltkey.get('MASKTYP', struct[0]).strip()
  
               
               xbin, ybin = saltkey.ccdbin( struct[0], img)
               obsdate=os.path.basename(img)[1:9]
               bstruct=None
               crtype=None
               thresh=5 
               mbox=11 
               bthresh=5.0,
               flux_ratio=0.2 
               bbox=25 
               gain=1.0 
               rdnoise=5.0 
               fthresh=5.0 
               bfactor=2
               gbox=3 
               maxiter=5
    
               subbias=False
               if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW':
                   bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin)
                   if os.path.exists(bfile):
                      bstruct=fits.open(bfile)
                      subbias=True
                   if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5:
                       crtype='edge' 
                       thresh=5 
                       mbox=11 
                       bthresh=5.0,
                       flux_ratio=0.2 
                       bbox=25 
                       gain=1.0 
                       rdnoise=5.0 
                       fthresh=5.0 
                       bfactor=2
                       gbox=3 
                       maxiter=3
    
               #process the image
               struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias,
                            bstruct=bstruct, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, 
                            crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox,      \
                            bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise, 
                            bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter,
                            log=log, verbose=verbose)

               

               #update the database
               updatedq(os.path.basename(img), struct, sdb)

               #write the file out
               # housekeeping keywords
               fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
               saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
               saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
               saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
               saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])

               # write FITS file
               saltio.writefits(struct,simg, clobber=clobber)
               saltio.closefits(struct)

               #mosaic the files--currently not in the proper format--will update when it is
               if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
                   mimg=outpath+'mbxgp'+os.path.basename(img)
                   saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
                        interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile,
                        verbose=verbose)

                   #remove the intermediate steps
                   saltio.delete(simg)

               #if the file is spectroscopic mode, apply the wavelength correction
               if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT':
                  dbfile=outpath+obsdate+'_specid.db'
                  try:
                     ximg = outpath+'xmbxgp'+os.path.basename(img)
                     specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line', 
                              function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
                              nw=None, blank=0.0, conserve=True, nearest=True, clobber=True, 
                              logfile=logfile, verbose=verbose)
                  except Exception, e:
                     log.message('%s' % e)


       #clean up the results
       if cleanup:
          #clean up the bias frames
          for i in masterbias_dict.keys():
               blist=masterbias_dict[i][1:]
               for b in blist: saltio.delete(b)

          #clean up the flat frames
          for i in masterflat_dict.keys():
               flist=masterflat_dict[i][1:]
               for f in flist: saltio.delete(f)
Exemple #21
0
def specred(infile_list,
            target,
            propcode,
            calfile=None,
            inter=True,
            automethod='Matchlines'):

    #set up the files
    infiles = ','.join(['%s' % x for x in infile_list])
    obsdate = os.path.basename(infile_list[0])[7:15]

    #set up some files that will be needed
    logfile = 'spec' + obsdate + '.log'
    dbfile = 'spec%s.db' % obsdate

    #create the observation log
    obs_dict = obslog(infile_list)

    for i in range(len(infile_list)):
        if obs_dict['OBJECT'][i].upper().strip(
        ) == 'ARC' and obs_dict['PROPID'][i].upper().strip() == propcode:
            lamp = obs_dict['LAMPID'][i].strip().replace(' ', '')
            arcimage = os.path.basename(infile_list[i])
            if lamp == 'NONE': lamp = 'CuAr'
            lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)

            specidentify(arcimage,
                         lampfile,
                         dbfile,
                         guesstype='rss',
                         guessfile='',
                         automethod=automethod,
                         function='legendre',
                         order=3,
                         rstep=100,
                         rstart='middlerow',
                         mdiff=20,
                         thresh=3,
                         niter=5,
                         smooth=3,
                         inter=False,
                         clobber=True,
                         logfile=logfile,
                         verbose=True)

            specrectify(arcimage,
                        outimages='',
                        outpref='x',
                        solfile=dbfile,
                        caltype='line',
                        function='legendre',
                        order=3,
                        inttype='interp',
                        w1=None,
                        w2=None,
                        dw=None,
                        nw=None,
                        blank=0.0,
                        clobber=True,
                        logfile=logfile,
                        verbose=True)

    objimages = ''
    spec_list = []
    for i in range(len(infile_list)):
        if obs_dict['CCDTYPE'][i].count('OBJECT') and obs_dict['INSTRUME'][
                i].count('RSS') and obs_dict['PROPID'][i].upper().strip(
                ) == propcode:
            img = infile_list[i]
            ##rectify it
            specrectify(img,
                        outimages='',
                        outpref='x',
                        solfile=dbfile,
                        caltype='line',
                        function='legendre',
                        order=3,
                        inttype='interp',
                        w1=None,
                        w2=None,
                        dw=None,
                        nw=None,
                        blank=0.0,
                        clobber=True,
                        logfile=logfile,
                        verbose=True)

            #extract the spectra
            spec_list.append(
                extract_spectra('x' + img,
                                yc=1030,
                                calfile=calfile,
                                findobject=True,
                                smooth=False,
                                maskzeros=True,
                                clobber=True))

    #combine the results
    w, f, e = speccombine(spec_list, obsdate)
    outfile = "%s_%s.spec" % (target, obsdate)
    write_spectra(outfile, w, f, e)
Exemple #22
0
import sys

from pyraf import iraf
from iraf import pysalt
from specidentify import specidentify

from astropy.io import fits

header = fits.open(sys.argv[1])[0].header
lampid = header['LAMPID'].strip().replace(' ', '')
print lampid
specidentify(images=sys.argv[1], linelist='/Users/crawford/programs/pysalt/data/linelists/%s.salt' % lampid, outfile='wav.db', guesstype='rss', guessfile='', automethod='Matchlines', function='poly', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, niter=5, smooth=5, inter=True, startext=0 ,clobber=True, textcolor='black', logfile='salt.log',
verbose=True)
Exemple #23
0
import sys

from pyraf import iraf
from iraf import pysalt
from specidentify import specidentify

specidentify(images=sys.argv[1],
             linelist=sys.argv[2],
             outfile='wav.db',
             guesstype='rss',
             guessfile='',
             automethod='Matchlines',
             function='poly',
             order=3,
             rstep=100,
             rstart='middlerow',
             mdiff=20,
             thresh=3,
             niter=5,
             smooth=3,
             inter=True,
             startext=0,
             clobber=True,
             textcolor='black',
             logfile='salt.log',
             verbose=True)