예제 #1
0
def saltgain(images,
             outimages,
             outpref,
             gaindb=None,
             usedb=False,
             mult=True,
             clobber=True,
             logfile='salt.log',
             verbose=True):

    #start logging
    with logging(logfile, debug) as log:

        # Check the input images
        infiles = saltio.argunpack('Input', images)

        # create list of output files
        outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')

        #verify that the input and output lists are the same length
        saltio.comparelists(infiles, outfiles, 'Input', 'output')

        # read in the database file if usedb is true
        if usedb:
            gaindb = gaindb.strip()
            dblist = saltio.readgaindb(gaindb)
        else:
            dblist = []

        for img, oimg in zip(infiles, outfiles):
            #open the fits file
            struct = saltio.openfits(img)

            # identify instrument
            instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid(
                struct)

            # has file been prepared already?
            if saltkey.found(keygain, struct[0]):
                message = 'SALTGAIN: %s has already been gain-corrected' % img
                raise SaltError(message)

            # gain correct the data
            struct = gain(struct,
                          mult=mult,
                          usedb=usedb,
                          dblist=dblist,
                          log=log,
                          verbose=verbose)

            # housekeeping keywords
            fname, hist = history(level=1,
                                  wrap=False,
                                  exclude=['images', 'outimages', 'outpref'])
            saltkey.housekeeping(struct[0], keygain,
                                 'Images have been gain corrected', hist)

            # write FITS file
            saltio.writefits(struct, oimg, clobber=clobber)
            saltio.closefits(struct)
예제 #2
0
def saltslot(images,
             outimages,
             outpref,
             gaindb='',
             xtalkfile='',
             usedb=False,
             clobber=False,
             logfile='salt.log',
             verbose=True):

    #start logging
    with logging(logfile, debug) as log:

        # Check the input images
        infiles = saltio.argunpack('Input', images)

        # create list of output files
        outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')

        # are input and output lists the same length?
        saltio.comparelists(infiles, outfiles, 'Input', 'output')

        # does crosstalk coefficient data exist
        if usedb:
            dblist = saltio.readgaindb(gaindb)
            xtalkfile = xtalkfile.strip()
            xdict = saltio.readxtalkcoeff(xtalkfile)
        else:
            dblist = []
            xdict = None

        for img, oimg in zip(infiles, outfiles):
            #open the fits file
            struct = saltio.openfits(img)

            # identify instrument
            instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid(
                struct)

            # has file been prepared already?
            if saltkey.found(keygain, struct[0]):
                message = '%s has already been reduced' % img
                raise SaltError(message)

            # housekeeping keywords
            fname, hist = history(level=1,
                                  wrap=False,
                                  exclude=['images', 'outimages', 'outpref'])
            saltkey.housekeeping(struct[0], keyslot,
                                 'Images have been slotmode reduced', hist)

            # write FITS file
            saltio.writefits(struct, oimg, clobber=clobber)
            saltio.closefits(struct)
예제 #3
0
def saltgain(images,outimages, outpref, gaindb=None,usedb=False, mult=True,
             clobber=True, logfile='salt.log',verbose=True):

   #start logging
   with logging(logfile,debug) as log:

       # Check the input images 
       infiles = saltio.argunpack ('Input',images)

       # create list of output files 
       outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')

       #verify that the input and output lists are the same length
       saltio.comparelists(infiles,outfiles,'Input','output')

       # read in the database file if usedb is true
       if usedb:
           gaindb = gaindb.strip()
           dblist= saltio.readgaindb(gaindb)
       else:
           dblist=[]


       for img, oimg in zip(infiles, outfiles):
           #open the fits file
           struct=saltio.openfits(img)

           # identify instrument
           instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)

           # has file been prepared already?
           if saltkey.found(keygain, struct[0]):
               message='SALTGAIN: %s has already been gain-corrected' % img
               raise SaltError(message)


           # gain correct the data
           struct = gain(struct,mult=mult, usedb=usedb, dblist=dblist, log=log, verbose=verbose)

           # housekeeping keywords
           fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
           saltkey.housekeeping(struct[0],keygain, 'Images have been gain corrected', hist)

           # write FITS file
           saltio.writefits(struct,oimg, clobber=clobber)
           saltio.closefits(struct)
예제 #4
0
파일: saltslot.py 프로젝트: dr-jpk/pysalt
def saltslot(images,outimages,outpref,gaindb='',xtalkfile='',usedb=False, clobber=False,logfile='salt.log',verbose=True):


   #start logging
   with logging(logfile,debug) as log:

       # Check the input images 
       infiles = saltio.argunpack ('Input',images)

       # create list of output files 
       outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')

       # are input and output lists the same length?
       saltio.comparelists(infiles,outfiles,'Input','output')

       # does crosstalk coefficient data exist
       if usedb:
           dblist= saltio.readgaindb(gaindb)
           xtalkfile = xtalkfile.strip()
           xdict = saltio.readxtalkcoeff(xtalkfile)
       else:
           dblist=[]
           xdict=None

       for img, oimg in zip(infiles, outfiles):
           #open the fits file
           struct=saltio.openfits(img)

           # identify instrument
           instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)

           # has file been prepared already?
           if saltkey.found(keygain, struct[0]):
               message='%s has already been reduced' % img
               raise SaltError(message)



           # housekeeping keywords
           fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
           saltkey.housekeeping(struct[0],keyslot, 'Images have been slotmode reduced', hist)

           # write FITS file
           saltio.writefits(struct,oimg, clobber=clobber)
           saltio.closefits(struct)
예제 #5
0
def saltclean(images,
              outpath,
              obslogfile=None,
              gaindb=None,
              xtalkfile=None,
              geomfile=None,
              subover=True,
              trim=True,
              masbias=None,
              subbias=False,
              median=False,
              function='polynomial',
              order=5,
              rej_lo=3,
              rej_hi=3,
              niter=5,
              interp='linear',
              clobber=False,
              logfile='salt.log',
              verbose=True):
    """SALTCLEAN will provide basic CCD reductions for a set of data.  It will 
      sort the data, and first process the biases, flats, and then the science 
      frames.  It will record basic quality control information about each of 
      the steps.
   """
    plotover = False

    #start logging
    with logging(logfile, debug) as log:

        # Check the input images
        infiles = saltio.argunpack('Input', images)

        # create list of output files
        outpath = saltio.abspath(outpath)

        #does the gain database file exist
        if gaindb:
            dblist = saltio.readgaindb(gaindb)
        else:
            dblist = []

        # does crosstalk coefficient data exist
        if xtalkfile:
            xtalkfile = xtalkfile.strip()
            xdict = saltio.readxtalkcoeff(xtalkfile)
        else:
            xdict = None
        #does the mosaic file exist--raise error if no
        saltio.fileexists(geomfile)

        # Delete the obslog file if it already exists
        if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)

        #read in the obsveration log or create it
        if os.path.isfile(obslogfile):
            msg = 'The observing log already exists.  Please either delete it or run saltclean with clobber=yes'
            raise SaltError(msg)
        else:
            headerDict = obslog(infiles, log)
            obsstruct = createobslogfits(headerDict)
            saltio.writefits(obsstruct, obslogfile)

        #create the list of bias frames and process them
        filename = obsstruct.data.field('FILENAME')
        detmode = obsstruct.data.field('DETMODE')
        ccdtype = obsstruct.data.field('CCDTYPE')

        #set the bias list of objects
        biaslist = filename[ccdtype == 'ZERO']
        masterbias_dict = {}
        for img in infiles:
            if os.path.basename(img) in biaslist:
                #open the image
                struct = fits.open(img)
                bimg = outpath + 'bxgp' + os.path.basename(img)

                #print the message
                if log:
                    message = 'Processing Zero frame %s' % img
                    log.message(message, with_stdout=verbose)

                #process the image
                struct = clean(struct,
                               createvar=False,
                               badpixelstruct=None,
                               mult=True,
                               dblist=dblist,
                               xdict=xdict,
                               subover=subover,
                               trim=trim,
                               subbias=False,
                               bstruct=None,
                               median=median,
                               function=function,
                               order=order,
                               rej_lo=rej_lo,
                               rej_hi=rej_hi,
                               niter=niter,
                               plotover=plotover,
                               log=log,
                               verbose=verbose)

                #write the file out
                # housekeeping keywords
                fname, hist = history(
                    level=1,
                    wrap=False,
                    exclude=['images', 'outimages', 'outpref'])
                saltkey.housekeeping(struct[0], 'SPREPARE',
                                     'Images have been prepared', hist)
                saltkey.new('SGAIN', time.asctime(time.localtime()),
                            'Images have been gain corrected', struct[0])
                saltkey.new('SXTALK', time.asctime(time.localtime()),
                            'Images have been xtalk corrected', struct[0])
                saltkey.new('SBIAS', time.asctime(time.localtime()),
                            'Images have been de-biased', struct[0])

                # write FITS file
                saltio.writefits(struct, bimg, clobber=clobber)
                saltio.closefits(struct)

                #add files to the master bias list
                masterbias_dict = compareimages(struct,
                                                bimg,
                                                masterbias_dict,
                                                keylist=biasheader_list)

        #create the master bias frame
        for i in masterbias_dict.keys():
            bkeys = masterbias_dict[i][0]
            blist = masterbias_dict[i][1:]
            mbiasname = outpath + createmasterbiasname(blist, bkeys)
            bfiles = ','.join(blist)
            saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
                        weight=False, blank=0, scale=None, statsec=None, lthresh=3,    \
                        hthresh=3, clobber=False, logfile=logfile,verbose=verbose)

        #create the list of flatfields and process them
        flatlist = filename[ccdtype == 'FLAT']
        masterflat_dict = {}
        for img in infiles:
            if os.path.basename(img) in flatlist:
                #open the image
                struct = fits.open(img)
                fimg = outpath + 'bxgp' + os.path.basename(img)

                #print the message
                if log:
                    message = 'Processing Flat frame %s' % img
                    log.message(message, with_stdout=verbose)

                #process the image
                struct = clean(struct,
                               createvar=False,
                               badpixelstruct=None,
                               mult=True,
                               dblist=dblist,
                               xdict=xdict,
                               subover=subover,
                               trim=trim,
                               subbias=False,
                               bstruct=None,
                               median=median,
                               function=function,
                               order=order,
                               rej_lo=rej_lo,
                               rej_hi=rej_hi,
                               niter=niter,
                               plotover=plotover,
                               log=log,
                               verbose=verbose)

                #write the file out
                # housekeeping keywords
                fname, hist = history(
                    level=1,
                    wrap=False,
                    exclude=['images', 'outimages', 'outpref'])
                saltkey.housekeeping(struct[0], 'SPREPARE',
                                     'Images have been prepared', hist)
                saltkey.new('SGAIN', time.asctime(time.localtime()),
                            'Images have been gain corrected', struct[0])
                saltkey.new('SXTALK', time.asctime(time.localtime()),
                            'Images have been xtalk corrected', struct[0])
                saltkey.new('SBIAS', time.asctime(time.localtime()),
                            'Images have been de-biased', struct[0])

                # write FITS file
                saltio.writefits(struct, fimg, clobber=clobber)
                saltio.closefits(struct)

                #add files to the master bias list
                masterflat_dict = compareimages(struct,
                                                fimg,
                                                masterflat_dict,
                                                keylist=flatheader_list)

        #create the master flat frame
        for i in masterflat_dict.keys():
            fkeys = masterflat_dict[i][0]
            flist = masterflat_dict[i][1:]
            mflatname = outpath + createmasterflatname(flist, fkeys)
            ffiles = ','.join(flist)
            saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
                        weight=False, blank=0, scale=None, statsec=None, lthresh=3,    \
                        hthresh=3, clobber=False, logfile=logfile,verbose=verbose)

        #process the science data
        for img in infiles:
            nimg = os.path.basename(img)
            #print nimg, nimg in flatlist, nimg in biaslist
            if not (nimg in biaslist):
                #open the image
                struct = fits.open(img)
                simg = outpath + 'bxgp' + os.path.basename(img)

                #print the message
                if log:
                    message = 'Processing science frame %s' % img
                    log.message(message, with_stdout=verbose)

                #process the image
                struct = clean(struct,
                               createvar=False,
                               badpixelstruct=None,
                               mult=True,
                               dblist=dblist,
                               xdict=xdict,
                               subover=subover,
                               trim=trim,
                               subbias=False,
                               bstruct=None,
                               median=median,
                               function=function,
                               order=order,
                               rej_lo=rej_lo,
                               rej_hi=rej_hi,
                               niter=niter,
                               plotover=plotover,
                               log=log,
                               verbose=verbose)

                #write the file out
                # housekeeping keywords
                fname, hist = history(
                    level=1,
                    wrap=False,
                    exclude=['images', 'outimages', 'outpref'])
                saltkey.housekeeping(struct[0], 'SPREPARE',
                                     'Images have been prepared', hist)
                saltkey.new('SGAIN', time.asctime(time.localtime()),
                            'Images have been gain corrected', struct[0])
                saltkey.new('SXTALK', time.asctime(time.localtime()),
                            'Images have been xtalk corrected', struct[0])
                saltkey.new('SBIAS', time.asctime(time.localtime()),
                            'Images have been de-biased', struct[0])

                # write FITS file
                saltio.writefits(struct, simg, clobber=clobber)
                saltio.closefits(struct)

                #mosaic the files--currently not in the proper format--will update when it is
                if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
                    mimg = outpath + 'mbxgp' + os.path.basename(img)
                    saltmosaic(images=simg,
                               outimages=mimg,
                               outpref='',
                               geomfile=geomfile,
                               interp=interp,
                               cleanup=True,
                               clobber=clobber,
                               logfile=logfile,
                               verbose=verbose)

                    #remove the intermediate steps
                    saltio.delete(simg)
예제 #6
0
def quickclean(filename, interp='linear', cleanup=True, clobber=False, logfile='saltclean.log', verbose=True):
   """Start the process to reduce the data and produce a single mosaicked image"""
   print filename

   #create the input file name
   status=0
   infile=os.path.basename(filename)
   rawpath=os.path.dirname(filename)
   outpath='./'
   outfile=outpath+'mbxp'+infile
   print infile, rawpath, outpath

   #check to see if it exists and return if clobber is no
   if os.path.isfile(outfile) and not clobber: return

   #set up the files needed
   if infile[0]=='P':
     gaindb = iraf.osfn('pysalt$data/rss/RSSamps.dat')
     xtalkfile = iraf.osfn('pysalt$data/rss/RSSxtalk.dat')
     geomfile = iraf.osfn('pysalt$data/rss/RSSgeom.dat')
   elif infile[0]=='S':
     gaindb = iraf.osfn('pysalt$data/scam/SALTICAMamps.dat')
     xtalkfile = iraf.osfn('pysalt$data/scam/SALTICAMxtalk.dat')
     geomfile = iraf.osfn('pysalt$data/scam/SALTICAMgeom.dat')
 
   #verify the file
   struct=saltio.openfits(rawpath+'/'+infile)
   struct.verify('exception')
   
   #check to see if detmode is there
   if not saltkey.found('DETMODE', struct[0]): 
      return 
 
   #reduce the file
   struct=prepare(struct, createvar=False, badpixelstruct=None)
 
      #reset the names in the structures
   for i in range(1,len(struct)):
       struct[i].name=struct[i].header['EXTNAME']


   #gain correct the files
   usedb=True
   dblist= saltio.readgaindb(gaindb)
   log=open(logfile, 'a')
   ampccd = struct[0].header['NAMPS'] / struct[0].header['NCCDS']
   struct=gain(struct, mult=True,usedb=usedb, dblist=dblist, ampccd=ampccd, log=None, verbose=verbose)

   struct=bias(struct, subover=True,trim=True,subbias=False, 
                    median=False,function='polynomial',order=5,rej_lo=3,rej_hi=3,niter=10,
                    plotover=False,log=None, verbose=verbose)

   if struct[0].header['CCDTYPE']=='OBJECT' and struct[0].header['EXPTIME']>90:
      struct = multicrclean(struct, crtype='median', thresh=5, mbox=5, bbox=25, bthresh=5, flux_ratio=0.2, \
                          gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=5, log=None, verbose=verbose)

   pinfile=outpath+'bxp'+infile
   saltio.writefits(struct, pinfile, clobber)

   saltred.saltmosaic(images=pinfile,
                   outimages='',outpref=outpath+'m',geomfile=geomfile, fill=True,
                   interp=interp,cleanup=cleanup,clobber=clobber,logfile=logfile,
                   verbose=verbose)
   profile=outpath+'mbxp'+infile

   #remove intermediate steps
   if cleanup:
      if os.path.isfile(pinfile): os.remove(pinfile)

   return
예제 #7
0
파일: saltclean.py 프로젝트: dr-jpk/pysalt
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None, 
	geomfile=None,subover=True,trim=True,masbias=None, 
        subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
        rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log', 
        verbose=True):
   """SALTCLEAN will provide basic CCD reductions for a set of data.  It will 
      sort the data, and first process the biases, flats, and then the science 
      frames.  It will record basic quality control information about each of 
      the steps.
   """
   plotover=False

   #start logging
   with logging(logfile,debug) as log:

       # Check the input images 
       infiles = saltio.argunpack ('Input',images)

       # create list of output files 
       outpath=saltio.abspath(outpath)


       #does the gain database file exist
       if gaindb:
           dblist= saltio.readgaindb(gaindb)
       else:
           dblist=[]

       # does crosstalk coefficient data exist
       if xtalkfile:
           xtalkfile = xtalkfile.strip()
           xdict = saltio.readxtalkcoeff(xtalkfile)
       else:
           xdict=None
       #does the mosaic file exist--raise error if no
       saltio.fileexists(geomfile)


       # Delete the obslog file if it already exists
       if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)

       #read in the obsveration log or create it
       if os.path.isfile(obslogfile):
           msg='The observing log already exists.  Please either delete it or run saltclean with clobber=yes'
           raise SaltError(msg)
       else:
           headerDict=obslog(infiles, log)
           obsstruct=createobslogfits(headerDict)
           saltio.writefits(obsstruct, obslogfile)

       #create the list of bias frames and process them
       filename=obsstruct.data.field('FILENAME')
       detmode=obsstruct.data.field('DETMODE')
       ccdtype=obsstruct.data.field('CCDTYPE')

       #set the bias list of objects
       biaslist=filename[ccdtype=='ZERO']
       masterbias_dict={}
       for img in infiles:
           if os.path.basename(img) in biaslist:
               #open the image
               struct=pyfits.open(img)
               bimg=outpath+'bxgp'+os.path.basename(img)

               #print the message
               if log:
                   message='Processing Zero frame %s' % img
                   log.message(message, with_stdout=verbose)

               #process the image
               struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
                            bstruct=None, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
                            verbose=verbose)

               #write the file out
               # housekeeping keywords
               fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
               saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
               saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
               saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
               saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])

               # write FITS file
               saltio.writefits(struct,bimg, clobber=clobber)
               saltio.closefits(struct)

               #add files to the master bias list
               masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)

       #create the master bias frame
       for i in masterbias_dict.keys():
           bkeys=masterbias_dict[i][0]
           blist=masterbias_dict[i][1:]
           mbiasname=outpath+createmasterbiasname(blist, bkeys)
           bfiles=','.join(blist)
           saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, 
                       weight=False, blank=0, scale=None, statsec=None, lthresh=3,    \
                       hthresh=3, clobber=False, logfile=logfile,verbose=verbose)

           

       #create the list of flatfields and process them
       flatlist=filename[ccdtype=='FLAT']
       masterflat_dict={}
       for img in infiles:
           if os.path.basename(img) in flatlist:
               #open the image
               struct=pyfits.open(img)
               fimg=outpath+'bxgp'+os.path.basename(img)

               #print the message
               if log:
                   message='Processing Flat frame %s' % img
                   log.message(message, with_stdout=verbose)

               #process the image
               struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
                            bstruct=None, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
                            verbose=verbose)

               #write the file out
               # housekeeping keywords
               fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
               saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
               saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
               saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
               saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])

               # write FITS file
               saltio.writefits(struct,fimg, clobber=clobber)
               saltio.closefits(struct)

               #add files to the master bias list
               masterflat_dict=compareimages(struct, fimg, masterflat_dict,  keylist=flatheader_list)

       #create the master flat frame
       for i in masterflat_dict.keys():
           fkeys=masterflat_dict[i][0]
           flist=masterflat_dict[i][1:]
           mflatname=outpath+createmasterflatname(flist, fkeys)
           ffiles=','.join(flist)
           saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, 
                       weight=False, blank=0, scale=None, statsec=None, lthresh=3,    \
                       hthresh=3, clobber=False, logfile=logfile,verbose=verbose)

       #process the science data
       for img in infiles:
           nimg=os.path.basename(img)
           if not nimg in flatlist or not nimg in biaslist:
               #open the image
               struct=pyfits.open(img)
               simg=outpath+'bxgp'+os.path.basename(img)

               #print the message
               if log:
                   message='Processing science frame %s' % img
                   log.message(message, with_stdout=verbose)

               #process the image
               struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
                            bstruct=None, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
                            verbose=verbose)

               #write the file out
               # housekeeping keywords
               fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
               saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
               saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
               saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
               saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])

               # write FITS file
               saltio.writefits(struct,simg, clobber=clobber)
               saltio.closefits(struct)

               #mosaic the files--currently not in the proper format--will update when it is
               if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
                   mimg=outpath+'mbxgp'+os.path.basename(img)
                   saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
                        interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
                        verbose=verbose)

                   #remove the intermediate steps
                   saltio.delete(simg)
예제 #8
0
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None, 
	geomfile=None,subover=True,trim=True,masbias=None, 
        subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
        rej_hi=3,niter=5,interp='linear',  sdbhost='',sdbname='',sdbuser='', password='',
        clobber=False, cleanup=True, logfile='salt.log', verbose=True):
   """SALTADVANCE provides advanced data reductions for a set of data.  It will 
      sort the data, and first process the biases, flats, and then the science 
      frames.  It will record basic quality control information about each of 
      the steps.
   """
   plotover=False

   #start logging
   with logging(logfile,debug) as log:

       # Check the input images 
       infiles = saltio.argunpack ('Input',images)
       infiles.sort()

       # create list of output files 
       outpath=saltio.abspath(outpath)

       #log into the database
       sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)

       #does the gain database file exist
       if gaindb:
           dblist= saltio.readgaindb(gaindb)
       else:
           dblist=[]

       # does crosstalk coefficient data exist
       if xtalkfile:
           xtalkfile = xtalkfile.strip()
           xdict = saltio.readxtalkcoeff(xtalkfile)
       else:
           xdict=None
       #does the mosaic file exist--raise error if no
       saltio.fileexists(geomfile)


       # Delete the obslog file if it already exists
       if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)

       #read in the obsveration log or create it
       if os.path.isfile(obslogfile):
           msg='The observing log already exists.  Please either delete it or run saltclean with clobber=yes'
           raise SaltError(msg)
       else:
           headerDict=obslog(infiles, log)
           obsstruct=createobslogfits(headerDict)
           saltio.writefits(obsstruct, obslogfile)

       #create the list of bias frames and process them
       filename=obsstruct.data.field('FILENAME')
       detmode=obsstruct.data.field('DETMODE')
       obsmode=obsstruct.data.field('OBSMODE')
       ccdtype=obsstruct.data.field('CCDTYPE')
       propcode=obsstruct.data.field('PROPID')
       masktype=obsstruct.data.field('MASKTYP')

       #set the bias list of objects
       biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')]
       masterbias_dict={}
       for img in infiles:
           if os.path.basename(img) in biaslist:
               #open the image
               struct=fits.open(img)
               bimg=outpath+'bxgp'+os.path.basename(img)

               #print the message
               if log:
                   message='Processing Zero frame %s' % img
                   log.message(message, with_stdout=verbose)

               #process the image
               struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
                            bstruct=None, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
                            verbose=verbose)
 
               #update the database
               updatedq(os.path.basename(img), struct, sdb)

               #write the file out
               # housekeeping keywords
               fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
               saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
               saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
               saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
               saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])

               # write FITS file
               saltio.writefits(struct,bimg, clobber=clobber)
               saltio.closefits(struct)

               #add files to the master bias list
               masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)

       #create the master bias frame
       for i in masterbias_dict.keys():
           bkeys=masterbias_dict[i][0]
           blist=masterbias_dict[i][1:]
           mbiasname=outpath+createmasterbiasname(blist, bkeys)
           bfiles=','.join(blist)
           saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, 
                       weight=False, blank=0, scale=None, statsec=None, lthresh=3,    \
                       hthresh=3, clobber=False, logfile=logfile,verbose=verbose)

           

       #create the list of flatfields and process them
       flatlist=filename[ccdtype=='FLAT']
       masterflat_dict={}
       for img in infiles:
           if os.path.basename(img) in flatlist:
               #open the image
               struct=fits.open(img)
               fimg=outpath+'bxgp'+os.path.basename(img)

               #print the message
               if log:
                   message='Processing Flat frame %s' % img
                   log.message(message, with_stdout=verbose)

               #process the image
               struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
                            bstruct=None, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
                            verbose=verbose)

               #update the database
               updatedq(os.path.basename(img), struct, sdb)

               #write the file out
               # housekeeping keywords
               fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
               saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
               saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
               saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
               saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])

               # write FITS file
               saltio.writefits(struct,fimg, clobber=clobber)
               saltio.closefits(struct)

               #add files to the master bias list
               masterflat_dict=compareimages(struct, fimg, masterflat_dict,  keylist=flatheader_list)

       #create the master flat frame
       for i in masterflat_dict.keys():
           fkeys=masterflat_dict[i][0]
           flist=masterflat_dict[i][1:]
           mflatname=outpath+createmasterflatname(flist, fkeys)
           ffiles=','.join(flist)
           saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, 
                       weight=False, blank=0, scale=None, statsec=None, lthresh=3,    \
                       hthresh=3, clobber=False, logfile=logfile,verbose=verbose)

       #process the arc data
       arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')]
       for i, img in enumerate(infiles):
           nimg=os.path.basename(img)
           if nimg in arclist:
               #open the image
               struct=fits.open(img)
               simg=outpath+'bxgp'+os.path.basename(img)
               obsdate=os.path.basename(img)[1:9]

               #print the message
               if log:
                   message='Processing ARC frame %s' % img
                   log.message(message, with_stdout=verbose)


               struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
                            bstruct=None, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, 
                            log=log, verbose=verbose)

               # write FITS file
               saltio.writefits(struct,simg, clobber=clobber)
               saltio.closefits(struct)

               #mosaic the images
               mimg=outpath+'mbxgp'+os.path.basename(img)
               saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
                    interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
                    verbose=verbose)

               #remove the intermediate steps
               saltio.delete(simg)


               #measure the arcdata
               arcimage=outpath+'mbxgp'+nimg
               dbfile=outpath+obsdate+'_specid.db'
               lamp = obsstruct.data.field('LAMPID')[i]
               lamp = lamp.replace(' ', '')
               lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
               print arcimage, lampfile, os.getcwd()
               specidentify(arcimage, lampfile, dbfile, guesstype='rss', 
                                guessfile='', automethod='Matchlines', function='legendre',
                                order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3,
                                startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile, 
                                verbose=verbose)
               try:
                   ximg = outpath+'xmbxgp'+os.path.basename(arcimage)
                   specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
                              function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
                              nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
                              logfile=logfile, verbose=verbose)
               except:
                   pass


              
       #process the science data
       for i, img in enumerate(infiles):
           nimg=os.path.basename(img)
           if not (nimg in flatlist or nimg in biaslist or nimg in arclist):
     
               #open the image
               struct=fits.open(img)
               if struct[0].header['PROPID'].count('CAL_GAIN'): continue
               simg=outpath+'bxgp'+os.path.basename(img)
   

               #print the message
               if log:
                   message='Processing science frame %s' % img
                   log.message(message, with_stdout=verbose)


               #Check to see if it is RSS 2x2 and add bias subtraction
               instrume=saltkey.get('INSTRUME', struct[0]).strip()
               gainset = saltkey.get('GAINSET', struct[0])    
               rospeed = saltkey.get('ROSPEED', struct[0])    
               target = saltkey.get('OBJECT', struct[0]).strip()
               exptime = saltkey.get('EXPTIME', struct[0])
               obsmode = saltkey.get('OBSMODE', struct[0]).strip()
               detmode = saltkey.get('DETMODE', struct[0]).strip()
               masktype = saltkey.get('MASKTYP', struct[0]).strip()
  
               
               xbin, ybin = saltkey.ccdbin( struct[0], img)
               obsdate=os.path.basename(img)[1:9]
               bstruct=None
               crtype=None
               thresh=5 
               mbox=11 
               bthresh=5.0,
               flux_ratio=0.2 
               bbox=25 
               gain=1.0 
               rdnoise=5.0 
               fthresh=5.0 
               bfactor=2
               gbox=3 
               maxiter=5
    
               subbias=False
               if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW':
                   bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin)
                   if os.path.exists(bfile):
                      bstruct=fits.open(bfile)
                      subbias=True
                   if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5:
                       crtype='edge' 
                       thresh=5 
                       mbox=11 
                       bthresh=5.0,
                       flux_ratio=0.2 
                       bbox=25 
                       gain=1.0 
                       rdnoise=5.0 
                       fthresh=5.0 
                       bfactor=2
                       gbox=3 
                       maxiter=3
    
               #process the image
               struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, 
                            dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias,
                            bstruct=bstruct, median=median, function=function, order=order,
                            rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, 
                            crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox,      \
                            bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise, 
                            bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter,
                            log=log, verbose=verbose)

               

               #update the database
               updatedq(os.path.basename(img), struct, sdb)

               #write the file out
               # housekeeping keywords
               fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
               saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
               saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
               saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
               saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])

               # write FITS file
               saltio.writefits(struct,simg, clobber=clobber)
               saltio.closefits(struct)

               #mosaic the files--currently not in the proper format--will update when it is
               if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
                   mimg=outpath+'mbxgp'+os.path.basename(img)
                   saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
                        interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile,
                        verbose=verbose)

                   #remove the intermediate steps
                   saltio.delete(simg)

               #if the file is spectroscopic mode, apply the wavelength correction
               if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT':
                  dbfile=outpath+obsdate+'_specid.db'
                  try:
                     ximg = outpath+'xmbxgp'+os.path.basename(img)
                     specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line', 
                              function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
                              nw=None, blank=0.0, conserve=True, nearest=True, clobber=True, 
                              logfile=logfile, verbose=verbose)
                  except Exception, e:
                     log.message('%s' % e)


       #clean up the results
       if cleanup:
          #clean up the bias frames
          for i in masterbias_dict.keys():
               blist=masterbias_dict[i][1:]
               for b in blist: saltio.delete(b)

          #clean up the flat frames
          for i in masterflat_dict.keys():
               flist=masterflat_dict[i][1:]
               for f in flist: saltio.delete(f)
예제 #9
0
def quickclean(filename,
               interp='linear',
               cleanup=True,
               clobber=False,
               logfile='saltclean.log',
               verbose=True):
    """Start the process to reduce the data and produce a single mosaicked image"""
    print filename

    #create the input file name
    status = 0
    infile = os.path.basename(filename)
    rawpath = os.path.dirname(filename)
    outpath = './'
    outfile = outpath + 'mbxp' + infile
    print infile, rawpath, outpath

    #check to see if it exists and return if clobber is no
    if os.path.isfile(outfile) and not clobber: return

    #set up the files needed
    if infile[0] == 'P':
        gaindb = iraf.osfn('pysalt$data/rss/RSSamps.dat')
        xtalkfile = iraf.osfn('pysalt$data/rss/RSSxtalk.dat')
        geomfile = iraf.osfn('pysalt$data/rss/RSSgeom.dat')
    elif infile[0] == 'S':
        gaindb = iraf.osfn('pysalt$data/scam/SALTICAMamps.dat')
        xtalkfile = iraf.osfn('pysalt$data/scam/SALTICAMxtalk.dat')
        geomfile = iraf.osfn('pysalt$data/scam/SALTICAMgeom.dat')

    #verify the file
    struct = saltio.openfits(rawpath + '/' + infile)
    struct.verify('exception')

    #check to see if detmode is there
    if not saltkey.found('DETMODE', struct[0]):
        return

    #reduce the file
    struct = prepare(struct, createvar=False, badpixelstruct=None)

    #reset the names in the structures
    for i in range(1, len(struct)):
        struct[i].name = struct[i].header['EXTNAME']

    #gain correct the files
    usedb = True
    dblist = saltio.readgaindb(gaindb)
    log = open(logfile, 'a')
    ampccd = struct[0].header['NAMPS'] / struct[0].header['NCCDS']
    struct = gain(struct,
                  mult=True,
                  usedb=usedb,
                  dblist=dblist,
                  ampccd=ampccd,
                  log=None,
                  verbose=verbose)

    struct = bias(struct,
                  subover=True,
                  trim=True,
                  subbias=False,
                  median=False,
                  function='polynomial',
                  order=5,
                  rej_lo=3,
                  rej_hi=3,
                  niter=10,
                  plotover=False,
                  log=None,
                  verbose=verbose)

    if struct[0].header[
            'CCDTYPE'] == 'OBJECT' and struct[0].header['EXPTIME'] > 90:
        struct = multicrclean(struct, crtype='median', thresh=5, mbox=5, bbox=25, bthresh=5, flux_ratio=0.2, \
                            gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=5, log=None, verbose=verbose)

    pinfile = outpath + 'bxp' + infile
    saltio.writefits(struct, pinfile, clobber)

    saltred.saltmosaic(images=pinfile,
                       outimages='',
                       outpref=outpath + 'm',
                       geomfile=geomfile,
                       fill=True,
                       interp=interp,
                       cleanup=cleanup,
                       clobber=clobber,
                       logfile=logfile,
                       verbose=verbose)
    profile = outpath + 'mbxp' + infile

    #remove intermediate steps
    if cleanup:
        if os.path.isfile(pinfile): os.remove(pinfile)

    return