Example #1
0
def imred(infilelist, prodir, bpmfile=None, crthresh='', gaindb = None, cleanup=True):
    #get the name of the files
    infiles=','.join(['%s' % x for x in infilelist])
    
    #get the current date for the files
    obsdate=os.path.basename(infilelist[0])[1:9]
    print "Observation Date: ",obsdate

    #set up some files that will be needed
    logfile='im'+obsdate+'.log'
    flatimage='FLAT%s.fits' % (obsdate)
    dbfile='spec%s.db' % obsdate
    geomline = datedline(datadir+'RSSgeom.dat',obsdate)
    if len(geomline) == 0:
        print 'Invalid geometry file, ',datadir+'RSSgeom.dat',', exitting'
        exit() 
    geomfile=obsdate+'_geom.txt'
    open(geomfile,'w').write(geomline)

    #create the observation log
#    obs_dict=obslog(infilelist)

    verbose=True

    with logging(logfile, debug) as log:
        log.message('Pysalt Version: '+pysalt.verno, with_header=False)
 
    #prepare the data

        for img in infilelist:
            hdu = pyfits.open(img)

            # for backwards compatibility
            hdu = remove_duplicate_keys(hdu)  
            if not 'XTALK' in hdu[1].header:
                hdu[1].header['XTALK']=1474
                hdu[2].header['XTALK']=1474
                hdu[3].header['XTALK']=1166
                hdu[4].header['XTALK']=1111
                hdu[5].header['XTALK']=1377
                hdu[6].header['XTALK']=1377

            img = os.path.basename(img)
                                                                    
            hdu = prepare(hdu, createvar=False, badpixelstruct=None)
            if not cleanup: hdu.writeto('p'+img, overwrite=True)

            hdu = bias(hdu,subover=True, trim=True, subbias=False,
                       bstruct=None, median=False, function='polynomial',
                       order=5, rej_lo=5.0, rej_hi=5.0, niter=10,
                       plotover=False, log=log, verbose=verbose)    
            if not cleanup: hdu.writeto('bp'+img, overwrite=True)

            # put windowed data into full image
            exts = len(hdu)
            if exts > 7:
                rows, cols = hdu[1].data.shape
                cbin, rbin = [int(x) for x in hdu[0].header['CCDSUM'].split(" ")]
                ampsecO = hdu[1].header["AMPSEC"].strip("[]").split(",")
                ampsecE = hdu[7].header["AMPSEC"].strip("[]").split(",")
                rO = int((float(ampsecO[1].split(":")[0]) - 1.)/rbin)
                rE = int((float(ampsecE[1].split(":")[0]) - 1.)/rbin)
                keylist = ['BIASSEC','DATASEC','AMPSEC','CCDSEC','DETSEC']
                oldlist = [hdu[1].header[key].strip("[]").split(",")[1] for key in keylist]
                newlist = 2*['1:'+str(int(0.5+4102/rbin))]+3*[str(int(rbin/2))+':4102']

                for amp in range(6):
                    hduO = hdu[amp+1].copy()                    
                    hdu[amp+1].data = np.zeros((4102/rbin,cols))
                    hdu[amp+1].data[rO:rO+rows] = hduO.data
                    hdu[amp+1].data[rE:rE+rows] = hdu[amp+7].data
                    hdu[amp+1].update_header
                    for k,key in enumerate(keylist): 
                        hdu[amp+1].header[key] = \
                            hdu[amp+1].header[key].replace(oldlist[k],newlist[k])
                del hdu[7:]
                hdu[0].header['NSCIEXT'] = 6
     
            badpixelstruct = saltio.openfits(bpmfile)
            hdu = add_variance(hdu, badpixelstruct)
             
            #gain correct the data 
            if gaindb: 
                usedb = True
                dblist = saltio.readgaindb(gaindb.strip())
            else:
                usedb = False
                dblist = ''
            hdu = gain(hdu, mult=True, usedb=usedb, dblist=dblist, log=log, verbose=verbose)
            if not cleanup: hdu.writeto('gbp'+img, overwrite=True)

            #cross talk correct the data
            hdu=xtalk(hdu, [], log=log, verbose=verbose)

            #cosmic ray clean the data
            #only clean the object data            
            if crthresh=='':
                thresh = 5.0
                if hdu[0].header['GRATING'].strip()=='PG0300': thresh = 7.0
            else: thresh=crthresh

            if hdu[0].header['CCDTYPE']=='OBJECT' and \
                hdu[0].header['LAMPID']=='NONE' and \
                hdu[0].header['INSTRUME']=='RSS':
                if crthresh != False:
                    log.message('Cleaning CR using thresh={}'.format(thresh))
                    hdu = multicrclean(hdu, crtype='edge', thresh=thresh, mbox=11, bthresh=5.0,
                        flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
                        gbox=3, maxiter=5, log=log, verbose=verbose)
                    for ext in range(13,19): hdu[ext].data = hdu[ext].data.astype('uint8')
                    hdu[0].header.add_history('CRCLEAN: multicrclean, thresh = ',thresh)
                else:
                    hdu[0].header.add_history('CRCLEAN: None')
            hdu.writeto('xgbp'+img, overwrite=True)
            hdu.close()
        
    #mosaic the data
    #khn: attempt to use most recent previous geometry to obsdate.  

    #geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
    
    try:
       saltmosaic('xgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
    except:
       saltmosaic('xgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
    #khn: fix mosaiced VAR and BPM extensions
    #khn: fix mosaiced bpm missing some of gap
    for img in infilelist:
        filename = 'mxgbp'+os.path.basename(img)
        hdu = pyfits.open(filename, 'update')
        hdu[2].header['EXTNAME'] = 'VAR'
        hdu[3].header['EXTNAME'] = 'BPM'
        bpm_rc = (hdu[3].data>0).astype('uint8')
        zeroscicol = hdu['SCI'].data.sum(axis=0) == 0
        bpmgapcol = bpm_rc.mean(axis=0) == 1
        addbpmcol = zeroscicol & ~bpmgapcol
        addbpmcol[np.argmax(addbpmcol)-4:np.argmax(addbpmcol)] = True    # allow for chip tilt
        bpm_rc[:,addbpmcol] = 1
        hdu[3].data = bpm_rc
        hdu.writeto(filename,overwrite=True)

    #clean up the images
    if cleanup:
           for f in glob.glob('p*fits'): os.remove(f)
           for f in glob.glob('bp*fits'): os.remove(f)
           for f in glob.glob('gbp*fits'): os.remove(f)
           for f in glob.glob('xgbp*fits'): os.remove(f)
Example #2
0
           date=xkey[abs(xkey-obsdate).argmin()]
           xcoeff=xdict[date]
       except Exception,e : 
           msg='WARNING--Can not find xtalk coefficient for %s because %s' % (e, infile)
           if log: log.warning(msg)
           xcoeff=xdict[xdict.keys()[-1]]
   else:
       xcoeff=[]
   struct = xtalk(struct, xcoeff, log=log, verbose=verbose)

   #bias correct the files
   if saltkey.fastmode(saltkey.get('DETMODE', struct[0])): order=1

   #crclean the files
   if crtype is not None:
      struct=multicrclean(struct, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \
                          gain, rdnoise, bfactor, fthresh, gbox, maxiter, log=log, verbose=True)

   #mosaic correct the files


   return struct

def createmasterbiasname(infiles, biaskeys):
    """Create the name for the master bias file based on its parameters.  The format for 
       hte name is 
       [S/P][YYYYMMDD]Bias[MODE][BINNING][GAINSET][ROSPEED].fits
    
       where the following abbreviations are used:
      
       [S/P]--Scam or RSS
       [YYYYMMDD]--obsdate of the data or most common obsdate if multiple dates
Example #3
0
def quickclean(filename, interp='linear', cleanup=True, clobber=False, logfile='saltclean.log', verbose=True):
   """Start the process to reduce the data and produce a single mosaicked image"""
   print filename

   #create the input file name
   status=0
   infile=os.path.basename(filename)
   rawpath=os.path.dirname(filename)
   outpath='./'
   outfile=outpath+'mbxp'+infile
   print infile, rawpath, outpath

   #check to see if it exists and return if clobber is no
   if os.path.isfile(outfile) and not clobber: return

   #set up the files needed
   if infile[0]=='P':
     gaindb = iraf.osfn('pysalt$data/rss/RSSamps.dat')
     xtalkfile = iraf.osfn('pysalt$data/rss/RSSxtalk.dat')
     geomfile = iraf.osfn('pysalt$data/rss/RSSgeom.dat')
   elif infile[0]=='S':
     gaindb = iraf.osfn('pysalt$data/scam/SALTICAMamps.dat')
     xtalkfile = iraf.osfn('pysalt$data/scam/SALTICAMxtalk.dat')
     geomfile = iraf.osfn('pysalt$data/scam/SALTICAMgeom.dat')
 
   #verify the file
   struct=saltio.openfits(rawpath+'/'+infile)
   struct.verify('exception')
   
   #check to see if detmode is there
   if not saltkey.found('DETMODE', struct[0]): 
      return 
 
   #reduce the file
   struct=prepare(struct, createvar=False, badpixelstruct=None)
 
      #reset the names in the structures
   for i in range(1,len(struct)):
       struct[i].name=struct[i].header['EXTNAME']


   #gain correct the files
   usedb=True
   dblist= saltio.readgaindb(gaindb)
   log=open(logfile, 'a')
   ampccd = struct[0].header['NAMPS'] / struct[0].header['NCCDS']
   struct=gain(struct, mult=True,usedb=usedb, dblist=dblist, ampccd=ampccd, log=None, verbose=verbose)

   struct=bias(struct, subover=True,trim=True,subbias=False, 
                    median=False,function='polynomial',order=5,rej_lo=3,rej_hi=3,niter=10,
                    plotover=False,log=None, verbose=verbose)

   if struct[0].header['CCDTYPE']=='OBJECT' and struct[0].header['EXPTIME']>90:
      struct = multicrclean(struct, crtype='median', thresh=5, mbox=5, bbox=25, bthresh=5, flux_ratio=0.2, \
                          gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=5, log=None, verbose=verbose)

   pinfile=outpath+'bxp'+infile
   saltio.writefits(struct, pinfile, clobber)

   saltred.saltmosaic(images=pinfile,
                   outimages='',outpref=outpath+'m',geomfile=geomfile, fill=True,
                   interp=interp,cleanup=cleanup,clobber=clobber,logfile=logfile,
                   verbose=verbose)
   profile=outpath+'mbxp'+infile

   #remove intermediate steps
   if cleanup:
      if os.path.isfile(pinfile): os.remove(pinfile)

   return
Example #4
0
def quickclean(filename,
               interp='linear',
               cleanup=True,
               clobber=False,
               logfile='saltclean.log',
               verbose=True):
    """Start the process to reduce the data and produce a single mosaicked image"""
    print filename

    #create the input file name
    status = 0
    infile = os.path.basename(filename)
    rawpath = os.path.dirname(filename)
    outpath = './'
    outfile = outpath + 'mbxp' + infile
    print infile, rawpath, outpath

    #check to see if it exists and return if clobber is no
    if os.path.isfile(outfile) and not clobber: return

    #set up the files needed
    if infile[0] == 'P':
        gaindb = iraf.osfn('pysalt$data/rss/RSSamps.dat')
        xtalkfile = iraf.osfn('pysalt$data/rss/RSSxtalk.dat')
        geomfile = iraf.osfn('pysalt$data/rss/RSSgeom.dat')
    elif infile[0] == 'S':
        gaindb = iraf.osfn('pysalt$data/scam/SALTICAMamps.dat')
        xtalkfile = iraf.osfn('pysalt$data/scam/SALTICAMxtalk.dat')
        geomfile = iraf.osfn('pysalt$data/scam/SALTICAMgeom.dat')

    #verify the file
    struct = saltio.openfits(rawpath + '/' + infile)
    struct.verify('exception')

    #check to see if detmode is there
    if not saltkey.found('DETMODE', struct[0]):
        return

    #reduce the file
    struct = prepare(struct, createvar=False, badpixelstruct=None)

    #reset the names in the structures
    for i in range(1, len(struct)):
        struct[i].name = struct[i].header['EXTNAME']

    #gain correct the files
    usedb = True
    dblist = saltio.readgaindb(gaindb)
    log = open(logfile, 'a')
    ampccd = struct[0].header['NAMPS'] / struct[0].header['NCCDS']
    struct = gain(struct,
                  mult=True,
                  usedb=usedb,
                  dblist=dblist,
                  ampccd=ampccd,
                  log=None,
                  verbose=verbose)

    struct = bias(struct,
                  subover=True,
                  trim=True,
                  subbias=False,
                  median=False,
                  function='polynomial',
                  order=5,
                  rej_lo=3,
                  rej_hi=3,
                  niter=10,
                  plotover=False,
                  log=None,
                  verbose=verbose)

    if struct[0].header[
            'CCDTYPE'] == 'OBJECT' and struct[0].header['EXPTIME'] > 90:
        struct = multicrclean(struct, crtype='median', thresh=5, mbox=5, bbox=25, bthresh=5, flux_ratio=0.2, \
                            gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=5, log=None, verbose=verbose)

    pinfile = outpath + 'bxp' + infile
    saltio.writefits(struct, pinfile, clobber)

    saltred.saltmosaic(images=pinfile,
                       outimages='',
                       outpref=outpath + 'm',
                       geomfile=geomfile,
                       fill=True,
                       interp=interp,
                       cleanup=cleanup,
                       clobber=clobber,
                       logfile=logfile,
                       verbose=verbose)
    profile = outpath + 'mbxp' + infile

    #remove intermediate steps
    if cleanup:
        if os.path.isfile(pinfile): os.remove(pinfile)

    return