def saltsurface(images,outimages,outpref, mask=True, order=3, minlevel=0, clobber=False, logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') # open each raw image file for infile, outfile in zip(infiles,outfiles): struct = saltio.openfits(infile) struct = surface_fit(struct, order=order, mask=mask, minlevel=minlevel) #add any header keywords like history fname, hist=history(level=1, wrap=False) saltkey.housekeeping(struct[0],'SURFIT', 'File fit by a surface', hist) #write it out and close it saltio.writefits(struct,outfile,clobber=clobber) saltio.closefits(struct) #output the information log.message('Surface fitted image %s ' % (infile), with_header=False, with_stdout=verbose)
def saltcombine(images,outimage, method='average', reject=None, mask=True, \ weight=True, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile='salt.log',verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) #set reject as None reject = saltio.checkfornone(reject) if reject is not None: reject = reject.lower() #set scale scale = saltio.checkfornone(scale) if scale is not None: scale = scale.lower() #set statsec statsec = saltio.checkfornone(statsec) statsec = saltio.getSection(statsec, iraf_format=True) #Fast combine the images outstruct=imcombine(infiles, method=method, reject=reject, mask=mask, \ weight=weight, blank=blank, scale=scale, \ statsec=statsec, lthresh=lthresh, hthresh=hthresh) # housekeeping keywords fname, hist = history(level=1, wrap=False) saltkey.housekeeping(outstruct[0], 'SCOMBINE', 'File Combined by SALTCOMBINE', hist) # write FITS file saltio.writefits(outstruct, outimage)
def saltprepare(images,outimages,outpref,createvar=False, badpixelimage=None, clobber=True,logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') # open the badpixel image if saltio.checkfornone(badpixelimage) is None: badpixelstruct=None else: try: badpixelstruct = saltio.openfits(badpixelimage) except saltio.SaltIOError,e: msg='badpixel image must be specificied\n %s' % e raise SaltError(msg) # open each raw image file for img, oimg, in zip(infiles, outfiles): #open the fits file struct=saltio.openfits(img) #if createvar, throw a warning if it is using slotmode if saltkey.fastmode(saltkey.get('DETMODE', struct[0])) and createvar: msg='Creating variance frames for slotmode data in %s' % img log.warning(msg) # identify instrument instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct) # has file been prepared already? try: key = struct[0].header[keyprep] message = 'ERROR -- SALTPREPARE: File ' + infile message += ' has already been prepared' raise SaltError(message) except: pass # prepare file struct=prepare(struct,createvar=createvar, badpixelstruct=badpixelstruct) # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],keyprep, 'File prepared for IRAF', hist) # write FITS file saltio.writefits(struct,oimg, clobber=clobber) saltio.closefits(struct) message = 'SALTPREPARE -- %s => %s' % (img, oimg) log.message(message, with_header=False)
def saltcombine(images,outimage, method='average', reject=None, mask=True, \ weight=True, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) #set reject as None reject=saltio.checkfornone(reject) if reject is not None: reject=reject.lower() #set scale scale=saltio.checkfornone(scale) if scale is not None: scale=scale.lower() #set statsec statsec=saltio.checkfornone(statsec) statsec=saltio.getSection(statsec, iraf_format=True) #Fast combine the images outstruct=imcombine(infiles, method=method, reject=reject, mask=mask, \ weight=weight, blank=blank, scale=scale, \ statsec=statsec, lthresh=lthresh, hthresh=hthresh) # housekeeping keywords fname, hist=history(level=1, wrap=False) saltkey.housekeeping(outstruct[0],'SCOMBINE', 'File Combined by SALTCOMBINE', hist) # write FITS file saltio.writefits(outstruct, outimage)
def saltillum(images,outimages,outpref, mbox=11, clobber=False, \ logfile='salt.log',verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') #verify that the input and output lists are the same length saltio.comparelists(infiles, outfiles, 'Input', 'output') # open each raw image file for infile, outfile in zip(infiles, outfiles): struct = saltio.openfits(infile) struct = illum_cor(struct, mbox) #add any header keywords like history fname, hist = history(level=1, wrap=False) saltkey.housekeeping(struct[0], 'SILLUM', 'File Illumination corrected', hist) #write it out and close it saltio.writefits(struct, outfile, clobber=clobber) saltio.closefits(struct) #output the information log.message('Illumination corrected image %s ' % (infile), with_header=False, with_stdout=verbose)
def saltgain(images, outimages, outpref, gaindb=None, usedb=False, mult=True, clobber=True, logfile='salt.log', verbose=True): #start logging with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') #verify that the input and output lists are the same length saltio.comparelists(infiles, outfiles, 'Input', 'output') # read in the database file if usedb is true if usedb: gaindb = gaindb.strip() dblist = saltio.readgaindb(gaindb) else: dblist = [] for img, oimg in zip(infiles, outfiles): #open the fits file struct = saltio.openfits(img) # identify instrument instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid( struct) # has file been prepared already? if saltkey.found(keygain, struct[0]): message = 'SALTGAIN: %s has already been gain-corrected' % img raise SaltError(message) # gain correct the data struct = gain(struct, mult=mult, usedb=usedb, dblist=dblist, log=log, verbose=verbose) # housekeeping keywords fname, hist = history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], keygain, 'Images have been gain corrected', hist) # write FITS file saltio.writefits(struct, oimg, clobber=clobber) saltio.closefits(struct)
def saltillum(images, outimages, outpref, mbox=11, clobber=False, logfile="salt.log", verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack("Input", images) # create list of output files outfiles = saltio.listparse("Outfile", outimages, outpref, infiles, "") # verify that the input and output lists are the same length saltio.comparelists(infiles, outfiles, "Input", "output") # open each raw image file for infile, outfile in zip(infiles, outfiles): struct = saltio.openfits(infile) struct = illum_cor(struct, mbox) # add any header keywords like history fname, hist = history(level=1, wrap=False) saltkey.housekeeping(struct[0], "SILLUM", "File Illumination corrected", hist) # write it out and close it saltio.writefits(struct, outfile, clobber=clobber) saltio.closefits(struct) # output the information log.message("Illumination corrected image %s " % (infile), with_header=False, with_stdout=verbose)
def saltcrclean(images,outimages,outpref,crtype='fast',thresh=5,mbox=3, \ bthresh=3, flux_ratio=0.2, bbox=11, gain=1, rdnoise=5, fthresh=5,\ bfactor=2, gbox=3, maxiter=5, multithread=False, update=True, clobber=True, logfile='salt.log', verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') #check to see if multithreading is available if mp: pass else: multithread=False log.warning('multiprocessing module is not available. Setting multiththread=False') # Begin processes each file for infile,outfile in zip(infiles,outfiles): #open the infile struct=saltio.openfits(infile) #clean the cosmic rays if multithread and len(struct)>1: struct=multicrclean(struct, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \ gain, rdnoise, bfactor, fthresh, gbox, maxiter, log, verbose=verbose) else: struct=crclean(struct, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \ gain, rdnoise, bfactor, fthresh, gbox, maxiter, update, log, verbose=verbose) #log the call #log.message('Cleaned %i cosmic rays from %s using %s method' % (totcr, infile, crtype), with_header=False) log.message('', with_header=False, with_stdout=verbose) #add house keeping keywords saltkey.put('SAL-TLM',time.asctime(time.localtime()), struct[0]) #add the history keyword fname, hist=history(level=1, wrap=False) saltkey.history(struct[0],hist) #write out the file saltio.writefits(struct, outfile, clobber=clobber) #close the image saltio.closefits(struct)
def saltxtalk(images,outimages,outpref,xtalkfile=None, usedb=False, clobber=True, logfile='salt.log',verbose=True): #start logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') # are input and output lists the same length? saltio.comparelists(infiles,outfiles,'Input','output') # does crosstalk coefficient data exist if usedb: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict=None for img, oimg in zip(infiles, outfiles): #open the fits file struct=saltio.openfits(img) #find the best xcoeff for the image if using the db if usedb: obsdate=saltkey.get('DATE-OBS', struct[0]) obsdate=int('%s%s%s' % (obsdate[0:4],obsdate[5:7], obsdate[8:])) xkey=np.array(xdict.keys()) date=xkey[abs(xkey-obsdate).argmin()] xcoeff=xdict[date] else: xcoeff=[] # identify instrument instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct) # has file been prepared already? if saltkey.found(keyxtalk, struct[0]): message='%s has already been xtalk corrected' % img raise SaltError(message) #apply the cross-talk correction struct = xtalk(struct, xcoeff, log=log, verbose=verbose) # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SXTALK', 'Images have been xtalk corrected', hist) # write FITS file saltio.writefits(struct,oimg, clobber=clobber) saltio.closefits(struct)
def saltcrclean(images,outimages,outpref,crtype='fast',thresh=5,mbox=3, \ bthresh=3, flux_ratio=0.2, bbox=11, gain=1, rdnoise=5, fthresh=5,\ bfactor=2, gbox=3, maxiter=5, multithread=False, update=True, clobber=True, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') #verify that the input and output lists are the same length saltio.comparelists(infiles, outfiles, 'Input', 'output') #check to see if multithreading is available if mp: pass else: multithread = False log.warning( 'multiprocessing module is not available. Setting multiththread=False' ) # Begin processes each file for infile, outfile in zip(infiles, outfiles): #open the infile struct = saltio.openfits(infile) #clean the cosmic rays if multithread and len(struct) > 1: struct=multicrclean(struct, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \ gain, rdnoise, bfactor, fthresh, gbox, maxiter, log, verbose=verbose) else: struct=crclean(struct, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \ gain, rdnoise, bfactor, fthresh, gbox, maxiter, update, log, verbose=verbose) #log the call #log.message('Cleaned %i cosmic rays from %s using %s method' % (totcr, infile, crtype), with_header=False) log.message('', with_header=False, with_stdout=verbose) #add house keeping keywords saltkey.put('SAL-TLM', time.asctime(time.localtime()), struct[0]) #add the history keyword fname, hist = history(level=1, wrap=False) saltkey.history(struct[0], hist) #write out the file saltio.writefits(struct, outfile, clobber=clobber) #close the image saltio.closefits(struct)
def saltslot(images, outimages, outpref, gaindb='', xtalkfile='', usedb=False, clobber=False, logfile='salt.log', verbose=True): #start logging with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') # are input and output lists the same length? saltio.comparelists(infiles, outfiles, 'Input', 'output') # does crosstalk coefficient data exist if usedb: dblist = saltio.readgaindb(gaindb) xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: dblist = [] xdict = None for img, oimg in zip(infiles, outfiles): #open the fits file struct = saltio.openfits(img) # identify instrument instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid( struct) # has file been prepared already? if saltkey.found(keygain, struct[0]): message = '%s has already been reduced' % img raise SaltError(message) # housekeeping keywords fname, hist = history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], keyslot, 'Images have been slotmode reduced', hist) # write FITS file saltio.writefits(struct, oimg, clobber=clobber) saltio.closefits(struct)
def saltarith(operand1, op, operand2, result, outpref, divzero=0, clobber=False, \ logfile='salt.log',verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', operand1) # create list of output files outfiles = saltio.listparse('Outfile', result, outpref, infiles, '') #verify that the input and output lists are the same length saltio.comparelists(infiles, outfiles, 'Input', 'output') #let's keep track of whether operand2 is an image or not is_image = False #load in operand2, or, if it's not an image, assume it's a number try: operand2struct = float(operand2) except ValueError: operand2struct = saltio.openfits(operand2) is_image = True #open the input image files for infile, outfile in zip(infiles, outfiles): struct = saltio.openfits(infile) #do some math! outstruct = arith(struct, op, operand2struct, is_image, divzero) try: pass except Exception, e: msg = 'Unable to do math %s because %s' % (infile, e) raise SaltError(msg) #update header stuff fname, hist = history(level=1, wrap=False) saltkey.housekeeping(struct[0], 'SARITH', 'Some arithmatic was performed', hist) #write it. close it. saltio.writefits(outstruct, outfile, clobber=clobber) saltio.closefits(struct) #output the information log.message('imarith: %s %s %s %s' % (infile, op, operand2, outfile), with_header=False, with_stdout=verbose) #close the operand2 image if is_image: saltio.closefits(operand2struct)
def slotreadtimefix(images, outimages, outpref, clobber=False, logfile='slot.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') #verify that the input and output lists are the same length saltio.comparelists(infiles, outfiles, 'Input', 'output') for img, oimg in zip(infiles, outfiles): #check to see if the out image already exists if not clobber and os.path.isfile(oimg): raise SaltIOError('%s alraedy exists' % oimg) #open the file struct = saltio.openfits(img) #log the message log.message('Updateing times in %s' % img, with_header=False, with_stdout=verbose) #now for each science frame, corrent the readtime #Assumes SALT format and that the first extension #is empty for i in range(1, len(struct)): try: struct[i] = readtimefix(struct[i]) except SaltIOError, e: raise SaltError('%s %s' % (img, e)) #Add history keywords # housekeeping keywords fname, hist = history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], "SLOTREAD", 'READTIME added', hist) #write to the output saltio.writefits(struct, oimg, clobber) return
def saltarith(operand1, op, operand2, result, outpref, divzero=0, clobber=False, \ logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',operand1) # create list of output files outfiles=saltio.listparse('Outfile', result, outpref, infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') #let's keep track of whether operand2 is an image or not is_image = False #load in operand2, or, if it's not an image, assume it's a number try: operand2struct = float(operand2) except ValueError: operand2struct = saltio.openfits(operand2) is_image = True #open the input image files for infile, outfile in zip(infiles,outfiles): struct = saltio.openfits(infile) #do some math! outstruct = arith(struct, op, operand2struct, is_image, divzero) try: pass except Exception as e: msg='Unable to do math %s because %s' % (infile, e) raise SaltError(msg) #update header stuff fname, hist = history(level=1, wrap=False) saltkey.housekeeping(struct[0],'SARITH', 'Some arithmatic was performed',hist) #write it. close it. saltio.writefits(outstruct,outfile,clobber=clobber) saltio.closefits(struct) #output the information log.message('imarith: %s %s %s %s' % (infile, op, operand2, outfile), with_header=False, with_stdout=verbose) #close the operand2 image if is_image: saltio.closefits(operand2struct)
def saltmosaic(images,outimages,outpref,geomfile,interp='linear',geotran=True, cleanup=True,clobber=False,logfile=None,verbose=True): #Start the logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') # does CCD geometry definition file exist geomfilefile = geomfile.strip() saltio.fileexists(geomfile) gap = 0 xshift = [0, 0] yshift = [0, 0] rotation = [0, 0] gap, xshift, yshift, rotation=saltio.readccdgeom(geomfile) # open each raw image file and apply the transformation to it for img, oimg in zip(infiles, outfiles): #open the structure struct = saltio.openfits(img) #create the mosaic ostruct=make_mosaic(struct, gap, xshift, yshift, rotation, interp_type=interp, geotran=geotran, cleanup=cleanup, log=log, verbose=verbose) #update the header information # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(ostruct[0], 'SMOSAIC', 'Images have been mosaicked ', hist) #write the image out saltio.writefits(ostruct,oimg, clobber=clobber) #close the files saltio.closefits(struct) saltio.closefits(ostruct)
def saltgain(images,outimages, outpref, gaindb=None,usedb=False, mult=True, clobber=True, logfile='salt.log',verbose=True): #start logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') # read in the database file if usedb is true if usedb: gaindb = gaindb.strip() dblist= saltio.readgaindb(gaindb) else: dblist=[] for img, oimg in zip(infiles, outfiles): #open the fits file struct=saltio.openfits(img) # identify instrument instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct) # has file been prepared already? if saltkey.found(keygain, struct[0]): message='SALTGAIN: %s has already been gain-corrected' % img raise SaltError(message) # gain correct the data struct = gain(struct,mult=mult, usedb=usedb, dblist=dblist, log=log, verbose=verbose) # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],keygain, 'Images have been gain corrected', hist) # write FITS file saltio.writefits(struct,oimg, clobber=clobber) saltio.closefits(struct)
def saltslot(images,outimages,outpref,gaindb='',xtalkfile='',usedb=False, clobber=False,logfile='salt.log',verbose=True): #start logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') # are input and output lists the same length? saltio.comparelists(infiles,outfiles,'Input','output') # does crosstalk coefficient data exist if usedb: dblist= saltio.readgaindb(gaindb) xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: dblist=[] xdict=None for img, oimg in zip(infiles, outfiles): #open the fits file struct=saltio.openfits(img) # identify instrument instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct) # has file been prepared already? if saltkey.found(keygain, struct[0]): message='%s has already been reduced' % img raise SaltError(message) # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],keyslot, 'Images have been slotmode reduced', hist) # write FITS file saltio.writefits(struct,oimg, clobber=clobber) saltio.closefits(struct)
def slotreadtimefix(images,outimages, outpref, clobber=False, logfile='slot.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') for img, oimg in zip(infiles, outfiles): #check to see if the out image already exists if not clobber and os.path.isfile(oimg): raise SaltIOError('%s alraedy exists' % oimg) #open the file struct=saltio.openfits(img) #log the message log.message('Updateing times in %s' % img, with_header=False, with_stdout=verbose) #now for each science frame, corrent the readtime #Assumes SALT format and that the first extension #is empty for i in range(1,len(struct)): try: struct[i]=readtimefix(struct[i]) except SaltIOError,e : raise SaltError('%s %s' % (img,e)) #Add history keywords # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],"SLOTREAD", 'READTIME added', hist) #write to the output saltio.writefits(struct, oimg, clobber) return
def saltsurface(images, outimages, outpref, mask=True, order=3, minlevel=0, clobber=False, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') #verify that the input and output lists are the same length saltio.comparelists(infiles, outfiles, 'Input', 'output') # open each raw image file for infile, outfile in zip(infiles, outfiles): struct = saltio.openfits(infile) struct = surface_fit(struct, order=order, mask=mask, minlevel=minlevel) #add any header keywords like history fname, hist = history(level=1, wrap=False) saltkey.housekeeping(struct[0], 'SURFIT', 'File fit by a surface', hist) #write it out and close it saltio.writefits(struct, outfile, clobber=clobber) saltio.closefits(struct) #output the information log.message('Surface fitted image %s ' % (infile), with_header=False, with_stdout=verbose)
def saltflat(images,outimages,outpref, flatimage,minflat=1, allext=False, clobber=False, \ logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') # check flatfield image exists flaimage= flatimage.strip() saltio.fileexists(flatimage) flatstruct= saltio.openfits(flatimage) # Normalize the flat field image # This requires to go through each science extension and divide it by # mean of the image. Things that have to be checked: # that data exist, that it is a science extension #determine the global mean fmean=0 fcount=0 #replace bad pixels for i in range(len(flatstruct)): if flatstruct[i].data is not None and (flatstruct[i].name=='SCI' or flatstruct[i].name=='PRIMARY'): data = flatstruct[i].data mask = (data > minflat) if (numpy.nan==flatstruct[i].data).sum() or (numpy.inf==flatstruct[i].data).sum(): message = '\nWARNING -- SALTFLAT: %s contains invalid values' % flatimage log.warning(message,with_stdout=verbose) flatstruct[i].data[mask==0] = minflat flatstruct[i].data[flatstruct[i].data==numpy.inf] = minflat #determine the mean mask = (data > minflat) fmean += data[mask].sum() fcount += data[mask].size if fcount>0: fmean=fmean/fcount for i in range(len(flatstruct)): if flatstruct[i].name=='PRIMARY': #is it a flat--if not throw a warning try: key_ccdtype=saltkey.get('CCDTYPE', flatstruct[i]) except: key_ccdtype=None if key_ccdtype!='FLAT': message = '\nWARNING -- SALTFLAT: %s does not have CCDTYPE=FLAT' % flatimage log.warning(message,with_stdout=verbose) #if there are data, normalize it if flatstruct[i].data is not None: flatstruct[i].data=flatnormalize(flatstruct[i].data, minflat) #Noramlize the science extensions if flatstruct[i].name=='SCI': if flatstruct[i].data is not None: if allext is False: fmean=flatstruct[i].data.mean() flatstruct[i].data=flatnormalize(flatstruct[i].data, minflat, fmean) #Apply to the variance frames if saltkey.found('VAREXT', flatstruct[i]): varext=saltkey.get('VAREXT',flatstruct[i]) flatstruct[varext].data=flatstruct[varext].data/fmean**2 # open each raw image file for infile, outfile in zip(infiles,outfiles): struct = saltio.openfits(infile) # flat field correct the image outstruct = flat(struct, flatstruct) try: pass except Exception,e: msg='Unable to flatten %s because %s' % (infile, e) raise SaltError(msg) #add any header keywords like history fname, hist=history(level=1, wrap=False) saltkey.housekeeping(struct[0],'SFLAT', 'File flatfield corrected', hist) #write it out and close it saltio.writefits(outstruct,outfile,clobber=clobber) saltio.closefits(struct) #output the information log.message('Flatfields image %s using %s' % (infile, flatimage), with_header=False, with_stdout=verbose) #clost the flatfield image saltio.closefits(flatstruct)
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None, geomfile=None,subover=True,trim=True,masbias=None, subbias=False, median=False, function='polynomial', order=5,rej_lo=3, rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='', clobber=False, cleanup=True, logfile='salt.log', verbose=True): """SALTADVANCE provides advanced data reductions for a set of data. It will sort the data, and first process the biases, flats, and then the science frames. It will record basic quality control information about each of the steps. """ plotover=False #start logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) infiles.sort() # create list of output files outpath=saltio.abspath(outpath) #log into the database sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) #does the gain database file exist if gaindb: dblist= saltio.readgaindb(gaindb) else: dblist=[] # does crosstalk coefficient data exist if xtalkfile: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict=None #does the mosaic file exist--raise error if no saltio.fileexists(geomfile) # Delete the obslog file if it already exists if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile) #read in the obsveration log or create it if os.path.isfile(obslogfile): msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes' raise SaltError(msg) else: headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') obsmode=obsstruct.data.field('OBSMODE') ccdtype=obsstruct.data.field('CCDTYPE') propcode=obsstruct.data.field('PROPID') masktype=obsstruct.data.field('MASKTYP') #set the bias list of objects biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')] masterbias_dict={} for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=fits.open(img) bimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #create the list of flatfields and process them flatlist=filename[ccdtype=='FLAT'] masterflat_dict={} for img in infiles: if os.path.basename(img) in flatlist: #open the image struct=fits.open(img) fimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Flat frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,fimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list) #create the master flat frame for i in masterflat_dict.keys(): fkeys=masterflat_dict[i][0] flist=masterflat_dict[i][1:] mflatname=outpath+createmasterflatname(flist, fkeys) ffiles=','.join(flist) saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #process the arc data arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')] for i, img in enumerate(infiles): nimg=os.path.basename(img) if nimg in arclist: #open the image struct=fits.open(img) simg=outpath+'bxgp'+os.path.basename(img) obsdate=os.path.basename(img)[1:9] #print the message if log: message='Processing ARC frame %s' % img log.message(message, with_stdout=verbose) struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the images mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg) #measure the arcdata arcimage=outpath+'mbxgp'+nimg dbfile=outpath+obsdate+'_specid.db' lamp = obsstruct.data.field('LAMPID')[i] lamp = lamp.replace(' ', '') lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) print arcimage, lampfile, os.getcwd() specidentify(arcimage, lampfile, dbfile, guesstype='rss', guessfile='', automethod='Matchlines', function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile, verbose=verbose) try: ximg = outpath+'xmbxgp'+os.path.basename(arcimage) specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, conserve=True, nearest=True, clobber=True, logfile=logfile, verbose=verbose) except: pass #process the science data for i, img in enumerate(infiles): nimg=os.path.basename(img) if not (nimg in flatlist or nimg in biaslist or nimg in arclist): #open the image struct=fits.open(img) if struct[0].header['PROPID'].count('CAL_GAIN'): continue simg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #Check to see if it is RSS 2x2 and add bias subtraction instrume=saltkey.get('INSTRUME', struct[0]).strip() gainset = saltkey.get('GAINSET', struct[0]) rospeed = saltkey.get('ROSPEED', struct[0]) target = saltkey.get('OBJECT', struct[0]).strip() exptime = saltkey.get('EXPTIME', struct[0]) obsmode = saltkey.get('OBSMODE', struct[0]).strip() detmode = saltkey.get('DETMODE', struct[0]).strip() masktype = saltkey.get('MASKTYP', struct[0]).strip() xbin, ybin = saltkey.ccdbin( struct[0], img) obsdate=os.path.basename(img)[1:9] bstruct=None crtype=None thresh=5 mbox=11 bthresh=5.0, flux_ratio=0.2 bbox=25 gain=1.0 rdnoise=5.0 fthresh=5.0 bfactor=2 gbox=3 maxiter=5 subbias=False if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW': bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin) if os.path.exists(bfile): bstruct=fits.open(bfile) subbias=True if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5: crtype='edge' thresh=5 mbox=11 bthresh=5.0, flux_ratio=0.2 bbox=25 gain=1.0 rdnoise=5.0 fthresh=5.0 bfactor=2 gbox=3 maxiter=3 #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \ bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise, bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the files--currently not in the proper format--will update when it is if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])): mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg) #if the file is spectroscopic mode, apply the wavelength correction if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT': dbfile=outpath+obsdate+'_specid.db' try: ximg = outpath+'xmbxgp'+os.path.basename(img) specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, conserve=True, nearest=True, clobber=True, logfile=logfile, verbose=verbose) except Exception, e: log.message('%s' % e) #clean up the results if cleanup: #clean up the bias frames for i in masterbias_dict.keys(): blist=masterbias_dict[i][1:] for b in blist: saltio.delete(b) #clean up the flat frames for i in masterflat_dict.keys(): flist=masterflat_dict[i][1:] for f in flist: saltio.delete(f)
def saltmosaic(images, outimages, outpref, geomfile, interp='linear', geotran=True, fill=False, cleanup=True, clobber=False, logfile=None, verbose=True): # Start the logging with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') # verify that the input and output lists are the same length saltio.comparelists(infiles, outfiles, 'Input', 'output') # does CCD geometry definition file exist geomfilefile = geomfile.strip() saltio.fileexists(geomfile) gap = 0 xshift = [0, 0] yshift = [0, 0] rotation = [0, 0] gap, xshift, yshift, rotation = saltio.readccdgeom(geomfile) # open each raw image file and apply the transformation to it for img, oimg in zip(infiles, outfiles): # open the structure struct = saltio.openfits(img) # create the mosaic ostruct = make_mosaic(struct, gap, xshift, yshift, rotation, interp_type=interp, geotran=geotran, fill=fill, cleanup=cleanup, log=log, verbose=verbose) # update the header information # housekeeping keywords fname, hist = history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(ostruct[0], 'SMOSAIC', 'Images have been mosaicked', hist) # write the image out ostruct.writeto(oimg, clobber=clobber, output_verify='ignore') # close the files struct.close() ostruct.close()
def saltbias(images, outimages, outpref, subover=True, trim=True, subbias=False, masterbias='bias.fits', median=False, function='polynomial', order=3, rej_lo=3, rej_hi=3, niter=10, plotover=False, turbo=False, clobber=False, logfile='salt.log', verbose=True): status = 0 ifil = 0 ii = 0 mbiasdata = [] bstruct = '' biasgn = '' biassp = '' biasbn = '' biasin = '' filetime = {} biastime = {} for i in range(1, 7): filetime[i] = [] biastime[i] = [] with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') # are input and output lists the same length? saltio.comparelists(infiles, outfiles, 'Input', 'output') # Does master bias frame exist? # gain, speed, binning and instrument of master bias frame if subbias: if os.path.isfile(masterbias): bstruct = saltio.openfits(masterbias) else: message = 'Master bias frame %s does not exist' % masterbias raise SaltError(message) else: bstruct = None # open each raw image file for img, oimg in zip(infiles, outfiles): #open the file struct = saltio.openfits(img) #check to see if it has already been bias subtracted instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid( struct) # has file been biaseded already? try: key = struct[0].header[keybias] message = 'File %s has already been de-biased ' % infile raise SaltError(message) except: pass #compare with the master bias to make sure they are the same if subbias: pass #subtract the bias struct = bias(struct, subover=subover, trim=trim, subbias=subbias, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], keybias, 'Images have been de-biased', hist) # write FITS file saltio.writefits(struct, oimg, clobber=clobber) saltio.closefits(struct)
def slotmerge(images, outimages, outpref, geomfile, clobber, logfile, verbose): with logging(logfile, debug) as log: # are the arguments defined saltsafeio.argdefined('images', images) saltsafeio.argdefined('geomfile', geomfile) saltsafeio.argdefined('logfile', logfile) # if the input file is a list, does it exist? if images[0] == '@': saltsafeio.listexists('Input', images) # parse list of input files infiles = saltsafeio.listparse('Raw image', images, '', '', '') # check input files exist saltsafeio.filesexist(infiles, '', 'r') # load output name list: @list, * and comma separated outimages = outimages.strip() outpref = outpref.strip() if len(outpref) == 0 and len(outimages) == 0: raise SaltIOError('Output file(s) not specified') # test output @filelist exists if len(outimages) > 0 and outimages[0] == '@': saltsafeio.listexists('Output', outimages) # parse list of output files outfiles = saltsafeio.listparse('Output image', outimages, outpref, infiles, '') # are input and output lists the same length? saltsafeio.comparelists(infiles, outfiles, 'Input', 'output') # do the output files already exist? if not clobber: saltsafeio.filesexist(outfiles, '', 'w') # does CCD geometry definition file exist geomfilefile = geomfile.strip() saltsafeio.fileexists(geomfile) # read geometry definition file gap = 0 xshift = [0, 0] yshift = [0, 0] rotation = [0, 0] gap, xshift, yshift, rotation = saltsafeio.readccdgeom(geomfile) for ro in rotation: if ro != 0: log.warning('SLOTMERGE currently ignores CCD rotation') # Begin processes each file for infile, outfile in zip(infiles, outfiles): # determine the name for the output file outpath = outfile.rstrip(os.path.basename(outfile)) if (len(outpath) == 0): outpath = '.' # open each raw image struct = saltsafeio.openfits(infile) # identify instrument instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltsafekey.instrumid( struct, infile) # how many amplifiers? nccds = saltsafekey.get('NCCDS', struct[0], infile) amplifiers = nccds * 2 #if (nccds != 2): # raise SaltError('Can not currently handle more than two CCDs') # CCD geometry coefficients if instrume == 'RSS' or instrume == 'PFIS': xsh = [xshift[0], 0., xshift[1]] ysh = [yshift[0], 0., yshift[1]] rot = [rotation[0], 0., rotation[1]] refid = 1 if instrume == 'SALTICAM': xsh = [xshift[0], 0.] ysh = [yshift[0], 0.] rot = [rotation[0], 0] refid = 1 # how many extensions? nextend = saltsafekey.get('NEXTEND', struct[0], infile) # how many exposures exposures = nextend / amplifiers # CCD on-chip binning xbin, ybin = saltsafekey.ccdbin(struct[0], infile) gp = int(gap / xbin) # create output hdu structure outstruct = [None] * int(exposures + 1) outstruct[0] = struct[0] # iterate over exposures, stitch them to produce file of CCD images for i in range(exposures): # Determine the total size of the image xsize = 0 ysize = 0 for j in range(amplifiers): hdu = i * amplifiers + j + 1 try: xsize += len(struct[hdu].data[0]) if ysize < len(struct[hdu].data): ysize = len(struct[hdu].data) except: msg = 'Unable to access extension %i ' % hdu raise SaltIOError(msg) xsize += gp * (nccds - 1) maxxsh, minxsh = determineshifts(xsh) maxysh, minysh = determineshifts(ysh) xsize += (maxxsh - minxsh) ysize += (maxysh - minysh) # Determine the x and y origins for each frame xdist = 0 ydist = 0 shid = 0 x0 = np.zeros(amplifiers) y0 = np.zeros(amplifiers) for j in range(amplifiers): x0[j] = xdist + xsh[shid] - minxsh y0[j] = ysh[shid] - minysh hdu = i * amplifiers + j + 1 darr = struct[hdu].data xdist += len(darr[0]) if j % 2 == 1: xdist += gp shid += 1 # make the out image outarr = np.zeros((ysize, xsize), np.float64) # Embed each frame into the output array for j in range(amplifiers): hdu = i * amplifiers + j + 1 darr = struct[hdu].data outarr = salttran.embed(darr, x0[j], y0[j], outarr) # Add the outimage to the output structure hdu = i * amplifiers + 1 outhdu = i + 1 outstruct[outhdu] = pyfits.ImageHDU(outarr) outstruct[outhdu].header = struct[hdu].header # Fix the headers in each extension datasec = '[1:%4i,1:%4i]' % (xsize, ysize) saltsafekey.put('DATASEC', datasec, outstruct[outhdu], outfile) saltsafekey.rem('DETSIZE', outstruct[outhdu], outfile) saltsafekey.rem('DETSEC', outstruct[outhdu], outfile) saltsafekey.rem('CCDSEC', outstruct[outhdu], outfile) saltsafekey.rem('AMPSEC', outstruct[outhdu], outfile) # add housekeeping key words outstruct[outhdu] = addhousekeeping(outstruct[outhdu], outhdu, outfile) # close input FITS file saltsafeio.closefits(struct) # housekeeping keywords keymosaic = 'SLOTMERG' fname, hist = history(level=1, wrap=False) saltsafekey.housekeeping(struct[0], keymosaic, 'Amplifiers have been mosaiced', hist) #saltsafekey.history(outstruct[0],hist) # this is added for later use by saltsafekey.put('NCCDS', 0.5, outstruct[0]) saltsafekey.put('NSCIEXT', exposures, outstruct[0]) saltsafekey.put('NEXTEND', exposures, outstruct[0]) # write FITS file of mosaiced image outstruct = pyfits.HDUList(outstruct) saltsafeio.writefits(outstruct, outfile, clobber=clobber)
def slotmerge(images,outimages,outpref,geomfile,clobber,logfile,verbose): with logging(logfile,debug) as log: # are the arguments defined saltsafeio.argdefined('images',images) saltsafeio.argdefined('geomfile',geomfile) saltsafeio.argdefined('logfile',logfile) # if the input file is a list, does it exist? if images[0] == '@': saltsafeio.listexists('Input',images) # parse list of input files infiles=saltsafeio.listparse('Raw image',images,'','','') # check input files exist saltsafeio.filesexist(infiles,'','r') # load output name list: @list, * and comma separated outimages = outimages.strip() outpref = outpref.strip() if len(outpref) == 0 and len(outimages) == 0: raise SaltIOError('Output file(s) not specified') # test output @filelist exists if len(outimages) > 0 and outimages[0] == '@': saltsafeio.listexists('Output',outimages) # parse list of output files outfiles=saltsafeio.listparse('Output image',outimages,outpref,infiles,'') # are input and output lists the same length? saltsafeio.comparelists(infiles,outfiles,'Input','output') # do the output files already exist? if not clobber: saltsafeio.filesexist(outfiles,'','w') # does CCD geometry definition file exist geomfilefile = geomfile.strip() saltsafeio.fileexists(geomfile) # read geometry definition file gap = 0 xshift = [0, 0] yshift = [0, 0] rotation = [0, 0] gap, xshift, yshift, rotation=saltsafeio.readccdgeom(geomfile) for ro in rotation: if ro!=0: log.warning('SLOTMERGE currently ignores CCD rotation') # Begin processes each file for infile, outfile in zip(infiles, outfiles): # determine the name for the output file outpath = outfile.rstrip(os.path.basename(outfile)) if (len(outpath) == 0): outpath = '.' # open each raw image struct=saltsafeio.openfits(infile) # identify instrument instrume,keyprep,keygain,keybias,keyxtalk,keyslot=saltsafekey.instrumid(struct,infile) # how many amplifiers? nccds=saltsafekey.get('NCCDS',struct[0],infile) amplifiers = nccds * 2 #if (nccds != 2): # raise SaltError('Can not currently handle more than two CCDs') # CCD geometry coefficients if instrume == 'RSS' or instrume == 'PFIS': xsh = [xshift[0], 0., xshift[1]] ysh = [yshift[0], 0., yshift[1]] rot = [rotation[0], 0., rotation[1]] refid = 1 if instrume == 'SALTICAM': xsh = [xshift[0], 0.] ysh = [yshift[0], 0.] rot = [rotation[0], 0] refid = 1 # how many extensions? nextend=saltsafekey.get('NEXTEND',struct[0],infile) # how many exposures exposures = nextend/amplifiers # CCD on-chip binning xbin, ybin=saltsafekey.ccdbin(struct[0],infile) gp = int(gap / xbin) # create output hdu structure outstruct = [None] * int(exposures+1) outstruct[0]=struct[0] # iterate over exposures, stitch them to produce file of CCD images for i in range(exposures): # Determine the total size of the image xsize=0 ysize=0 for j in range(amplifiers): hdu=i*amplifiers+j+1 try: xsize += len(struct[hdu].data[0]) if ysize < len(struct[hdu].data): ysize=len(struct[hdu].data) except: msg='Unable to access extension %i ' % hdu raise SaltIOError(msg) xsize += gp* (nccds-1) maxxsh, minxsh = determineshifts(xsh) maxysh, minysh = determineshifts(ysh) xsize += (maxxsh-minxsh) ysize += (maxysh-minysh) # Determine the x and y origins for each frame xdist=0 ydist=0 shid=0 x0=np.zeros(amplifiers) y0=np.zeros(amplifiers) for j in range(amplifiers): x0[j]=xdist+xsh[shid]-minxsh y0[j]=ysh[shid]-minysh hdu=i*amplifiers+j+1 darr=struct[hdu].data xdist += len(darr[0]) if j%2==1: xdist += gp shid += 1 # make the out image outarr=np.zeros((ysize, xsize), np.float64) # Embed each frame into the output array for j in range(amplifiers): hdu=i*amplifiers+j+1 darr=struct[hdu].data outarr=salttran.embed(darr, x0[j], y0[j], outarr) # Add the outimage to the output structure hdu=i*amplifiers+1 outhdu=i+1 outstruct[outhdu] = pyfits.ImageHDU(outarr) outstruct[outhdu].header=struct[hdu].header # Fix the headers in each extension datasec='[1:%4i,1:%4i]' % (xsize, ysize) saltsafekey.put('DATASEC',datasec, outstruct[outhdu], outfile) saltsafekey.rem('DETSIZE',outstruct[outhdu],outfile) saltsafekey.rem('DETSEC',outstruct[outhdu],outfile) saltsafekey.rem('CCDSEC',outstruct[outhdu],outfile) saltsafekey.rem('AMPSEC',outstruct[outhdu],outfile) # add housekeeping key words outstruct[outhdu]=addhousekeeping(outstruct[outhdu], outhdu, outfile) # close input FITS file saltsafeio.closefits(struct) # housekeeping keywords keymosaic='SLOTMERG' fname, hist=history(level=1, wrap=False) saltsafekey.housekeeping(struct[0],keymosaic,'Amplifiers have been mosaiced', hist) #saltsafekey.history(outstruct[0],hist) # this is added for later use by saltsafekey.put('NCCDS', 0.5, outstruct[0]) saltsafekey.put('NSCIEXT', exposures, outstruct[0]) saltsafekey.put('NEXTEND', exposures, outstruct[0]) # write FITS file of mosaiced image outstruct=pyfits.HDUList(outstruct) saltsafeio.writefits(outstruct, outfile, clobber=clobber)
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None, geomfile=None,subover=True,trim=True,masbias=None, subbias=False, median=False, function='polynomial', order=5,rej_lo=3, rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log', verbose=True): """SALTCLEAN will provide basic CCD reductions for a set of data. It will sort the data, and first process the biases, flats, and then the science frames. It will record basic quality control information about each of the steps. """ plotover=False #start logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outpath=saltio.abspath(outpath) #does the gain database file exist if gaindb: dblist= saltio.readgaindb(gaindb) else: dblist=[] # does crosstalk coefficient data exist if xtalkfile: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict=None #does the mosaic file exist--raise error if no saltio.fileexists(geomfile) # Delete the obslog file if it already exists if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile) #read in the obsveration log or create it if os.path.isfile(obslogfile): msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes' raise SaltError(msg) else: headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') ccdtype=obsstruct.data.field('CCDTYPE') #set the bias list of objects biaslist=filename[ccdtype=='ZERO'] masterbias_dict={} for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=pyfits.open(img) bimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #create the list of flatfields and process them flatlist=filename[ccdtype=='FLAT'] masterflat_dict={} for img in infiles: if os.path.basename(img) in flatlist: #open the image struct=pyfits.open(img) fimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Flat frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,fimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list) #create the master flat frame for i in masterflat_dict.keys(): fkeys=masterflat_dict[i][0] flist=masterflat_dict[i][1:] mflatname=outpath+createmasterflatname(flist, fkeys) ffiles=','.join(flist) saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #process the science data for img in infiles: nimg=os.path.basename(img) if not nimg in flatlist or not nimg in biaslist: #open the image struct=pyfits.open(img) simg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the files--currently not in the proper format--will update when it is if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])): mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg)
def saltclean(images, outpath, obslogfile=None, gaindb=None, xtalkfile=None, geomfile=None, subover=True, trim=True, masbias=None, subbias=False, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log', verbose=True): """SALTCLEAN will provide basic CCD reductions for a set of data. It will sort the data, and first process the biases, flats, and then the science frames. It will record basic quality control information about each of the steps. """ plotover = False #start logging with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outpath = saltio.abspath(outpath) #does the gain database file exist if gaindb: dblist = saltio.readgaindb(gaindb) else: dblist = [] # does crosstalk coefficient data exist if xtalkfile: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict = None #does the mosaic file exist--raise error if no saltio.fileexists(geomfile) # Delete the obslog file if it already exists if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile) #read in the obsveration log or create it if os.path.isfile(obslogfile): msg = 'The observing log already exists. Please either delete it or run saltclean with clobber=yes' raise SaltError(msg) else: headerDict = obslog(infiles, log) obsstruct = createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) #create the list of bias frames and process them filename = obsstruct.data.field('FILENAME') detmode = obsstruct.data.field('DETMODE') ccdtype = obsstruct.data.field('CCDTYPE') #set the bias list of objects biaslist = filename[ccdtype == 'ZERO'] masterbias_dict = {} for img in infiles: if os.path.basename(img) in biaslist: #open the image struct = fits.open(img) bimg = outpath + 'bxgp' + os.path.basename(img) #print the message if log: message = 'Processing Zero frame %s' % img log.message(message, with_stdout=verbose) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) saltkey.new('SXTALK', time.asctime(time.localtime()), 'Images have been xtalk corrected', struct[0]) saltkey.new('SBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict = compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys = masterbias_dict[i][0] blist = masterbias_dict[i][1:] mbiasname = outpath + createmasterbiasname(blist, bkeys) bfiles = ','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #create the list of flatfields and process them flatlist = filename[ccdtype == 'FLAT'] masterflat_dict = {} for img in infiles: if os.path.basename(img) in flatlist: #open the image struct = fits.open(img) fimg = outpath + 'bxgp' + os.path.basename(img) #print the message if log: message = 'Processing Flat frame %s' % img log.message(message, with_stdout=verbose) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) saltkey.new('SXTALK', time.asctime(time.localtime()), 'Images have been xtalk corrected', struct[0]) saltkey.new('SBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, fimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterflat_dict = compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list) #create the master flat frame for i in masterflat_dict.keys(): fkeys = masterflat_dict[i][0] flist = masterflat_dict[i][1:] mflatname = outpath + createmasterflatname(flist, fkeys) ffiles = ','.join(flist) saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #process the science data for img in infiles: nimg = os.path.basename(img) #print nimg, nimg in flatlist, nimg in biaslist if not (nimg in biaslist): #open the image struct = fits.open(img) simg = outpath + 'bxgp' + os.path.basename(img) #print the message if log: message = 'Processing science frame %s' % img log.message(message, with_stdout=verbose) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) saltkey.new('SXTALK', time.asctime(time.localtime()), 'Images have been xtalk corrected', struct[0]) saltkey.new('SBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, simg, clobber=clobber) saltio.closefits(struct) #mosaic the files--currently not in the proper format--will update when it is if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])): mimg = outpath + 'mbxgp' + os.path.basename(img) saltmosaic(images=simg, outimages=mimg, outpref='', geomfile=geomfile, interp=interp, cleanup=True, clobber=clobber, logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg)
def saltxtalk(images, outimages, outpref, xtalkfile=None, usedb=False, clobber=True, logfile='salt.log', verbose=True): #start logging with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') # are input and output lists the same length? saltio.comparelists(infiles, outfiles, 'Input', 'output') # does crosstalk coefficient data exist if usedb: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict = None for img, oimg in zip(infiles, outfiles): #open the fits file struct = saltio.openfits(img) #find the best xcoeff for the image if using the db if usedb: obsdate = saltkey.get('DATE-OBS', struct[0]) obsdate = int('%s%s%s' % (obsdate[0:4], obsdate[5:7], obsdate[8:])) xkey = np.array(xdict.keys()) date = xkey[abs(xkey - obsdate).argmin()] xcoeff = xdict[date] else: xcoeff = [] # identify instrument instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid( struct) # has file been prepared already? if saltkey.found(keyxtalk, struct[0]): message = '%s has already been xtalk corrected' % img raise SaltError(message) #apply the cross-talk correction struct = xtalk(struct, xcoeff, log=log, verbose=verbose) # housekeeping keywords fname, hist = history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SXTALK', 'Images have been xtalk corrected', hist) # write FITS file saltio.writefits(struct, oimg, clobber=clobber) saltio.closefits(struct)
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None, subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log',verbose=True): """Convert MEF HRS data into a single image. If variance frames and BPMs, then convert them to the same format as well. Returns an MEF image but that is combined into a single frame """ with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outpath=saltio.abspath(outpath) if saltio.checkfornone(obslogfile) is None: raise SaltError('Obslog file is required') # Delete the obslog file if it already exists if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile): if os.path.isfile(obslogfile): saltio.delete(obslogfile) #read in the obsveration log or create it headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) else: obsstruct=saltio.openfits(obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') ccdtype=obsstruct.data.field('OBJECT') biaslist=filename[ccdtype=='Bias'] masterbias_dict={} if log: log.message('Processing Bias Frames') for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=pyfits.open(img) bimg=outpath+'bgph'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose, with_header=False) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=False, imstack=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list) #create the master bias frame for i in list(masterbias_dict.keys()): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys, x1=5, x2=13) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #apply full reductions to the science data for img in infiles: nimg=os.path.basename(img) if not nimg in biaslist: #open the image struct=pyfits.open(img) simg=outpath+'mbgph'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #get master bias frame masterbias=get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list) if masterbias: subbias=True bstruct=saltio.openfits(masterbias) else: subbias=False bstruct=None #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=subbias, imstack=True, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) return
def saltbias(images,outimages,outpref,subover=True,trim=True,subbias=False, masterbias='bias.fits', median=False, function='polynomial', order=3, rej_lo=3, rej_hi=3, niter=10, plotover=False, turbo=False, clobber=False, logfile='salt.log', verbose=True): status = 0 ifil = 0 ii = 0 mbiasdata = [] bstruct = '' biasgn = '' biassp = '' biasbn = '' biasin = '' filetime = {} biastime = {} for i in range(1,7): filetime[i] = [] biastime[i] = [] with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') # are input and output lists the same length? saltio.comparelists(infiles,outfiles,'Input','output') # Does master bias frame exist? # gain, speed, binning and instrument of master bias frame if subbias: if os.path.isfile(masterbias): bstruct = saltio.openfits(masterbias) else: message = 'Master bias frame %s does not exist' % masterbias raise SaltError(message) else: bstruct=None # open each raw image file for img, oimg in zip(infiles, outfiles): #open the file struct = saltio.openfits(img) #check to see if it has already been bias subtracted instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct) # has file been biaseded already? try: key = struct[0].header[keybias] message = 'File %s has already been de-biased ' % infile raise SaltError(message) except: pass #compare with the master bias to make sure they are the same if subbias: pass #subtract the bias struct=bias(struct,subover=subover, trim=trim, subbias=subbias, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],keybias, 'Images have been de-biased', hist) # write FITS file saltio.writefits(struct,oimg, clobber=clobber) saltio.closefits(struct)
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None, subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log', verbose=True): """Convert MEF HRS data into a single image. If variance frames and BPMs, then convert them to the same format as well. Returns an MEF image but that is combined into a single frame """ with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outpath = saltio.abspath(outpath) if saltio.checkfornone(obslogfile) is None: raise SaltError('Obslog file is required') # Delete the obslog file if it already exists if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile): if os.path.isfile(obslogfile): saltio.delete(obslogfile) #read in the obsveration log or create it headerDict = obslog(infiles, log) obsstruct = createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) else: obsstruct = saltio.openfits(obslogfile) #create the list of bias frames and process them filename = obsstruct.data.field('FILENAME') detmode = obsstruct.data.field('DETMODE') ccdtype = obsstruct.data.field('OBJECT') biaslist = filename[ccdtype == 'Bias'] masterbias_dict = {} if log: log.message('Processing Bias Frames') for img in infiles: if os.path.basename(img) in biaslist: #open the image struct = pyfits.open(img) bimg = outpath + 'bgph' + os.path.basename(img) #print the message if log: message = 'Processing Zero frame %s' % img log.message(message, with_stdout=verbose, with_header=False) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=False, imstack=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict = compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys = masterbias_dict[i][0] blist = masterbias_dict[i][1:] mbiasname = outpath + createmasterbiasname( blist, bkeys, x1=5, x2=13) bfiles = ','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #apply full reductions to the science data for img in infiles: nimg = os.path.basename(img) if not nimg in biaslist: #open the image struct = pyfits.open(img) simg = outpath + 'mbgph' + os.path.basename(img) #print the message if log: message = 'Processing science frame %s' % img log.message(message, with_stdout=verbose) #get master bias frame masterbias = get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list) if masterbias: subbias = True bstruct = saltio.openfits(masterbias) else: subbias = False bstruct = None #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=subbias, imstack=True, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, simg, clobber=clobber) saltio.closefits(struct) return