def createobslogfits(headerDict): """Create the fits table for the observation log""" # define generic columns of output table col=[] for k, f in zip(headerList, formatList): print k,f, headerDict[k] col.append(pyfits.Column(name=k, format=f, array=headerDict[k])) for k, f in zip(scamheaderList, scamformatList): print k,f, headerDict[k] col.append(pyfits.Column(name=k, format=f, array=headerDict[k])) for k, f in zip(rssheaderList, rssformatList): print k,f, headerDict[k] col.append(pyfits.Column(name=k, format=f, array=headerDict[k])) # construct FITS table from columns table = saltio.fitscolumns(col) # write FITS table to output file struct=saltio.newfitstable(table) # name the table extension saltkey.new('EXTNAME','OBSLOG','extension name', struct) # housekeeping keywords saltkey.put('SAL-TLM',time.asctime(time.localtime()), struct) return struct
def addhousekeeping(struct, outhdu, outfile): """housekeeping keywords""" saltsafekey.put('SAL-TLM',time.asctime(time.localtime()),struct,outfile) saltsafekey.new('EXTNAME','SCI','Extension name',struct,outfile) saltsafekey.new('EXTVER',outhdu,'Extension number',struct,outfile) return struct
def addhousekeeping(struct, outhdu, outfile): """housekeeping keywords""" saltsafekey.put('SAL-TLM', time.asctime(time.localtime()), struct, outfile) saltsafekey.new('EXTNAME', 'SCI', 'Extension name', struct, outfile) saltsafekey.new('EXTVER', outhdu, 'Extension number', struct, outfile) return struct
def prepare(struct,createvar=False, badpixelstruct=None, namps=2): #set up the file name infile=saltkey.getimagename(struct[0]) #include information about which headers are science extensoisn #and check to make sure each array is a data array nextend=len(struct)-1 nsciext=nextend for i in range(1,nextend+1): try: struct[i].header['XTENSION'] == 'IMAGE' except: msg='Extension %i in %s is not an image data' raise SaltError(msg) saltkey.new('EXTNAME','SCI','Extension name',struct[i]) saltkey.new('EXTVER',i,'Extension number',struct[i]) #check the number of amplifiers #TODO: This is current hardwired at a value of 2 nccds = saltkey.get('NCCDS',struct[0]) if (nextend%(nccds*namps) != 0): message = 'ERROR -- SALTPREPARE: Number of file extensions and ' message += 'number of amplifiers are not consistent in ' + infile raise SaltError(message) #Add the inv. variance frame if requested--assumes no variance frame #and the science frames are in the first n frames if createvar: #create the inv. variance frames for i in range(1, nsciext+1): try: hdu=CreateVariance(struct[i], i, nextend+i) except Exception, e: msg='Cannot create variance frame in extension %i of %s because %s' % (nextend+i, infile, e) raise SaltError(msg) struct[i].header.update('VAREXT',nextend+i, comment='Extension for Variance Frame') struct.append(hdu) nextend+=nsciext #create the badpixelframes for i in range(1, nsciext+1): try: hdu=createbadpixel(struct, badpixelstruct, i, nextend+i) except Exception, e: msg='Could not create bad pixel extension in ext %i of %s because %s' % (nextend+i, infile, e) raise SaltError(msg) struct[i].header.update('BPMEXT',nextend+i, comment='Extension for Bad Pixel Mask') struct.append(hdu)
def updateheaders(struct, ext, tdiff, real_expt, utc, infile): # exit if tdiff wasn't updated if tdiff == real_expt: msg='No adequate correction found for frame %i in file %s' % (ext, infile) raise SaltError(msg) return struct # calculate the new utc value try: ntime=salttime.sex2dec(utc) ntime=ntime-tdiff/3600.0 newutc=salttime.dec2sex(ntime) except Exception as e: msg='Could not update UTC in %i header of image %s because %s' % (ext, infile, e) raise SaltError(msg) return struct # update the headers if utc==saltsafekey.get('UTC-OBS', struct): expt_string='%5.4f' % real_expt td_string='%5.4f' % tdiff if not saltsafekey.found('DUTC', struct): try: saltsafekey.put('UTC-OBS', newutc, struct, infile) saltsafekey.put('TIME-OBS', newutc, struct, infile) saltsafekey.new('DWETIME', expt_string, 'Dwell Time', struct, infile) saltsafekey.new('DUTC', td_string, 'Change in UTC time', struct, infile) except Exception as e: msg='Could not update %i header of image %s because %s' % (ext, infile, e) raise SaltIOError(msg) else: try: saltsafekey.put('UTC-OBS', newutc, struct, infile) saltsafekey.put('TIME-OBS', newutc, struct, infile) saltsafekey.put('DWETIME', real_expt, struct, infile) saltsafekey.put('DUTC', tdiff, struct, infile) except Exception as e: msg='Could not update %i header of image %s because %s' % (ext, infile, e) raise SaltError(msg) else: raise SaltIOError('Frame missing from list of times') return struct
def readtimefix(hdu, dsteps=7, transtime=4e-3): """Update the hdu with the correct time for when the exposure started and add the READTIME keyword dsteps--the number of readouts to correct for transtime--the transfer time between each frame """ #check for if the data has already been processed if saltkey.found('READTIME', hdu): raise SaltIOError(' has already been processed') #determine the UTC time utctime = saltkey.get('UTC-OBS', hdu) timeobs = saltkey.get('TIME-OBS', hdu) dateobs = saltkey.get('DATE-OBS', hdu) exptime = float(saltkey.get('EXPTIME', hdu)) #add the readtime header saltkey.new("READTIME", utctime, 'Time of the readout of the frame', hdu) #correct the utctime--first switch to datetime to properly handle #dates around changes in hours y, m, d = dateobs.split('-') H, M, S = utctime.split(':') s = int(float(S)) ms = int(1e6 * (float(S) - s)) newtime = datetime.datetime(int(y), int(m), int(d), int(H), int(M), s, ms) #correct the datetime dtime = dsteps * (exptime + transtime) s = int(dtime) ms = int(1e6 * (dtime - s)) newtime = newtime - datetime.timedelta(0, s, ms) #update the headkeywords hdu.header["UTC-OBS"] = str(newtime.time()) saltkey.put("UTC-OBS", str(newtime.time()), hdu) saltkey.put("TIME-OBS", str(newtime.time()), hdu) saltkey.put("DATE-OBS", str(newtime.date()), hdu) return hdu
def readtimefix(hdu, dsteps=7, transtime=4e-3): """Update the hdu with the correct time for when the exposure started and add the READTIME keyword dsteps--the number of readouts to correct for transtime--the transfer time between each frame """ #check for if the data has already been processed if saltkey.found('READTIME', hdu): raise SaltIOError(' has already been processed') #determine the UTC time utctime=saltkey.get('UTC-OBS', hdu) timeobs=saltkey.get('TIME-OBS', hdu) dateobs=saltkey.get('DATE-OBS', hdu) exptime=float(saltkey.get('EXPTIME', hdu)) #add the readtime header saltkey.new("READTIME",utctime,'Time of the readout of the frame', hdu) #correct the utctime--first switch to datetime to properly handle #dates around changes in hours y,m,d=dateobs.split('-') H,M,S=utctime.split(':') s=int(float(S)) ms=int(1e6*(float(S)-s)) newtime=datetime.datetime(int(y),int(m),int(d),int(H),int(M),s,ms) #correct the datetime dtime=dsteps*(exptime+transtime) s=int(dtime) ms=int(1e6*(dtime-s)) newtime=newtime-datetime.timedelta(0, s, ms) #update the headkeywords hdu.header.update("UTC-OBS", str(newtime.time())) saltkey.put("UTC-OBS", str(newtime.time()), hdu) saltkey.put("TIME-OBS", str(newtime.time()), hdu) saltkey.put("DATE-OBS", str(newtime.date()), hdu) return hdu
def createrecord(recfile, fitcol, keycol, oldcol, newcol, clobber): """Create the fits table record of all of the changes that were made""" col1 = fits.Column(name='FILE',format='32A',array=fitcol) col2 = fits.Column(name='KEYWD',format='8A',array=keycol) col3 = fits.Column(name='OLD_VAL',format='24A',array=oldcol) col4 = fits.Column(name='NEW_VAL',format='24A',array=newcol) cols = fits.ColDefs([col1,col2,col3,col4]) rectable = fits.BinTableHDU.from_columns(cols) #add some additional keywords saltkey.new('EXTNAME','NEWKEYS','Name of this binary table extension', rectable) saltkey.new('OBSERVAT','SALT','South African Large Telescope', rectable) saltkey.new('SAL-TLM',time.asctime(time.localtime()), 'File last updated by PySALT tools',rectable) saltkey.new('SAL-EDT',time.asctime(time.localtime()), 'Keywords updated by SALTEDTKY',rectable) #write it out saltio.writefits(rectable, recfile, clobber)
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None, subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log', verbose=True): """Convert MEF HRS data into a single image. If variance frames and BPMs, then convert them to the same format as well. Returns an MEF image but that is combined into a single frame """ with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outpath = saltio.abspath(outpath) if saltio.checkfornone(obslogfile) is None: raise SaltError('Obslog file is required') # Delete the obslog file if it already exists if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile): if os.path.isfile(obslogfile): saltio.delete(obslogfile) #read in the obsveration log or create it headerDict = obslog(infiles, log) obsstruct = createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) else: obsstruct = saltio.openfits(obslogfile) #create the list of bias frames and process them filename = obsstruct.data.field('FILENAME') detmode = obsstruct.data.field('DETMODE') ccdtype = obsstruct.data.field('OBJECT') biaslist = filename[ccdtype == 'Bias'] masterbias_dict = {} if log: log.message('Processing Bias Frames') for img in infiles: if os.path.basename(img) in biaslist: #open the image struct = pyfits.open(img) bimg = outpath + 'bgph' + os.path.basename(img) #print the message if log: message = 'Processing Zero frame %s' % img log.message(message, with_stdout=verbose, with_header=False) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=False, imstack=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict = compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys = masterbias_dict[i][0] blist = masterbias_dict[i][1:] mbiasname = outpath + createmasterbiasname( blist, bkeys, x1=5, x2=13) bfiles = ','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #apply full reductions to the science data for img in infiles: nimg = os.path.basename(img) if not nimg in biaslist: #open the image struct = pyfits.open(img) simg = outpath + 'mbgph' + os.path.basename(img) #print the message if log: message = 'Processing science frame %s' % img log.message(message, with_stdout=verbose) #get master bias frame masterbias = get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list) if masterbias: subbias = True bstruct = saltio.openfits(masterbias) else: subbias = False bstruct = None #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=subbias, imstack=True, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, simg, clobber=clobber) saltio.closefits(struct) return
def wavemap( hdu, soldict, caltype="line", function="poly", order=3, blank=0, nearest=False, array_only=False, clobber=True, log=None, verbose=True, ): """Read in an image and a set of wavlength solutions. Calculate the best wavelength solution for a given dataset and then apply that data set to the image return """ # set up the time of the observation dateobs = saltkey.get("DATE-OBS", hdu[0]) utctime = saltkey.get("TIME-OBS", hdu[0]) exptime = saltkey.get("EXPTIME", hdu[0]) instrume = saltkey.get("INSTRUME", hdu[0]).strip() grating = saltkey.get("GRATING", hdu[0]).strip() if caltype == "line": grang = saltkey.get("GRTILT", hdu[0]) arang = saltkey.get("CAMANG", hdu[0]) else: grang = saltkey.get("GR-ANGLE", hdu[0]) arang = saltkey.get("AR-ANGLE", hdu[0]) filtername = saltkey.get("FILTER", hdu[0]).strip() slitname = saltkey.get("MASKID", hdu[0]) slit = st.getslitsize(slitname) xbin, ybin = saltkey.ccdbin(hdu[0]) timeobs = sr.enterdatetime("%s %s" % (dateobs, utctime)) # check to see if there is more than one solution if caltype == "line": if len(soldict) == 1: sol = soldict.keys()[0] slitid = None if not sr.matchobservations(soldict[sol], instrume, grating, grang, arang, filtername, slitid): msg = "Observations do not match setup for transformation but using the solution anyway" if log: log.warning(msg) for i in range(1, len(hdu)): if hdu[i].name == "SCI": if log: log.message("Correcting extension %i" % i) istart = int(0.5 * len(hdu[i].data)) # open up the data # set up the xarr and initial wavlength solution xarr = np.arange(len(hdu[i].data[istart]), dtype="int64") # get the slitid try: slitid = saltkey.get("SLITNAME", hdu[i]) except: slitid = None # check to see if wavext is already there and if so, then check update # that for the transformation from xshift to wavelength if saltkey.found("WAVEXT", hdu[i]): w_ext = saltkey.get("WAVEXT", hdu[i]) - 1 wavemap = hdu[w_ext].data function, order, coef = sr.findlinesol( soldict, istart, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slitid, xarr, ) ws = WavelengthSolution.WavelengthSolution(xarr, xarr, function=function, order=order) ws.set_coef(coef) for j in range(len(hdu[i].data)): wavemap[j, :] = ws.value(wavemap[j, :]) if array_only: return wavemap hdu[w_ext].data = wavemap continue # set up a wavelength solution -- still in here for testing MOS data try: w_arr = sr.findsol( xarr, soldict, istart, caltype, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slit, xbin, ybin, slitid, function, order, ) except SALTSpecError as e: if slitid: msg = "SLITID %s: %s" % (slitid, e) if log: log.warning(msg) continue else: raise SALTSpecError(e) if w_arr is None: w_arr = sr.findsol( xarr, soldict, istart, "rss", nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slit, xbin, ybin, slitid, function, order, ) # for each line in the data, determine the wavelength solution # for a given line in the image wavemap = np.zeros_like(hdu[i].data) for j in range(len(hdu[i].data)): # find the wavelength solution for the data w_arr = sr.findsol( xarr, soldict, j, caltype, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slit, xbin, ybin, slitid, function, order, ) if w_arr is not None: wavemap[j, :] = w_arr if array_only: return wavemap # write out the oimg hduwav = fits.ImageHDU(data=wavemap, header=hdu[i].header, name="WAV") hdu.append(hduwav) saltkey.new("WAVEXT", len(hdu) - 1, "Extension for Wavelength Map", hdu[i]) return hdu
def make_mosaic(struct, gap, xshift, yshift, rotation, interp_type='linear', boundary='constant', constant=0, geotran=True, fill=False, cleanup=True, log=None, verbose=False): """Given a SALT image struct, combine each of the individual amplifiers and apply the geometric CCD transformations to the image """ # get the name of the file infile = saltkey.getimagename(struct[0], base=True) outpath = './' # identify instrument instrume, keyprep, keygain, keybias, keyxtalk, keyslot = \ saltkey.instrumid(struct) # how many amplifiers? nsciext = saltkey.get('NSCIEXT', struct[0]) nextend = saltkey.get('NEXTEND', struct[0]) nccds = saltkey.get('NCCDS', struct[0]) amplifiers = nccds * 2 if nextend > nsciext: varframe = True else: varframe = False # CCD geometry coefficients if (instrume == 'RSS' or instrume == 'PFIS'): xsh = [0., xshift[0], 0., xshift[1]] ysh = [0., yshift[0], 0., yshift[1]] rot = [0., rotation[0], 0., rotation[1]] elif instrume == 'SALTICAM': xsh = [0., xshift[0], 0.] ysh = [0., yshift[0], 0.] rot = [0., rotation[0], 0] # how many extensions? nextend = saltkey.get('NEXTEND', struct[0]) # CCD on-chip binning xbin, ybin = saltkey.ccdbin(struct[0]) # create temporary primary extension outstruct = [] outstruct.append(struct[0]) # define temporary FITS file store tiled CCDs tilefile = saltio.tmpfile(outpath) tilefile += 'tile.fits' if varframe: tilehdu = [None] * (3 * int(nsciext / 2) + 1) else: tilehdu = [None] * int(nsciext / 2 + 1) tilehdu[0] = fits.PrimaryHDU() #tilehdu[0].header = struct[0].header if log: log.message('', with_stdout=verbose) # iterate over amplifiers, stich them to produce file of CCD images for i in range(int(nsciext / 2)): hdu = i * 2 + 1 # amplifier = hdu%amplifiers # if (amplifier == 0): amplifier = amplifiers # read DATASEC keywords datasec1 = saltkey.get('DATASEC', struct[hdu]) datasec2 = saltkey.get('DATASEC', struct[hdu + 1]) xdsec1, ydsec1 = saltstring.secsplit(datasec1) xdsec2, ydsec2 = saltstring.secsplit(datasec2) # read images imdata1 = saltio.readimage(struct, hdu) imdata2 = saltio.readimage(struct, hdu + 1) # tile 2n amplifiers to yield n CCD images outdata = numpy.zeros( (int(ydsec1[1] + abs(ysh[i + 1] / ybin)), int(xdsec1[1] + xdsec2[1] + abs(xsh[i + 1] / xbin))), numpy.float32) # set up the variance frame if varframe: vardata = outdata.copy() vdata1 = saltio.readimage(struct, struct[hdu].header['VAREXT']) vdata2 = saltio.readimage(struct, struct[hdu + 1].header['VAREXT']) bpmdata = outdata.copy() bdata1 = saltio.readimage(struct, struct[hdu].header['BPMEXT']) bdata2 = saltio.readimage(struct, struct[hdu + 1].header['BPMEXT']) x1 = xdsec1[0] - 1 if x1 != 0: msg = 'The data in %s have not been trimmed prior to mosaicking.' \ % infile log.error(msg) if xsh[i + 1] < 0: x1 += int(abs(xsh[i + 1] / xbin)) x2 = x1 + xdsec1[1] y1 = ydsec1[0] - 1 if ysh[i + 1] < 0: y1 += int(abs(ysh[i + 1] / ybin)) y2 = y1 + ydsec1[1] outdata[y1:y2, x1:x2] =\ imdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] if varframe: vardata[y1:y2, x1:x2] =\ vdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] bpmdata[y1:y2, x1:x2] =\ bdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] x1 = x2 x2 = x1 + xdsec2[1] y1 = ydsec2[0] - 1 if ysh[i + 1] < 0: y1 += abs(ysh[i + 1] / ybin) y2 = y1 + ydsec2[1] outdata[y1:y2, x1:x2] =\ imdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] if varframe: vardata[y1:y2, x1:x2] =\ vdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] bpmdata[y1:y2, x1:x2] =\ bdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] # size of new image naxis1 = str(xdsec1[1] + xdsec2[1]) naxis2 = str(ydsec1[1]) # add image and keywords to HDU list tilehdu[i + 1] = fits.ImageHDU(outdata) tilehdu[i + 1].header = struct[hdu].header #tilehdu[ # i + 1].header['DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' if varframe: vext = i + 1 + int(nsciext / 2.) tilehdu[vext] = fits.ImageHDU(vardata) #tilehdu[vext].header = struct[struct[hdu].header['VAREXT']].header #tilehdu[vext].header[ # 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' bext = i + 1 + 2 * int(nsciext / 2.) tilehdu[bext] = fits.ImageHDU(bpmdata) #tilehdu[bext].header = struct[struct[hdu].header['BPMEXT']].header #tilehdu[bext].header[ # 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' # image tile log message #1 if log: message = os.path.basename(infile) + '[' + str(hdu) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> ' message += os.path.basename(tilefile) + '[' + str(i + 1) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + ']' log.message(message, with_stdout=verbose, with_header=False) message = os.path.basename(infile) + '[' + str(hdu + 1) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> ' message += os.path.basename(tilefile) + '[' + str(i + 1) + '][' message += str(xdsec1[1] + 1) + ':' + \ str(xdsec1[1] + xdsec2[1]) + ',' message += str(ydsec2[0]) + ':' + str(ydsec2[1]) + ']' log.message(message, with_stdout=verbose, with_header=False) # write temporary file of tiled CCDs hdulist = fits.HDUList(tilehdu) hdulist.writeto(tilefile) # iterate over CCDs, transform and rotate images yrot = [None] * 4 xrot = [None] * 4 tranfile = [' '] tranhdu = [0] if varframe: tranfile = [''] * (3 * int(nsciext / 2) + 1) tranhdu = [0] * (3 * int(nsciext / 2) + 1) else: tranfile = [''] * int(nsciext / 2 + 1) tranhdu = [0] * int(nsciext / 2 + 1) # this is hardwired for SALT where the second CCD is considered the # fiducial for hdu in range(1, int(nsciext / 2 + 1)): tranfile[hdu] = saltio.tmpfile(outpath) tranfile[hdu] += 'tran.fits' if varframe: tranfile[hdu + nccds] = saltio.tmpfile(outpath) + 'tran.fits' tranfile[hdu + 2 * nccds] = saltio.tmpfile(outpath) + 'tran.fits' ccd = hdu % nccds if (ccd == 0): ccd = nccds # correct rotation for CCD binning yrot[ccd] = rot[ccd] * ybin / xbin xrot[ccd] = rot[ccd] * xbin / ybin dxshift = xbin * int(float(int(gap) / xbin) + 0.5) - gap # transformation using geotran IRAF task # if (ccd == 1): if (ccd != 2): if geotran: message = '\nSALTMOSAIC -- geotran ' + tilefile + \ '[' + str(ccd) + '] ' + tranfile[hdu] message += ' \"\" \"\" xshift=' + \ str((xsh[ccd] + (2 - ccd) * dxshift) / xbin) + ' ' message += 'yshift=' + \ str(ysh[ccd] / ybin) + ' xrotation=' + str(xrot[ccd]) + ' ' message += 'yrotation=' + \ str(yrot[ccd]) + ' xmag=1 ymag=1 xmin=\'INDEF\'' message += 'xmax=\'INDEF\' ymin=\'INDEF\' ymax=\'INDEF\' ' message += 'ncols=\'INDEF\' ' message += 'nlines=\'INDEF\' verbose=\'no\' ' message += 'fluxconserve=\'yes\' nxblock=2048 ' message += 'nyblock=2048 interpolant=\'' + \ interp_type + '\' boundary=\'constant\' constant=0' log.message(message, with_stdout=verbose) yd, xd = tilehdu[ccd].data.shape ncols = 'INDEF' # ncols=xd+abs(xsh[ccd]/xbin) nlines = 'INDEF' # nlines=yd+abs(ysh[ccd]/ybin) geo_xshift = xsh[ccd] + (2 - ccd) * dxshift / xbin geo_yshift = ysh[ccd] / ybin iraf.images.immatch.geotran(tilefile + "[" + str(ccd) + "]", tranfile[hdu], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) if varframe: var_infile = tilefile + "[" + str(ccd + nccds) + "]" iraf.images.immatch.geotran(var_infile, tranfile[hdu + nccds], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) var2_infile = tilefile + "[" + str(ccd + 2 * nccds) + "]" iraf.images.immatch.geotran(var2_infile, tranfile[hdu + 2 * nccds], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) # open the file and copy the data to tranhdu tstruct = fits.open(tranfile[hdu]) tranhdu[hdu] = tstruct[0].data tstruct.close() if varframe: tranhdu[hdu + nccds] = fits.open(tranfile[hdu + nccds])[0].data tranhdu[hdu + 2 * nccds] = fits.open( tranfile[hdu + 2 * nccds])[0].data else: log.message("Transform CCD #%i using dx=%s, dy=%s, rot=%s" % (ccd, xsh[ccd] / 2.0, ysh[ccd] / 2.0, xrot[ccd]), with_stdout=verbose, with_header=False) tranhdu[hdu] = geometric_transform( tilehdu[ccd].data, tran_func, prefilter=False, order=1, extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) tstruct = fits.PrimaryHDU(tranhdu[hdu]) tstruct.writeto(tranfile[hdu]) if varframe: tranhdu[hdu + nccds] = geometric_transform( tilehdu[hdu + 3].data, tran_func, prefilter=False, order=1, extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) tranhdu[hdu + 2 * nccds] = geometric_transform( tilehdu[hdu + 6].data, tran_func, prefilter=False, order=1, extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) else: log.message("Transform CCD #%i using dx=%s, dy=%s, rot=%s" % (ccd, 0, 0, 0), with_stdout=verbose, with_header=False) tranhdu[hdu] = tilehdu[ccd].data if varframe: tranhdu[hdu + nccds] = tilehdu[ccd + nccds].data tranhdu[hdu + 2 * nccds] = tilehdu[ccd + 2 * nccds].data # open outfile if varframe: outlist = 4 * [None] else: outlist = 2 * [None] #outlist[0] = struct[0].copy() outlist[0] = fits.PrimaryHDU() outlist[0].header = struct[0].header naxis1 = int(gap / xbin * (nccds - 1)) naxis2 = 0 for i in range(1, nccds + 1): yw, xw = tranhdu[i].shape naxis1 += xw + int(abs(xsh[ccd] / xbin)) + 1 naxis2 = max(naxis2, yw) outdata = numpy.zeros((naxis2, naxis1), numpy.float32) outdata.shape = naxis2, naxis1 if varframe: vardata = outdata * 0 bpmdata = outdata * 0 + 1 # iterate over CCDs, stich them to produce a full image hdu = 0 totxshift = 0 for hdu in range(1, nccds + 1): # read DATASEC keywords ydsec, xdsec = tranhdu[hdu].shape # define size and shape of final image # tile CCDs to yield mosaiced image x1 = int((hdu - 1) * (xdsec + gap / xbin)) + int(totxshift) x2 = xdsec + x1 y1 = int(0) y2 = int(ydsec) outdata[y1:y2, x1:x2] = tranhdu[hdu] totxshift += int(abs(xsh[hdu] / xbin)) + 1 if varframe: vardata[y1:y2, x1:x2] = tranhdu[hdu + nccds] bpmdata[y1:y2, x1:x2] = tranhdu[hdu + 2 * nccds] # make sure to cover up all the gaps include bad areas if varframe: baddata = (outdata == 0) baddata = nd.maximum_filter(baddata, size=3) bpmdata[baddata] = 1 # fill in the gaps if requested if fill: if varframe: outdata = fill_gaps(outdata, 0) else: outdata = fill_gaps(outdata, 0) # add to the file outlist[1] = fits.ImageHDU(outdata) if varframe: outlist[2] = fits.ImageHDU(vardata, name='VAR') outlist[3] = fits.ImageHDU(bpmdata, name='BPM') # create the image structure outstruct = fits.HDUList(outlist) # update the head informaation # housekeeping keywords saltkey.put('NEXTEND', 2, outstruct[0]) saltkey.new('EXTNAME', 'SCI', 'Extension name', outstruct[1]) saltkey.new('EXTVER', 1, 'Extension number', outstruct[1]) if varframe: saltkey.new('VAREXT', 2, 'Variance frame extension', outstruct[1]) saltkey.new('BPMEXT', 3, 'BPM Extension', outstruct[1]) try: saltkey.copy(struct[1], outstruct[1], 'CCDSUM') except: pass # Add keywords associated with geometry saltkey.new('SGEOMGAP', gap, 'SALT Chip Gap', outstruct[0]) c1str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[0], yshift[0], rotation[0]) saltkey.new('SGEOM1', c1str, 'SALT Chip 1 Transform', outstruct[0]) c2str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[1], yshift[1], rotation[1]) saltkey.new('SGEOM2', c2str, 'SALT Chip 2 Transform', outstruct[0]) # WCS keywords saltkey.new('CRPIX1', 0, 'WCS: X reference pixel', outstruct[1]) saltkey.new('CRPIX2', 0, 'WCS: Y reference pixel', outstruct[1]) saltkey.new('CRVAL1', float(xbin), 'WCS: X reference coordinate value', outstruct[1]) saltkey.new('CRVAL2', float(ybin), 'WCS: Y reference coordinate value', outstruct[1]) saltkey.new('CDELT1', float(xbin), 'WCS: X pixel size', outstruct[1]) saltkey.new('CDELT2', float(ybin), 'WCS: Y pixel size', outstruct[1]) saltkey.new('CTYPE1', 'pixel', 'X type', outstruct[1]) saltkey.new('CTYPE2', 'pixel', 'Y type', outstruct[1]) # cleanup temporary files if cleanup: for tfile in tranfile: if os.path.isfile(tfile): saltio.delete(tfile) if os.path.isfile(tilefile): status = saltio.delete(tilefile) # return the file return outstruct
def write_ext_header (hdu, file, bhdu, ut,date,bscale,bzero,exptime,gain,rdnoise, datasec,detsec,ccdsec,ampsec,biassec, deadtime=None, framecnt=None): """Update the header values for the extension returns """ #if (status==0): status = saltkey.new("BSCALE",bscale,"Val=BSCALE*pix+BZERO",hdu,file,logfile) #if (status==0): status = saltkey.new("BZERO",bzero,"Val=BSCALE*pix+BZERO",hdu,file,logfile) saltkey.new("UTC-OBS",ut,"UTC start of observation",hdu,file) saltkey.new("TIME-OBS",ut,"UTC start of observation",hdu,file) saltkey.new("DATE-OBS",date,"Date of the observation",hdu,file) saltkey.new("EXPTIME",exptime,"Exposure time",hdu,file) saltkey.new("GAIN",gain,"Nominal CCD gain (e/ADU)",hdu,file) saltkey.new("RDNOISE",rdnoise,"Nominal readout noise in e",hdu,file) saltkey.new("CCDSUM", bhdu.header['CCDSUM'], 'On chip summation', hdu, file) saltkey.copy(hdu,bhdu,"DETSIZE") saltkey.new("DATASEC",datasec,"Data Section",hdu,file) saltkey.new("DETSEC",detsec,"Detector Section",hdu,file) saltkey.new("CCDSEC ",ccdsec,"CCD Section",hdu,file) saltkey.new("AMPSEC",ampsec,"Amplifier Section",hdu,file) saltkey.new("BIASSEC",biassec,"Bias Section",hdu,file) if deadtime: saltkey.new("DEADTIME",deadtime,"Milliseconds waiting for readout", hdu, file) if framecnt: saltkey.new("FRAMECNT",framecnt,"Frame counter", hdu, file) return hdu
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None, geomfile=None,subover=True,trim=True,masbias=None, subbias=False, median=False, function='polynomial', order=5,rej_lo=3, rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='', clobber=False, cleanup=True, logfile='salt.log', verbose=True): """SALTADVANCE provides advanced data reductions for a set of data. It will sort the data, and first process the biases, flats, and then the science frames. It will record basic quality control information about each of the steps. """ plotover=False #start logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) infiles.sort() # create list of output files outpath=saltio.abspath(outpath) #log into the database sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) #does the gain database file exist if gaindb: dblist= saltio.readgaindb(gaindb) else: dblist=[] # does crosstalk coefficient data exist if xtalkfile: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict=None #does the mosaic file exist--raise error if no saltio.fileexists(geomfile) # Delete the obslog file if it already exists if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile) #read in the obsveration log or create it if os.path.isfile(obslogfile): msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes' raise SaltError(msg) else: headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') obsmode=obsstruct.data.field('OBSMODE') ccdtype=obsstruct.data.field('CCDTYPE') propcode=obsstruct.data.field('PROPID') masktype=obsstruct.data.field('MASKTYP') #set the bias list of objects biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')] masterbias_dict={} for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=fits.open(img) bimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #create the list of flatfields and process them flatlist=filename[ccdtype=='FLAT'] masterflat_dict={} for img in infiles: if os.path.basename(img) in flatlist: #open the image struct=fits.open(img) fimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Flat frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,fimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list) #create the master flat frame for i in masterflat_dict.keys(): fkeys=masterflat_dict[i][0] flist=masterflat_dict[i][1:] mflatname=outpath+createmasterflatname(flist, fkeys) ffiles=','.join(flist) saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #process the arc data arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')] for i, img in enumerate(infiles): nimg=os.path.basename(img) if nimg in arclist: #open the image struct=fits.open(img) simg=outpath+'bxgp'+os.path.basename(img) obsdate=os.path.basename(img)[1:9] #print the message if log: message='Processing ARC frame %s' % img log.message(message, with_stdout=verbose) struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the images mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg) #measure the arcdata arcimage=outpath+'mbxgp'+nimg dbfile=outpath+obsdate+'_specid.db' lamp = obsstruct.data.field('LAMPID')[i] lamp = lamp.replace(' ', '') lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) print arcimage, lampfile, os.getcwd() specidentify(arcimage, lampfile, dbfile, guesstype='rss', guessfile='', automethod='Matchlines', function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile, verbose=verbose) try: ximg = outpath+'xmbxgp'+os.path.basename(arcimage) specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, conserve=True, nearest=True, clobber=True, logfile=logfile, verbose=verbose) except: pass #process the science data for i, img in enumerate(infiles): nimg=os.path.basename(img) if not (nimg in flatlist or nimg in biaslist or nimg in arclist): #open the image struct=fits.open(img) if struct[0].header['PROPID'].count('CAL_GAIN'): continue simg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #Check to see if it is RSS 2x2 and add bias subtraction instrume=saltkey.get('INSTRUME', struct[0]).strip() gainset = saltkey.get('GAINSET', struct[0]) rospeed = saltkey.get('ROSPEED', struct[0]) target = saltkey.get('OBJECT', struct[0]).strip() exptime = saltkey.get('EXPTIME', struct[0]) obsmode = saltkey.get('OBSMODE', struct[0]).strip() detmode = saltkey.get('DETMODE', struct[0]).strip() masktype = saltkey.get('MASKTYP', struct[0]).strip() xbin, ybin = saltkey.ccdbin( struct[0], img) obsdate=os.path.basename(img)[1:9] bstruct=None crtype=None thresh=5 mbox=11 bthresh=5.0, flux_ratio=0.2 bbox=25 gain=1.0 rdnoise=5.0 fthresh=5.0 bfactor=2 gbox=3 maxiter=5 subbias=False if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW': bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin) if os.path.exists(bfile): bstruct=fits.open(bfile) subbias=True if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5: crtype='edge' thresh=5 mbox=11 bthresh=5.0, flux_ratio=0.2 bbox=25 gain=1.0 rdnoise=5.0 fthresh=5.0 bfactor=2 gbox=3 maxiter=3 #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \ bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise, bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the files--currently not in the proper format--will update when it is if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])): mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg) #if the file is spectroscopic mode, apply the wavelength correction if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT': dbfile=outpath+obsdate+'_specid.db' try: ximg = outpath+'xmbxgp'+os.path.basename(img) specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, conserve=True, nearest=True, clobber=True, logfile=logfile, verbose=verbose) except Exception, e: log.message('%s' % e) #clean up the results if cleanup: #clean up the bias frames for i in masterbias_dict.keys(): blist=masterbias_dict[i][1:] for b in blist: saltio.delete(b) #clean up the flat frames for i in masterflat_dict.keys(): flist=masterflat_dict[i][1:] for f in flist: saltio.delete(f)
def saltfpzeropoint(images,outimages, outpref, calfile, fpa, fpb, fpc, fpd, fpe, fpf, clobber=False,logfile='salt.log',verbose=True): """Adds zeropoint information to each image""" with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create the output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') #calculate the zeropoint coefficients fpcoef=np.array([fpa,fpb,fpc,fpd,fpe,fpf]) zpcoef,tstart=calc_zpcoef(calfile, fpcoef) # open each image and detect the ring for img, oimg in zip(infiles, outfiles): hdu=saltio.openfits(img) #get the image time t=(get_datetime(hdu)-tstart).seconds #add the header values to the image saltkey.new('FPA',fpa+zpcoef[1]+zpcoef[0]*t,'FPA Coef',hdu[0]) saltkey.new('FPB',fpb,'FPB Coef',hdu[0]) saltkey.new('FPC',fpc,'FPC Coef',hdu[0]) saltkey.new('FPD',fpd,'FPD Coef',hdu[0]) saltkey.new('FPE', 0,'FPE Coef',hdu[0]) saltkey.new('FPF',fpf,'FPF Coef',hdu[0]) #write the file out saltio.writefits(hdu, oimg, clobber) #log the action msg='Updating the FP information in %s' % (img) log.message(msg, with_stdout=verbose, with_header=False)
except Exception, e: msg = 'Could not update UTC in %i header of image %s because %s' % ( ext, infile, e) raise SaltError(msg) return struct # update the headers if utc == saltsafekey.get('UTC-OBS', struct): expt_string = '%5.4f' % real_expt td_string = '%5.4f' % tdiff if not saltsafekey.found('DUTC', struct): try: saltsafekey.put('UTC-OBS', newutc, struct, infile) saltsafekey.put('TIME-OBS', newutc, struct, infile) saltsafekey.new('DWETIME', expt_string, 'Dwell Time', struct, infile) saltsafekey.new('DUTC', td_string, 'Change in UTC time', struct, infile) except Exception, e: msg = 'Could not update %i header of image %s because %s' % ( ext, infile, e) raise SaltIOError(msg) else: try: saltsafekey.put('UTC-OBS', newutc, struct, infile) saltsafekey.put('TIME-OBS', newutc, struct, infile) saltsafekey.put('DWETIME', real_expt, struct, infile) saltsafekey.put('DUTC', tdiff, struct, infile) except Exception, e: msg = 'Could not update %i header of image %s because %s' % ( ext, infile, e)
def slot(struct, infile, dbspeed, dbrate, dbgain, dbnoise, dbbias, dbamp, xcoeff, gaindb, xtalkfile, logfile, verbose): import saltprint, saltkey, saltio, saltstat, time # identify instrument instrume, keyprep, keygain, keybias, keyxtalk, keyslot, status = saltkey.instrumid( struct, infile, logfile) # number of image HDU nextend = 0 while (status == 0): try: struct[nextend + 1].header['XTENSION'] nextend += 1 except: break nccds, status = saltkey.get('NCCDS', struct[0], infile, logfile) amplifiers = nccds * 2 if (nextend % (amplifiers) != 0): message = '\nERROR -- SALTSLOT: Number of image extensions and' message += 'number of amplifiers are not consistent' status = saltprint.err(saltlog, message) status = saltkey.new('NSCIEXT', nextend, 'Number of science extensions', struct[0], infile, logfile) status = saltkey.new('NEXTEND', nextend, 'Number of data extensions', struct[0], infile, logfile) # check image file and gain database are compatible if (status == 0): ngains = len(dbgain) if (int(max(dbamp)) != amplifiers): message = '\nERROR -- SALTGSLOT: ' + infile + ' contains ' + str( amplifiers) + ' amplifiers' message += ', the gaindb file ' + gaindb + ' contains ' + str( max(dbamp)) + ' amplifiers' status = saltprint.err(logfile, message) # check image file and cross talk database are compatible if (status == 0): if (len(xcoeff) - 1 != amplifiers): message = '\nERROR -- SALTSLOT: ' + infile + ' contains ' + str( amplifiers) + ' amplifiers' message += ', the cross talk file ' + xtalkfile + ' contains ' + str( len(xcoeff) - 1) + ' amplifiers' status = saltprint.err(logfile, message) # housekeeping keywords if (status == 0): status = saltkey.put('SAL-TLM', time.asctime(time.localtime()), struct[0], infile, logfile) status = saltkey.new(keyslot, time.asctime(time.localtime()), 'Data have been cleaned by SALTSLOT', struct[0], infile, logfile) # keywords for image extensions for i in range(nextend): hdu = i + 1 status = saltkey.new('EXTNAME', 'SCI', 'Extension name', struct[hdu], infile, logfile) status = saltkey.new('EXTVER', hdu, 'Extension number', struct[hdu], infile, logfile) # log coefficent table if (status == 0): message = '%30s %5s %4s %8s' % ('HDU', 'Gain', 'Bias', 'Xtalk') saltprint.log(logfile, '\n ---------------------------------------------', verbose) saltprint.log(logfile, message, verbose) saltprint.log(logfile, ' ---------------------------------------------', verbose) # loop over image extensions if (status == 0): for i in range(nextend / 2): hdu = i * 2 + 1 amplifier = hdu % amplifiers if (amplifier == 0): amplifier = amplifiers if (status == 0): value, status = saltkey.get('NAXIS1', struct[hdu], infile, logfile) naxis1 = int(value) if (status == 0): value, status = saltkey.get('NAXIS1', struct[hdu + 1], infile, logfile) naxis2 = int(value) if (status == 0 and hdu == 1): biassec, status = saltkey.get('BIASSEC', struct[hdu], infile, logfile) if (status == 0 and hdu == 1): ranges = biassec.lstrip('[').rstrip(']').split(',') x1_1 = int(ranges[0].split(':')[0]) - 1 x2_1 = int(ranges[0].split(':')[1]) - 1 y1_1 = int(ranges[1].split(':')[0]) - 1 y2_1 = int(ranges[1].split(':')[1]) - 1 if (status == 0 and hdu == 1): biassec, status = saltkey.get('BIASSEC', struct[hdu + 1], infile, logfile) if (status == 0 and hdu == 1): ranges = biassec.lstrip('[').rstrip(']').split(',') x1_2 = int(ranges[0].split(':')[0]) - 1 x2_2 = int(ranges[0].split(':')[1]) - 1 y1_2 = int(ranges[1].split(':')[0]) - 1 y2_2 = int(ranges[1].split(':')[1]) - 1 if (status == 0 and hdu == 1): datasec, status = saltkey.get('DATASEC', struct[hdu], infile, logfile) if (status == 0 and hdu == 1): ranges = datasec.lstrip('[').rstrip(']').split(',') dx1_1 = int(ranges[0].split(':')[0]) - 1 dx2_1 = int(ranges[0].split(':')[1]) dy1_1 = int(ranges[1].split(':')[0]) - 1 dy2_1 = int(ranges[1].split(':')[1]) if (status == 0 and hdu == 1): datasec, status = saltkey.get('DATASEC', struct[hdu + 1], infile, logfile) if (status == 0 and hdu == 1): ranges = datasec.lstrip('[').rstrip(']').split(',') dx1_2 = int(ranges[0].split(':')[0]) - 1 dx2_2 = int(ranges[0].split(':')[1]) dy1_2 = int(ranges[1].split(':')[0]) - 1 dy2_2 = int(ranges[1].split(':')[1]) if (status == 0 and dx2_1 - dx1_1 != dx2_2 - dx1_2): message = 'ERROR -- SALTSLOT: HDUs ' + infile message += '[' + str(hdu) + '] and ' + infile + '[' + str( hdu + 1) + ']' message += ' have different dimensions' status = saltprint.err(logfile, message) # read speed and gain of each exposure if (status == 0 and hdu == 1): gainset, status = saltkey.get('GAINSET', struct[0], infile, logfile) rospeed, status = saltkey.get('ROSPEED', struct[0], infile, logfile) if (rospeed == 'NONE'): saltprint.log(logfile, " ", verbose) message = "ERROR -- SALTSLOT: Readout speed is 'NONE' in " message += "primary keywords of " + infile status = saltprint.err(logfile, message) # read raw images if (status == 0): imagedata1, status = saltio.readimage(struct, hdu, logfile) imagedata2, status = saltio.readimage(struct, hdu + 1, logfile) # gain correction if (status == 0): for j in range(len(dbgain)): if (gainset == dbrate[j] and rospeed == dbspeed[j] and amplifier == int(dbamp[j])): try: gain1 = float(dbgain[j]) imagedata1 *= gain1 except: mesage = 'ERROR -- SALTSLOT: Cannot perform gain correction on image ' message += infile + '[' + str(hdu) + ']' status = saltprint.err(logfile, message) elif (gainset == dbrate[j] and rospeed == dbspeed[j] and amplifier + 1 == int(dbamp[j])): try: gain2 = float(dbgain[j]) imagedata2 *= gain2 except: mesage = 'ERROR -- SALTSLOT: Cannot perform gain correction on image ' message += infile + '[' + str(hdu + 1) + ']' status = saltprint.err(logfile, message) # crosstalk correction if (status == 0): revimage1 = imagedata1 * float(xcoeff[amplifier]) revimage2 = imagedata2 * float(xcoeff[amplifier + 1]) for j in range(dx2_1 - dx1_1 + 1): imagedata1[:, j] -= revimage2[:, dx2_2 - j - 1] imagedata2[:, j] -= revimage1[:, dx2_1 - j - 1] # bias subtraction if (status == 0): overx_val_1 = [] overx_val_2 = [] for x in range(x1_1, x2_1 + 1): list_1 = imagedata1[y1_1:y2_1, x] * 1.0 overx_val_1.append(saltstat.median(list_1, logfile)) overlevel_1 = saltstat.median(overx_val_1, logfile) for x in range(x1_2, x2_2 + 1): list_2 = imagedata2[y1_2:y2_2, x] * 1.0 overx_val_2.append(saltstat.median(list_2, logfile)) overlevel_2 = saltstat.median(overx_val_2, logfile) imagedata1 -= overlevel_1 imagedata2 -= overlevel_2 # trim overscan if (status == 0): imagedata1 = imagedata1[dy1_1:dy2_1, dx1_1:dx2_1] imagedata2 = imagedata2[dy1_2:dy2_2, dx1_2:dx2_2] datasec = '[1:' + str(dx2_1 - dx1_1) + ',1:' + str(dy2_1 - dy1_1) + ']' status = saltkey.put('DATASEC', datasec, struct[hdu], infile, logfile) status = saltkey.rem('BIASSEC', struct[hdu], infile, logfile) datasec = '[1:' + str(dx2_2 - dx1_2) + ',1:' + str(dy2_2 - dy1_2) + ']' status = saltkey.put('DATASEC', datasec, struct[hdu + 1], infile, logfile) status = saltkey.rem('BIASSEC', struct[hdu + 1], infile, logfile) # log coefficient table if (status == 0): infilename = infile.split('/') infilename = infilename[len(infilename) - 1] message = '%25s[%3d] %5.2f %4d %8.6f' % \ (infilename, hdu, gain1, overlevel_1, float(xcoeff[amplifier+1])) saltprint.log(logfile, message, verbose) message = '%25s[%3d] %5.2f %4d %8.6f' % \ (infilename, hdu+1, gain2, overlevel_2,float(xcoeff[amplifier])) saltprint.log(logfile, message, verbose) # update image in HDU structure if (status == 0): struct, status = saltio.writeimage(struct, hdu, imagedata1, logfile) struct, status = saltio.writeimage(struct, hdu + 1, imagedata2, logfile) return struct, status
def wavemap(hdu, soldict, caltype='line', function='poly', order=3, blank=0, nearest=False, array_only=False, clobber=True, log=None, verbose=True): """Read in an image and a set of wavlength solutions. Calculate the best wavelength solution for a given dataset and then apply that data set to the image return """ # set up the time of the observation dateobs = saltkey.get('DATE-OBS', hdu[0]) utctime = saltkey.get('TIME-OBS', hdu[0]) exptime = saltkey.get('EXPTIME', hdu[0]) instrume = saltkey.get('INSTRUME', hdu[0]).strip() grating = saltkey.get('GRATING', hdu[0]).strip() if caltype == 'line': grang = saltkey.get('GRTILT', hdu[0]) arang = saltkey.get('CAMANG', hdu[0]) else: grang = saltkey.get('GR-ANGLE', hdu[0]) arang = saltkey.get('AR-ANGLE', hdu[0]) filtername = saltkey.get('FILTER', hdu[0]).strip() slitname = saltkey.get('MASKID', hdu[0]) slit = st.getslitsize(slitname) xbin, ybin = saltkey.ccdbin(hdu[0]) timeobs = sr.enterdatetime('%s %s' % (dateobs, utctime)) # check to see if there is more than one solution if caltype == 'line': if len(soldict) == 1: sol = soldict.keys()[0] slitid = None if not sr.matchobservations(soldict[sol], instrume, grating, grang, arang, filtername, slitid): msg = 'Observations do not match setup for transformation but using the solution anyway' if log: log.warning(msg) for i in range(1, len(hdu)): if hdu[i].name == 'SCI': if log: log.message('Correcting extension %i' % i) istart = int(0.5 * len(hdu[i].data)) # open up the data # set up the xarr and initial wavlength solution xarr = np.arange(len(hdu[i].data[istart]), dtype='int64') # get the slitid try: slitid = saltkey.get('SLITNAME', hdu[i]) except: slitid = None #check to see if wavext is already there and if so, then check update #that for the transformation from xshift to wavelength if saltkey.found('WAVEXT', hdu[i]): w_ext = saltkey.get('WAVEXT', hdu[i]) - 1 wavemap = hdu[w_ext].data function, order, coef = sr.findlinesol( soldict, istart, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slitid, xarr) ws = WavelengthSolution.WavelengthSolution(xarr, xarr, function=function, order=order) ws.set_coef(coef) for j in range(len(hdu[i].data)): wavemap[j, :] = ws.value(wavemap[j, :]) if array_only: return wavemap hdu[w_ext].data = wavemap continue # set up a wavelength solution -- still in here for testing MOS data try: w_arr = sr.findsol(xarr, soldict, istart, caltype, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slit, xbin, ybin, slitid, function, order) except SALTSpecError as e: if slitid: msg = 'SLITID %s: %s' % (slitid, e) if log: log.warning(msg) continue else: raise SALTSpecError(e) if w_arr is None: w_arr = sr.findsol(xarr, soldict, istart, 'rss', nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slit, xbin, ybin, slitid, function, order) # for each line in the data, determine the wavelength solution # for a given line in the image wavemap = np.zeros_like(hdu[i].data) for j in range(len(hdu[i].data)): # find the wavelength solution for the data w_arr = sr.findsol(xarr, soldict, j, caltype, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slit, xbin, ybin, slitid, function, order) if w_arr is not None: wavemap[j, :] = w_arr if array_only: return wavemap # write out the oimg hduwav = fits.ImageHDU(data=wavemap, header=hdu[i].header, name='WAV') hdu.append(hduwav) saltkey.new('WAVEXT', len(hdu) - 1, 'Extension for Wavelength Map', hdu[i]) return hdu
def gain(struct,mult=True,usedb=False, dblist=None, ampccd=2, log=None, verbose=True): """gain processes a image hduList and gain corrects each amplifier. It can either use gain settings in the header or those supplied in a config file which would be suppleid in the dblist (see helpfile for structure of the config file). If variance frames exist, it will update those for changes in the header value as well. In the end, it will update the gain with a value of one signfing the data has been transformed into e- from ADU The program will look for the non-linear gain settings which are given by: e = GAIN*(1 + GAIN1*E-6*ADU)*ADU mult--if true, multiple the gains usedb--use the values in the dblist, if false use the header values dblist--values for the gain and readnoise from the ampccd--number of amplifiers per ccd dblist should have the following lists: speed, rate, gain, noise, bias, amp """ #get the infile name infile=saltkey.getimagename(struct[0]) #how many science extensions nsciext = saltkey.get('NSCIEXT',struct[0]) #how many data extensions nextend = saltkey.get('NSCIEXT',struct[0]) # how many amplifiers? amplifiers = ampccd*saltkey.get('NCCDS',struct[0]) #read the gain and rospeed for the image gainset = saltkey.get('GAINSET',struct[0]) rospeed = saltkey.get('ROSPEED',struct[0]) #loop through each amplifier and gain correct it if log: message = '%28s %6s %5s %3s %5s %5s' \ % ('HDU','GAIN','SPEED','AMP','GAIN','NOISE') log.message('\n ---------------------------------------------------', \ with_header=False, with_stdout=verbose) log.message(message, with_header=False, with_stdout=verbose) log.message(' ---------------------------------------------------', \ with_header=False, with_stdout=verbose) for i in range(nsciext): hdu = i + 1 amp = i%amplifiers+1 #get the gain and rdnoise values for the array if usedb: gain, rdnoise=get_values(dblist, gainset, rospeed, amp) gain1=0 else: gain = saltkey.get('GAIN',struct[hdu]) rdnoise = saltkey.get('RDNOISE',struct[hdu]) try: gain1=saltkey.get('GAIN1',struct[hdu]) except: gain1=0 if mult: #correct the gain gainmult=1 try: data=struct[hdu].data struct[hdu].data=gain*data+gain1*data**2 except Exception as e: msg='Cannot gain correct %s[%i] because %s' % (infile, hdu, e) raise SaltError(msg) #correct the variance frame if saltkey.found('VAREXT', struct[hdu]): vhdu=saltkey.get('VAREXT', struct[hdu]) try: vdata=struct[vhdu].data struct[vhdu].data=vdata*gain*(1+2*gain1*1e-6*data) except Exception as e: msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e) raise SaltError(msg) else: gainmult=gain #update the headers if usedb: saltkey.put('GAIN',gain,struct[hdu]) saltkey.put('RDNOISE',rdnoise,struct[hdu]) #add a keyword indicating what action was taken saltkey.new('GAINMULT',gainmult,'Gain multiplication', struct[hdu]) #if logging is true, then print out the following information if log: message = '%25s[%1d] %6s %5s %2s %6.2f %5.2f' \ % (infile,hdu,gainset,rospeed,amp, gain, rdnoise) log.message(message, with_header=False, with_stdout=verbose) #just to make it look pretty if log: log.message('', with_header=False, with_stdout=verbose) return struct
def addWCS(struct, xbin, ybin, outfile): """WCS keywords""" saltsafekey.new('CRPIX1',1,'WCS: X reference pixel',struct,outfile) saltsafekey.new('CRPIX2',1,'WCS: Y reference pixel',struct,outfile) saltsafekey.new('CRVAL1',float(xbin), 'WCS: X reference coordinate value',struct,outfile) saltsafekey.new('CRVAL2',float(ybin), 'WCS: Y reference coordinate value',struct,outfile) saltsafekey.new('CDELT1',float(xbin),'WCS: X pixel size',struct,outfile) saltsafekey.new('CDELT2',float(ybin),'WCS: Y pixel size',struct,outfile) saltsafekey.new('CTYPE1','pixel','X type',struct,outfile) saltsafekey.new('CTYPE2','pixel','Y type',struct,outfile) return struct
def imcombine(infiles, method='average', reject=None, mask=True, weight=True, \ blank=0, scale=None, statsec=None, lthresh=3, hthresh=3): """Fast combine is for special circumstances where the files and the combination can be down one by one and not all of the data need to be read in all at once. It computes the average of the files input in infile Input Variables: infiles: List of files to combine method: Combination method to use, either average or median reject: Reject outliers using different methods -ccdclip--reject the pixels according to the CCD parameters -sigclip--reject the pixels according to a sigma clipping mask: Whether to correct the data using the BPM frame weight: If true, weight by the inverse of the variant frame blank: Value if zero pixels are combined scale: Method to scale the images by statsec: Region for determining statistics if images are to be scaled lthresh: low rejection threshold hthresh: high rejection threshold Return Variables: hdustruct: Output struct """ hdu_list=[] #read in all of the data for infile in infiles: #hdu_list.append(saltio.openfits(infile)) hdu_list.append(pyfits.open(infile, memmap=True)) #Copy the first image and create the HDU for the other images try: outhdu=hdu_list[0] except Exception as e: message='Cannot create output hduList because %s' % e raise SaltError(message) #determine the size of a single extension. Assumes all arrays are the same size hdu=hdu_list[0] #For each science extension, construct the array of arrays to go into that #Three arrays need to be constructed including one for the data array, #one for the inv variance array, and one for the BPM #This assumes all the data are the same shape and size nimages=len(hdu_list) mem_lim=0.2 if reject or weight or mask: mem_lim=mem_lim/4.0 for i in range(len(outhdu)): if outhdu[i].name=='SCI': nb=outhdu[i].data.nbytes outhdu[i].data *= 0.0 #let's set the limit at 0.2 GB of memory if nb*nimages/1e9>mem_lim: imod=int((nb*nimages/1e9)/mem_lim)+1 y2,x2=outhdu[i].data.shape ystep=y2/imod xstep=x2/imod for j in range(imod): for k in range(imod): datasec=[j*ystep, min(y2,(j+1)*ystep), k*xstep, min(x2, (k+1)*xstep)] outhdu=hducombine(hdu_list, outhdu, ext=i, method=method, datasec=datasec, reject=reject, mask=mask,weight=weight, scale=scale, statsec=statsec, blank=blank, lthresh=lthresh, hthresh=hthresh) #this step is needed to remove data arrays that are read into memory #--not sure what object they are linked to, but this works try: for j in range(nimages): del hdu_list[j][i].data except: pass else: outhdu=hducombine(hdu_list, outhdu, ext=i, method=method, datasec=None, reject=reject, mask=mask,weight=weight, scale=scale, statsec=statsec, lthresh=lthresh, hthresh=hthresh) #Add any header frames saltkey.new('NCOMBINE',nimages,'Number of images combines', outhdu[0]) return outhdu
try: vdata=struct[vhdu].data struct[vhdu].data=vdata*gain*(1+2*gain1*1e-6*data) except Exception, e: msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e) raise SaltError(msg) else: gainmult=gain #update the headers if usedb: saltkey.put('GAIN',gain,struct[hdu]) saltkey.put('RDNOISE',rdnoise,struct[hdu]) #add a keyword indicating what action was taken saltkey.new('GAINMULT',gainmult,'Gain multiplication', struct[hdu]) #if logging is true, then print out the following information if log: message = '%25s[%1d] %6s %5s %2s %6.2f %5.2f' \ % (infile,hdu,gainset,rospeed,amp, gain, rdnoise) log.message(message, with_header=False, with_stdout=verbose) #just to make it look pretty if log: log.message('', with_header=False, with_stdout=verbose) return struct def get_values(dblist, gainset, rospeed, amp): """Get values for gain and rdnoise from the dblist. The input of the dblist should be:
nextend+=nsciext #create the badpixelframes for i in range(1, nsciext+1): try: hdu=createbadpixel(struct, badpixelstruct, i, nextend+i) except Exception, e: msg='Could not create bad pixel extension in ext %i of %s because %s' % (nextend+i, infile, e) raise SaltError(msg) struct[i].header.update('BPMEXT',nextend+i, comment='Extension for Bad Pixel Mask') struct.append(hdu) nextend+=nsciext #update the number of extensions saltkey.new('NSCIEXT',nsciext,'Number of science extensions', struct[0]) saltkey.new('NEXTEND',nextend,'Number of data extensions', struct[0]) return struct def CreateVariance(inhdu, sci_ext, var_ext): """Create a variance hdu from an input hdu""" rdnoise=inhdu.header['RDNOISE'] gain=inhdu.header['GAIN'] #create the variance array data=inhdu.data.copy() if (data <=0).any(): j=np.where(data>0) min_pos=data[j].min() j=np.where(data<=0)
def salteditkey(images,outimages,outpref, keyfile, recfile=None,clobber=False,logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') #verify that the input and output lists are the same length saltio.comparelists(infiles,outfiles,'Input','output') #is key file defined saltio.argdefined('keyfile',keyfile) keyfile = keyfile.strip() saltio.fileexists(keyfile) # if the data are the same, set up to use update instead of write openmode='copyonwrite' if (infiles!=outfiles): openmode='copyonwrite' # determine the date of the observations obsdate=saltstring.makeobsdatestr(infiles, 1,9) if len(obsdate)!=8: message = 'Either FITS files from multiple dates exist, ' message += 'or raw FITS files exist with non-standard names.' log.warning(message) # FITS file columns to record keyword changes fitcol = [] keycol = [] oldcol = [] newcol = [] # Set up the rules to change the files keyedits=readkeyfile(keyfile, log=log, verbose=verbose) #now step through the images for img, oimg in zip(infiles, outfiles): #determine the appropriate keyword edits for the image klist=[] for frange in keyedits: if checkfitsfile(img, frange, keyedits[frange]): klist.append(keyedits[frange][3]) if klist: #open up the new files struct = saltio.openfits(img,mode=openmode) struct.verify('fix') for kdict in klist: for keyword in kdict: #record the changes value=kdict[keyword] fitcol.append(img) keycol.append(keyword) newcol.append(value) try: oldcol.append(struct[0].header[keyword].lstrip()) except: oldcol.append('None') #update the keyword if saltkey.found(keyword, struct[0]): try: saltkey.put(keyword,value,struct[0]) message='\tUpdating %s in %s to %s' % (keyword, os.path.basename(img), value) log.message(message, with_header=False, with_stdout=verbose) except Exception, e: message = 'Could not update %s in %s because %s' % (keyword, img, str(e)) raise SaltError(message) else: try: saltkey.new(keyword.strip(),value,'Added Comment',struct[0]) message='\tAdding %s in %s to %s' % (keyword, os.path.basename(img), value) log.message(message, with_header=False, with_stdout=verbose) except Exception,e : message = 'Could not update %s in %s because %s' % (keyword, img, str(e)) raise SaltError(message) #updat the history keywords #fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) #saltkey.housekeeping(struct[0],'SAL-EDT', 'Keywords updated by SALTEDITKEY', hist) #write the file out if openmode=='update': saltio.updatefits(struct) message = 'Updated file ' + os.path.basename(oimg) else: saltio.writefits(struct, oimg, clobber) message = 'Created file ' + os.path.basename(oimg) log.message(message, with_header=False, with_stdout=True) struct.close()
def rectify(hdu, soldict, caltype='line', function='poly', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0, pixscale=0.0, time_interp=False, conserve=False, nearest=False, clobber=True, log=None, verbose=True): """Read in an image and a set of wavlength solutions. Calculate the best wavelength solution for a given dataset and then apply that data set to the image return """ # set the basic values set_w1 = (w1 is None) set_w2 = (w2 is None) set_dw = (dw is None) set_nw = (nw is None) # set up the time of the observation dateobs = saltkey.get('DATE-OBS', hdu[0]) utctime = saltkey.get('TIME-OBS', hdu[0]) exptime = saltkey.get('EXPTIME', hdu[0]) instrume = saltkey.get('INSTRUME', hdu[0]).strip() grating = saltkey.get('GRATING', hdu[0]).strip() if caltype == 'line': grang = saltkey.get('GRTILT', hdu[0]) arang = saltkey.get('CAMANG', hdu[0]) else: grang = saltkey.get('GR-ANGLE', hdu[0]) arang = saltkey.get('AR-ANGLE', hdu[0]) filtername = saltkey.get('FILTER', hdu[0]).strip() slitname = saltkey.get('MASKID', hdu[0]) slit = st.getslitsize(slitname) xbin, ybin = saltkey.ccdbin(hdu[0]) timeobs = enterdatetime('%s %s' % (dateobs, utctime)) # check to see if there is more than one solution if caltype == 'line': if len(soldict) == 1: sol = soldict.keys()[0] slitid = None if not matchobservations( soldict[sol], instrume, grating, grang, arang, filtername, slitid): msg = 'Observations do not match setup for transformation but using the solution anyway' if log: log.warning(msg) for i in range(1, len(hdu)): if hdu[i].name == 'SCI': if log: log.message('Correcting extension %i' % i) istart = int(0.5 * len(hdu[i].data)) # open up the data # set up the xarr and initial wavlength solution xarr = np.arange(len(hdu[i].data[istart]), dtype='int64') # get the slitid try: slitid = saltkey.get('SLITNAME', hdu[i]) except: slitid = None # set up a wavelength solution try: w_arr = findsol(xarr, soldict, istart, caltype, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slit, xbin, ybin, slitid, function, order) except SALTSpecError as e: if slitid: msg = 'SLITID %s: %s' % (slitid, e) if log: log.warning(msg) continue else: raise SALTSpecError(e) if w_arr is None: w_arr = findsol(xarr, soldict, istart, 'rss', nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slit, xbin, ybin, slitid, function, order) # set up the output x-axis if set_w1: w1 = w_arr.min() if set_w2: w2 = w_arr.max() if set_nw: nw = len(xarr) if set_dw: dw = float(w2 - w1) / nw nw_arr = createoutputxaxis(w1, w2, nw) # setup the VARIANCE and BPM frames if saltkey.found('VAREXT', hdu[i]): varext = saltkey.get('VAREXT', hdu[i]) else: varext = None # setup the BPM frames if saltkey.found('BPMEXT', hdu[i]): bpmext = saltkey.get('BPMEXT', hdu[i]) else: bpmext = None # for each line in the data, determine the wavelength solution # for a given line in the image for j in range(len(hdu[i].data)): # find the wavelength solution for the data w_arr = findsol(xarr, soldict, j, caltype, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername, slit, xbin, ybin, slitid, function, order) # apply that wavelength solution to the data if w_arr is not None: try: hdu[i].data[ j, :] = st.interpolate( nw_arr, w_arr, hdu[i].data[ j, :], inttype, left=blank, right=blank) except Exception as e: hdu[i].data[j, :] = hdu[i].data[j, :] * 0.0 + blank msg = 'In row %i, solution cannot be found due to %s' % ( i, e) # correct the variance frame if varext: try: hdu[varext].data[ j, :] = st.interpolate( nw_arr, w_arr, hdu[varext].data[ j, :], inttype, left=blank, right=blank) except Exception as e: msg = 'In row %i, solution cannot be found due to %s' % ( i, e) # correct the BPM frame if bpmext: try: hdu[bpmext].data[ j, :] = st.interpolate( nw_arr, w_arr, hdu[bpmext].data[ j, :], inttype, left=blank, right=blank) except Exception as e: msg = 'In row %i, solution cannot be found due to %s' % ( i, e) else: hdu[i].data[j, :] = hdu[i].data[j, :] * 0.0 + blank if conserve: hdu[i].data = hdu[i].data / dw if varext: hdu[varext].data = hdu[varext].data / dw # Add WCS information saltkey.new('CTYPE1', 'LAMBDA', 'Coordinate Type', hdu[i]) saltkey.new('CTYPE2', 'PIXEL', 'Coordinate Type', hdu[i]) saltkey.new( 'CD1_1', dw, 'WCS: Wavelength Dispersion in angstrom/pixel', hdu[i]) saltkey.new('CD2_1', 0.0, 'WCS: ', hdu[i]) saltkey.new('CD1_2', 0.0, 'WCS: ', hdu[i]) saltkey.new('CD2_2', ybin * pixscale, 'WCS: ', hdu[i]) saltkey.new('CRPIX1', 0.0, 'WCS: X Reference pixel', hdu[i]) saltkey.new('CRPIX2', 0.0, 'WCS: Y Reference pixel', hdu[i]) saltkey.new('CRVAL1', w1, 'WCS: X Reference pixel', hdu[i]) saltkey.new('CRVAL2', 0.0, 'WCS: Y Reference pixel', hdu[i]) saltkey.new('CDELT1', 1.0, 'WCS: X pixel size', hdu[i]) saltkey.new('CDELT2', 1.0, 'WCS: Y pixel size', hdu[i]) saltkey.new('DC-FLAG', 0, 'Dispesion Corrected image', hdu[i]) return hdu
def addWCS(struct, xbin, ybin, outfile): """WCS keywords""" saltsafekey.new('CRPIX1', 1, 'WCS: X reference pixel', struct, outfile) saltsafekey.new('CRPIX2', 1, 'WCS: Y reference pixel', struct, outfile) saltsafekey.new('CRVAL1', float(xbin), 'WCS: X reference coordinate value', struct, outfile) saltsafekey.new('CRVAL2', float(ybin), 'WCS: Y reference coordinate value', struct, outfile) saltsafekey.new('CDELT1', float(xbin), 'WCS: X pixel size', struct, outfile) saltsafekey.new('CDELT2', float(ybin), 'WCS: Y pixel size', struct, outfile) saltsafekey.new('CTYPE1', 'pixel', 'X type', struct, outfile) saltsafekey.new('CTYPE2', 'pixel', 'Y type', struct, outfile) return struct
newutc=salttime.dec2sex(ntime) except Exception,e: msg='Could not update UTC in %i header of image %s because %s' % (ext, infile, e) raise SaltError(msg) return struct # update the headers if utc==saltsafekey.get('UTC-OBS', struct): expt_string='%5.4f' % real_expt td_string='%5.4f' % tdiff if not saltsafekey.found('DUTC', struct): try: saltsafekey.put('UTC-OBS', newutc, struct, infile) saltsafekey.put('TIME-OBS', newutc, struct, infile) saltsafekey.new('DWETIME', expt_string, 'Dwell Time', struct, infile) saltsafekey.new('DUTC', td_string, 'Change in UTC time', struct, infile) except Exception, e: msg='Could not update %i header of image %s because %s' % (ext, infile, e) raise SaltIOError(msg) else: try: saltsafekey.put('UTC-OBS', newutc, struct, infile) saltsafekey.put('TIME-OBS', newutc, struct, infile) saltsafekey.put('DWETIME', real_expt, struct, infile) saltsafekey.put('DUTC', tdiff, struct, infile) except Exception, e: msg='Could not update %i header of image %s because %s' % (ext, infile, e) raise SaltError(msg) else: raise SaltIOError('Frame missing from list of times')
def bias(struct, subover=True, trim=True, subbias=False, bstruct=None, median=False, function='polynomial', order=3, rej_lo=3, rej_hi=3, niter=10, plotover=False, log=None, verbose=True): """Bias subtracts the bias levels from a frame. It will fit and subtract the overscan region, trim the images, and subtract a master bias if required. struct--image structure subover--subtract the overscan region trim--trim the image subbias--subtract master bias bstruct--master bias image structure median--use the median instead of mean in image statistics function--form to fit to the overscan region order--order for the function rej_lo--sigma of low points to reject in the fit rej_hi--sigma of high points to reject in the fit niter--number of iterations log--saltio log for recording information verbose--whether to print to stdout """ infile = saltkey.getimagename(struct[0]) # how many extensions? nsciext = saltkey.get('NSCIEXT', struct[0]) nextend = saltkey.get('NEXTEND', struct[0]) nccd = saltkey.get('NCCDS', struct[0]) # how many amplifiers?--this is hard wired amplifiers = 2 * nccd #log the process if subover and log: message = '%28s %7s %5s %4s %6s' % \ ('HDU','Overscan','Order','RMS','Niter') log.message( '\n --------------------------------------------------', with_header=False, with_stdout=verbose) log.message(message, with_header=False, with_stdout=verbose) log.message(' --------------------------------------------------', with_header=False, with_stdout=verbose) if (plotover): plt.figure(1) plt.axes([0.1, 0.1, 0.8, 0.8]) plt.xlabel('CCD Column') plt.ylabel('Pixel Counts (e-)') plt.ion() #loop through the extensions and subtract the bias for i in range(1, nsciext + 1): if struct[i].name == 'SCI': #get the bias section biassec = saltkey.get('BIASSEC', struct[i]) y1, y2, x1, x2 = saltio.getSection(biassec, iraf_format=True) #get the data section datasec = saltkey.get('DATASEC', struct[i]) dy1, dy2, dx1, dx2 = saltio.getSection(datasec, iraf_format=True) #setup the overscan region if subover: yarr = np.arange(y1, y2, dtype=float) data = struct[i].data odata = struct[i].data[y1:y2, x1:x2] if median: odata = np.median((struct[i].data[y1:y2, x1:x2]), axis=1) olevel = np.median((struct[i].data[y1:y2, x1:x2])) saltkey.new('OVERSCAN', '%f' % (olevel), 'Overscan median value', struct[i]) else: odata = np.mean((struct[i].data[y1:y2, x1:x2]), axis=1) olevel = np.mean((struct[i].data[y1:y2, x1:x2])) saltkey.new('OVERSCAN', '%f' % (olevel), 'Overscan mean value', struct[i]) #fit the overscan region ifit=saltfit.interfit(yarr, odata, function=function, \ order=order, thresh=rej_hi, niter=niter) try: ifit.interfit() coeffs = ifit.coef ofit = ifit(yarr) omean, omed, osigma = saltstat.iterstat((odata - ofit), sig=3, niter=5) except ValueError: #catch the error if it is a zero array ofit = np.array(yarr) * 0.0 osigma = 0.0 except TypeError: #catch the error if it is a zero array ofit = np.array(yarr) * 0.0 osigma = 0.0 #if it hasn't been already, convert image to #double format struct[i].data = 1.0 * struct[i].data try: struct[i].header.remove('BZERO') struct[i].header.remove('BSCALE') except: pass #subtract the overscan region for j in range(len(struct[i].data[0])): struct[i].data[y1:y2, j] -= ofit #report the information if log: message = '%25s[%1d] %8.2f %3d %7.2f %3d' % \ (infile, i, olevel, order, osigma, niter) log.message(message, with_stdout=verbose, with_header=False) #add the statistics to the image header saltkey.new('OVERRMS', '%f' % (osigma), 'Overscan RMS value', struct[i]) #update the variance frame if saltkey.found('VAREXT', struct[i]): vhdu = saltkey.get('VAREXT', struct[i]) try: vdata = struct[vhdu].data #The bias level should not be included in the noise from the signal for j in range(len(struct[i].data[0])): vdata[y1:y2, j] -= ofit #add a bit to make sure that the minimum error is the rednoise rdnoise = saltkey.get('RDNOISE', struct[i]) vdata[vdata < rdnoise**2] = rdnoise**2 struct[vhdu].data = vdata + osigma**2 except Exception, e: msg = 'Cannot update the variance frame in %s[%i] because %s' % ( infile, vhdu, e) raise SaltError(msg) #plot the overscan region if plotover: plt.plot(yarr, odata) plt.plot(yarr, ofit) #trim the data and update the headers if trim: struct[i].data = struct[i].data[dy1:dy2, dx1:dx2] datasec = '[1:' + str(dx2 - dx1) + ',1:' + str(dy2 - dy1) + ']' saltkey.put('DATASEC', datasec, struct[i]) #update the variance frame if saltkey.found('VAREXT', struct[i]): vhdu = saltkey.get('VAREXT', struct[i]) struct[vhdu].data = struct[vhdu].data[dy1:dy2, dx1:dx2] datasec = '[1:' + str(dx2 - dx1) + ',1:' + str(dy2 - dy1) + ']' saltkey.put('DATASEC', datasec, struct[vhdu]) #update the BPM frame if saltkey.found('BPMEXT', struct[i]): bhdu = saltkey.get('BPMEXT', struct[i]) struct[bhdu].data = struct[bhdu].data[dy1:dy2, dx1:dx2] datasec = '[1:' + str(dx2 - dx1) + ',1:' + str(dy2 - dy1) + ']' saltkey.put('DATASEC', datasec, struct[bhdu]) #subtract the master bias if necessary if subbias and bstruct: struct[i].data -= bstruct[i].data #update the variance frame if saltkey.found('VAREXT', struct[i]): vhdu = saltkey.get('VAREXT', struct[i]) try: vdata = struct[vhdu].data struct[vhdu].data = vdata + bstruct[vhdu].data except Exception, e: msg = 'Cannot update the variance frame in %s[%i] because %s' % ( infile, vhdu, e) raise SaltError(msg)
reject=reject, mask=mask,weight=weight, scale=scale, statsec=statsec, blank=blank, lthresh=lthresh, hthresh=hthresh) #this step is needed to remove data arrays that are read into memory #--not sure what object they are linked to, but this works try: for j in range(nimages): del hdu_list[j][i].data except: pass else: outhdu=hducombine(hdu_list, outhdu, ext=i, method=method, datasec=None, reject=reject, mask=mask,weight=weight, scale=scale, statsec=statsec, lthresh=lthresh, hthresh=hthresh) #Add any header frames saltkey.new('NCOMBINE',nimages,'Number of images combines', outhdu[0]) return outhdu def hducombine(hdu_list, outhdu, ext, method='average', datasec=None, reject=None, mask=False, weight=False, scale=None, statsec=None, blank=0, lthresh=3, hthresh=3): """Combine a set of images in imlist """ #set up i as the extionsion variable as shortcut i=ext nimages=len(hdu_list) #set all the data arrays data_list=[] gain=np.zeros(nimages) rdnoise=np.zeros(nimages)
hdu[varext].data[j,:]=st.interpolate(nw_arr, w_arr, hdu[varext].data[j,:], inttype, left=blank, right=blank) except Exception, e: msg='In row %i, solution cannot be found due to %s' % (i, e) #correct the BPM frame if bpmext: try: hdu[bpmext].data[j,:]=st.interpolate(nw_arr, w_arr, hdu[bpmext].data[j,:], inttype, left=blank, right=blank) except Exception, e: msg='In row %i, solution cannot be found due to %s' % (i, e) else: hdu[i].data[j,:]=hdu[i].data[j,:]*0.0+blank #Add WCS information saltkey.new('CTYPE1', 'LAMBDA', 'Coordinate Type', hdu[i]) saltkey.new('CTYPE2', 'PIXEL', 'Coordinate Type', hdu[i]) saltkey.new('CD1_1', dw, 'WCS: Wavelength Dispersion in angstrom/pixel', hdu[i]) saltkey.new('CD2_1', 0.0, 'WCS: ', hdu[i]) saltkey.new('CD1_2', 0.0, 'WCS: ', hdu[i]) saltkey.new('CD2_2', ybin*pixscale, 'WCS: ', hdu[i]) saltkey.new('CRPIX1', 0.0, 'WCS: X Reference pixel', hdu[i]) saltkey.new('CRPIX2', 0.0, 'WCS: Y Reference pixel', hdu[i]) saltkey.new('CRVAL1', w1, 'WCS: X Reference pixel', hdu[i]) saltkey.new('CRVAL2', 0.0, 'WCS: Y Reference pixel', hdu[i]) saltkey.new('CDELT1', 1.0, 'WCS: X pixel size', hdu[i]) saltkey.new('CDELT2', 1.0, 'WCS: Y pixel size', hdu[i]) return hdu def makeinterpsolution(xarr, soldict, timeobs, exptime, instrume, grating, grang, arang, filtername, slitid):
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None, subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log',verbose=True): """Convert MEF HRS data into a single image. If variance frames and BPMs, then convert them to the same format as well. Returns an MEF image but that is combined into a single frame """ with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outpath=saltio.abspath(outpath) if saltio.checkfornone(obslogfile) is None: raise SaltError('Obslog file is required') # Delete the obslog file if it already exists if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile): if os.path.isfile(obslogfile): saltio.delete(obslogfile) #read in the obsveration log or create it headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) else: obsstruct=saltio.openfits(obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') ccdtype=obsstruct.data.field('OBJECT') biaslist=filename[ccdtype=='Bias'] masterbias_dict={} if log: log.message('Processing Bias Frames') for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=pyfits.open(img) bimg=outpath+'bgph'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose, with_header=False) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=False, imstack=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list) #create the master bias frame for i in list(masterbias_dict.keys()): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys, x1=5, x2=13) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #apply full reductions to the science data for img in infiles: nimg=os.path.basename(img) if not nimg in biaslist: #open the image struct=pyfits.open(img) simg=outpath+'mbgph'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #get master bias frame masterbias=get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list) if masterbias: subbias=True bstruct=saltio.openfits(masterbias) else: subbias=False bstruct=None #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=subbias, imstack=True, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) return
def make_mosaic(struct, gap, xshift, yshift, rotation, interp_type='linear', boundary='constant', constant=0, geotran=True, fill=False, cleanup=True, log=None, verbose=False): """Given a SALT image struct, combine each of the individual amplifiers and apply the geometric CCD transformations to the image """ # get the name of the file infile = saltkey.getimagename(struct[0], base=True) outpath = './' # identify instrument instrume, keyprep, keygain, keybias, keyxtalk, keyslot = \ saltkey.instrumid(struct) # how many amplifiers? nsciext = saltkey.get('NSCIEXT', struct[0]) nextend = saltkey.get('NEXTEND', struct[0]) nccds = saltkey.get('NCCDS', struct[0]) amplifiers = nccds * 2 if nextend > nsciext: varframe = True else: varframe = False # CCD geometry coefficients if (instrume == 'RSS' or instrume == 'PFIS'): xsh = [0., xshift[0], 0., xshift[1]] ysh = [0., yshift[0], 0., yshift[1]] rot = [0., rotation[0], 0., rotation[1]] elif instrume == 'SALTICAM': xsh = [0., xshift[0], 0.] ysh = [0., yshift[0], 0.] rot = [0., rotation[0], 0] # how many extensions? nextend = saltkey.get('NEXTEND', struct[0]) # CCD on-chip binning xbin, ybin = saltkey.ccdbin(struct[0]) # create temporary primary extension outstruct = [] outstruct.append(struct[0]) # define temporary FITS file store tiled CCDs tilefile = saltio.tmpfile(outpath) tilefile += 'tile.fits' if varframe: tilehdu = [None] * (3 * int(nsciext / 2) + 1) else: tilehdu = [None] * int(nsciext / 2 + 1) tilehdu[0] = fits.PrimaryHDU() #tilehdu[0].header = struct[0].header if log: log.message('', with_stdout=verbose) # iterate over amplifiers, stich them to produce file of CCD images for i in range(int(nsciext / 2)): hdu = i * 2 + 1 # amplifier = hdu%amplifiers # if (amplifier == 0): amplifier = amplifiers # read DATASEC keywords datasec1 = saltkey.get('DATASEC', struct[hdu]) datasec2 = saltkey.get('DATASEC', struct[hdu + 1]) xdsec1, ydsec1 = saltstring.secsplit(datasec1) xdsec2, ydsec2 = saltstring.secsplit(datasec2) # read images imdata1 = saltio.readimage(struct, hdu) imdata2 = saltio.readimage(struct, hdu + 1) # tile 2n amplifiers to yield n CCD images outdata = numpy.zeros((ydsec1[1] + abs(ysh[i + 1] / ybin), xdsec1[1] + xdsec2[1] + abs(xsh[i + 1] / xbin)), numpy.float32) # set up the variance frame if varframe: vardata = outdata.copy() vdata1 = saltio.readimage(struct, struct[hdu].header['VAREXT']) vdata2 = saltio.readimage(struct, struct[hdu + 1].header['VAREXT']) bpmdata = outdata.copy() bdata1 = saltio.readimage(struct, struct[hdu].header['BPMEXT']) bdata2 = saltio.readimage(struct, struct[hdu + 1].header['BPMEXT']) x1 = xdsec1[0] - 1 if x1 != 0: msg = 'The data in %s have not been trimmed prior to mosaicking.' \ % infile log.error(msg) if xsh[i + 1] < 0: x1 += abs(xsh[i + 1] / xbin) x2 = x1 + xdsec1[1] y1 = ydsec1[0] - 1 if ysh[i + 1] < 0: y1 += abs(ysh[i + 1] / ybin) y2 = y1 + ydsec1[1] outdata[y1:y2, x1:x2] =\ imdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] if varframe: vardata[y1:y2, x1:x2] =\ vdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] bpmdata[y1:y2, x1:x2] =\ bdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] x1 = x2 x2 = x1 + xdsec2[1] y1 = ydsec2[0] - 1 if ysh[i + 1] < 0: y1 += abs(ysh[i + 1] / ybin) y2 = y1 + ydsec2[1] outdata[y1:y2, x1:x2] =\ imdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] if varframe: vardata[y1:y2, x1:x2] =\ vdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] bpmdata[y1:y2, x1:x2] =\ bdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] # size of new image naxis1 = str(xdsec1[1] + xdsec2[1]) naxis2 = str(ydsec1[1]) # add image and keywords to HDU list tilehdu[i + 1] = fits.ImageHDU(outdata) tilehdu[i + 1].header = struct[hdu].header #tilehdu[ # i + 1].header['DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' if varframe: vext = i + 1 + int(nsciext / 2.) tilehdu[vext] = fits.ImageHDU(vardata) #tilehdu[vext].header = struct[struct[hdu].header['VAREXT']].header #tilehdu[vext].header[ # 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' bext = i + 1 + 2 * int(nsciext / 2.) tilehdu[bext] = fits.ImageHDU(bpmdata) #tilehdu[bext].header = struct[struct[hdu].header['BPMEXT']].header #tilehdu[bext].header[ # 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' # image tile log message #1 if log: message = os.path.basename(infile) + '[' + str(hdu) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> ' message += os.path.basename(tilefile) + '[' + str(i + 1) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + ']' log.message(message, with_stdout=verbose, with_header=False) message = os.path.basename(infile) + '[' + str(hdu + 1) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> ' message += os.path.basename(tilefile) + '[' + str(i + 1) + '][' message += str(xdsec1[1] + 1) + ':' + \ str(xdsec1[1] + xdsec2[1]) + ',' message += str(ydsec2[0]) + ':' + str(ydsec2[1]) + ']' log.message(message, with_stdout=verbose, with_header=False) # write temporary file of tiled CCDs hdulist = fits.HDUList(tilehdu) hdulist.writeto(tilefile) # iterate over CCDs, transform and rotate images yrot = [None] * 4 xrot = [None] * 4 tranfile = [' '] tranhdu = [0] if varframe: tranfile = [''] * (3 * int(nsciext / 2) + 1) tranhdu = [0] * (3 * int(nsciext / 2) + 1) else: tranfile = [''] * int(nsciext / 2 + 1) tranhdu = [0] * int(nsciext / 2 + 1) # this is hardwired for SALT where the second CCD is considered the # fiducial for hdu in range(1, int(nsciext / 2 + 1)): tranfile[hdu] = saltio.tmpfile(outpath) tranfile[hdu] += 'tran.fits' if varframe: tranfile[hdu + nccds] = saltio.tmpfile(outpath) + 'tran.fits' tranfile[hdu + 2 * nccds] = saltio.tmpfile(outpath) + 'tran.fits' ccd = hdu % nccds if (ccd == 0): ccd = nccds # correct rotation for CCD binning yrot[ccd] = rot[ccd] * ybin / xbin xrot[ccd] = rot[ccd] * xbin / ybin dxshift = xbin * int(float(int(gap) / xbin) + 0.5) - gap # transformation using geotran IRAF task # if (ccd == 1): if (ccd != 2): if geotran: message = '\nSALTMOSAIC -- geotran ' + tilefile + \ '[' + str(ccd) + '] ' + tranfile[hdu] message += ' \"\" \"\" xshift=' + \ str((xsh[ccd] + (2 - ccd) * dxshift) / xbin) + ' ' message += 'yshift=' + \ str(ysh[ccd] / ybin) + ' xrotation=' + str(xrot[ccd]) + ' ' message += 'yrotation=' + \ str(yrot[ccd]) + ' xmag=1 ymag=1 xmin=\'INDEF\'' message += 'xmax=\'INDEF\' ymin=\'INDEF\' ymax=\'INDEF\' ' message += 'ncols=\'INDEF\' ' message += 'nlines=\'INDEF\' verbose=\'no\' ' message += 'fluxconserve=\'yes\' nxblock=2048 ' message += 'nyblock=2048 interpolant=\'' + \ interp_type + '\' boundary=\'constant\' constant=0' log.message(message, with_stdout=verbose) yd, xd = tilehdu[ccd].data.shape ncols = 'INDEF' # ncols=xd+abs(xsh[ccd]/xbin) nlines = 'INDEF' # nlines=yd+abs(ysh[ccd]/ybin) geo_xshift = xsh[ccd] + (2 - ccd) * dxshift / xbin geo_yshift = ysh[ccd] / ybin iraf.images.immatch.geotran(tilefile + "[" + str(ccd) + "]", tranfile[hdu], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) if varframe: var_infile = tilefile + "[" + str(ccd + nccds) + "]" iraf.images.immatch.geotran(var_infile, tranfile[hdu + nccds], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) var2_infile = tilefile + "[" + str(ccd + 2 * nccds) + "]" iraf.images.immatch.geotran(var2_infile, tranfile[hdu + 2 * nccds], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) # open the file and copy the data to tranhdu tstruct = fits.open(tranfile[hdu]) tranhdu[hdu] = tstruct[0].data tstruct.close() if varframe: tranhdu[ hdu + nccds] = fits.open( tranfile[ hdu + nccds])[0].data tranhdu[ hdu + 2 * nccds] = fits.open( tranfile[ hdu + 2 * nccds])[0].data else: log.message( "Transform CCD #%i using dx=%s, dy=%s, rot=%s" % (ccd, xsh[ccd] / 2.0, ysh[ccd] / 2.0, xrot[ccd]), with_stdout=verbose, with_header=False) tranhdu[hdu] = geometric_transform( tilehdu[ccd].data, tran_func, prefilter=False, order=1, extra_arguments=( xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) tstruct = fits.PrimaryHDU(tranhdu[hdu]) tstruct.writeto(tranfile[hdu]) if varframe: tranhdu[hdu + nccds] = geometric_transform( tilehdu[hdu + 3].data, tran_func, prefilter=False, order=1, extra_arguments=( xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) tranhdu[hdu + 2 * nccds] = geometric_transform( tilehdu[hdu + 6].data, tran_func, prefilter=False, order=1, extra_arguments=( xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) else: log.message( "Transform CCD #%i using dx=%s, dy=%s, rot=%s" % (ccd, 0, 0, 0), with_stdout=verbose, with_header=False) tranhdu[hdu] = tilehdu[ccd].data if varframe: tranhdu[hdu + nccds] = tilehdu[ccd + nccds].data tranhdu[hdu + 2 * nccds] = tilehdu[ccd + 2 * nccds].data # open outfile if varframe: outlist = 4 * [None] else: outlist = 2 * [None] #outlist[0] = struct[0].copy() outlist[0] = fits.PrimaryHDU() outlist[0].header = struct[0].header naxis1 = int(gap / xbin * (nccds - 1)) naxis2 = 0 for i in range(1, nccds + 1): yw, xw = tranhdu[i].shape naxis1 += xw + int(abs(xsh[ccd] / xbin)) + 1 naxis2 = max(naxis2, yw) outdata = numpy.zeros((naxis2, naxis1), numpy.float32) outdata.shape = naxis2, naxis1 if varframe: vardata = outdata * 0 bpmdata = outdata * 0 + 1 # iterate over CCDs, stich them to produce a full image hdu = 0 totxshift = 0 for hdu in range(1, nccds + 1): # read DATASEC keywords ydsec, xdsec = tranhdu[hdu].shape # define size and shape of final image # tile CCDs to yield mosaiced image x1 = int((hdu - 1) * (xdsec + gap / xbin)) + int(totxshift) x2 = xdsec + x1 y1 = int(0) y2 = int(ydsec) outdata[y1:y2, x1:x2] = tranhdu[hdu] totxshift += int(abs(xsh[hdu] / xbin)) + 1 if varframe: vardata[y1:y2, x1:x2] = tranhdu[hdu + nccds] bpmdata[y1:y2, x1:x2] = tranhdu[hdu + 2 * nccds] # make sure to cover up all the gaps include bad areas if varframe: baddata = (outdata == 0) baddata = nd.maximum_filter(baddata, size=3) bpmdata[baddata] = 1 # fill in the gaps if requested if fill: if varframe: outdata = fill_gaps(outdata, 0) else: outdata = fill_gaps(outdata, 0) # add to the file outlist[1] = fits.ImageHDU(outdata) if varframe: outlist[2] = fits.ImageHDU(vardata,name='VAR') outlist[3] = fits.ImageHDU(bpmdata,name='BPM') # create the image structure outstruct = fits.HDUList(outlist) # update the head informaation # housekeeping keywords saltkey.put('NEXTEND', 2, outstruct[0]) saltkey.new('EXTNAME', 'SCI', 'Extension name', outstruct[1]) saltkey.new('EXTVER', 1, 'Extension number', outstruct[1]) if varframe: saltkey.new('VAREXT', 2, 'Variance frame extension', outstruct[1]) saltkey.new('BPMEXT', 3, 'BPM Extension', outstruct[1]) try: saltkey.copy(struct[1], outstruct[1], 'CCDSUM') except: pass # Add keywords associated with geometry saltkey.new('SGEOMGAP', gap, 'SALT Chip Gap', outstruct[0]) c1str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[0], yshift[0], rotation[0]) saltkey.new('SGEOM1', c1str, 'SALT Chip 1 Transform', outstruct[0]) c2str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[1], yshift[1], rotation[1]) saltkey.new('SGEOM2', c2str, 'SALT Chip 2 Transform', outstruct[0]) # WCS keywords saltkey.new('CRPIX1', 0, 'WCS: X reference pixel', outstruct[1]) saltkey.new('CRPIX2', 0, 'WCS: Y reference pixel', outstruct[1]) saltkey.new( 'CRVAL1', float(xbin), 'WCS: X reference coordinate value', outstruct[1]) saltkey.new( 'CRVAL2', float(ybin), 'WCS: Y reference coordinate value', outstruct[1]) saltkey.new('CDELT1', float(xbin), 'WCS: X pixel size', outstruct[1]) saltkey.new('CDELT2', float(ybin), 'WCS: Y pixel size', outstruct[1]) saltkey.new('CTYPE1', 'pixel', 'X type', outstruct[1]) saltkey.new('CTYPE2', 'pixel', 'Y type', outstruct[1]) # cleanup temporary files if cleanup: for tfile in tranfile: if os.path.isfile(tfile): saltio.delete(tfile) if os.path.isfile(tilefile): status = saltio.delete(tilefile) # return the file return outstruct
struct[vhdu].data = vdata * gain * ( 1 + 2 * gain1 * 1e-6 * data) except Exception, e: msg = 'Cannot update the variance frame in %s[%i] because %s' % ( infile, vhdu, e) raise SaltError(msg) else: gainmult = gain #update the headers if usedb: saltkey.put('GAIN', gain, struct[hdu]) saltkey.put('RDNOISE', rdnoise, struct[hdu]) #add a keyword indicating what action was taken saltkey.new('GAINMULT', gainmult, 'Gain multiplication', struct[hdu]) #if logging is true, then print out the following information if log: message = '%25s[%1d] %6s %5s %2s %6.2f %5.2f' \ % (infile,hdu,gainset,rospeed,amp, gain, rdnoise) log.message(message, with_header=False, with_stdout=verbose) #just to make it look pretty if log: log.message('', with_header=False, with_stdout=verbose) return struct def get_values(dblist, gainset, rospeed, amp):
def saltclean(images, outpath, obslogfile=None, gaindb=None, xtalkfile=None, geomfile=None, subover=True, trim=True, masbias=None, subbias=False, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log', verbose=True): """SALTCLEAN will provide basic CCD reductions for a set of data. It will sort the data, and first process the biases, flats, and then the science frames. It will record basic quality control information about each of the steps. """ plotover = False #start logging with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outpath = saltio.abspath(outpath) #does the gain database file exist if gaindb: dblist = saltio.readgaindb(gaindb) else: dblist = [] # does crosstalk coefficient data exist if xtalkfile: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict = None #does the mosaic file exist--raise error if no saltio.fileexists(geomfile) # Delete the obslog file if it already exists if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile) #read in the obsveration log or create it if os.path.isfile(obslogfile): msg = 'The observing log already exists. Please either delete it or run saltclean with clobber=yes' raise SaltError(msg) else: headerDict = obslog(infiles, log) obsstruct = createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) #create the list of bias frames and process them filename = obsstruct.data.field('FILENAME') detmode = obsstruct.data.field('DETMODE') ccdtype = obsstruct.data.field('CCDTYPE') #set the bias list of objects biaslist = filename[ccdtype == 'ZERO'] masterbias_dict = {} for img in infiles: if os.path.basename(img) in biaslist: #open the image struct = fits.open(img) bimg = outpath + 'bxgp' + os.path.basename(img) #print the message if log: message = 'Processing Zero frame %s' % img log.message(message, with_stdout=verbose) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) saltkey.new('SXTALK', time.asctime(time.localtime()), 'Images have been xtalk corrected', struct[0]) saltkey.new('SBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict = compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys = masterbias_dict[i][0] blist = masterbias_dict[i][1:] mbiasname = outpath + createmasterbiasname(blist, bkeys) bfiles = ','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #create the list of flatfields and process them flatlist = filename[ccdtype == 'FLAT'] masterflat_dict = {} for img in infiles: if os.path.basename(img) in flatlist: #open the image struct = fits.open(img) fimg = outpath + 'bxgp' + os.path.basename(img) #print the message if log: message = 'Processing Flat frame %s' % img log.message(message, with_stdout=verbose) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) saltkey.new('SXTALK', time.asctime(time.localtime()), 'Images have been xtalk corrected', struct[0]) saltkey.new('SBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, fimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterflat_dict = compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list) #create the master flat frame for i in masterflat_dict.keys(): fkeys = masterflat_dict[i][0] flist = masterflat_dict[i][1:] mflatname = outpath + createmasterflatname(flist, fkeys) ffiles = ','.join(flist) saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #process the science data for img in infiles: nimg = os.path.basename(img) #print nimg, nimg in flatlist, nimg in biaslist if not (nimg in biaslist): #open the image struct = fits.open(img) simg = outpath + 'bxgp' + os.path.basename(img) #print the message if log: message = 'Processing science frame %s' % img log.message(message, with_stdout=verbose) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) saltkey.new('SXTALK', time.asctime(time.localtime()), 'Images have been xtalk corrected', struct[0]) saltkey.new('SBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, simg, clobber=clobber) saltio.closefits(struct) #mosaic the files--currently not in the proper format--will update when it is if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])): mimg = outpath + 'mbxgp' + os.path.basename(img) saltmosaic(images=simg, outimages=mimg, outpref='', geomfile=geomfile, interp=interp, cleanup=True, clobber=clobber, logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg)
else: outhdu = hducombine(hdu_list, outhdu, ext=i, method=method, datasec=None, reject=reject, mask=mask, weight=weight, scale=scale, statsec=statsec, lthresh=lthresh, hthresh=hthresh) #Add any header frames saltkey.new('NCOMBINE', nimages, 'Number of images combines', outhdu[0]) return outhdu def hducombine(hdu_list, outhdu, ext, method='average', datasec=None, reject=None, mask=False, weight=False, scale=None, statsec=None, blank=0,
def slot(struct,infile,dbspeed,dbrate,dbgain,dbnoise,dbbias,dbamp,xcoeff,gaindb,xtalkfile, logfile,verbose): import saltprint, saltkey, saltio, saltstat, time # identify instrument instrume,keyprep,keygain,keybias,keyxtalk,keyslot,status = saltkey.instrumid(struct,infile,logfile) # number of image HDU nextend = 0 while (status == 0): try: struct[nextend+1].header['XTENSION'] nextend += 1 except: break nccds,status = saltkey.get('NCCDS',struct[0],infile,logfile) amplifiers = nccds * 2 if (nextend%(amplifiers) != 0): message = '\nERROR -- SALTSLOT: Number of image extensions and' message += 'number of amplifiers are not consistent' status = saltprint.err(saltlog,message) status = saltkey.new('NSCIEXT',nextend,'Number of science extensions',struct[0],infile,logfile) status = saltkey.new('NEXTEND',nextend,'Number of data extensions',struct[0],infile,logfile) # check image file and gain database are compatible if (status == 0): ngains = len(dbgain) if (int(max(dbamp)) != amplifiers): message = '\nERROR -- SALTGSLOT: ' + infile + ' contains ' + str(amplifiers) + ' amplifiers' message += ', the gaindb file ' + gaindb + ' contains ' + str(max(dbamp)) + ' amplifiers' status = saltprint.err(logfile,message) # check image file and cross talk database are compatible if (status == 0): if (len(xcoeff)-1 != amplifiers): message = '\nERROR -- SALTSLOT: ' + infile + ' contains ' + str(amplifiers) + ' amplifiers' message += ', the cross talk file ' + xtalkfile + ' contains ' + str(len(xcoeff)-1) + ' amplifiers' status = saltprint.err(logfile,message) # housekeeping keywords if (status == 0): status = saltkey.put('SAL-TLM',time.asctime(time.localtime()),struct[0],infile,logfile) status = saltkey.new(keyslot,time.asctime(time.localtime()), 'Data have been cleaned by SALTSLOT',struct[0],infile,logfile) # keywords for image extensions for i in range(nextend): hdu = i + 1 status = saltkey.new('EXTNAME','SCI','Extension name',struct[hdu],infile,logfile) status = saltkey.new('EXTVER',hdu,'Extension number',struct[hdu],infile,logfile) # log coefficent table if (status == 0): message = '%30s %5s %4s %8s' % ('HDU','Gain','Bias','Xtalk') saltprint.log(logfile,'\n ---------------------------------------------',verbose) saltprint.log(logfile,message,verbose) saltprint.log(logfile,' ---------------------------------------------',verbose) # loop over image extensions if (status == 0): for i in range(nextend/2): hdu = i * 2 + 1 amplifier = hdu%amplifiers if (amplifier == 0): amplifier = amplifiers if (status == 0): value,status = saltkey.get('NAXIS1',struct[hdu],infile,logfile) naxis1 = int(value) if (status == 0): value,status = saltkey.get('NAXIS1',struct[hdu+1],infile,logfile) naxis2 = int(value) if (status == 0 and hdu == 1): biassec, status = saltkey.get('BIASSEC',struct[hdu],infile,logfile) if (status == 0 and hdu == 1): ranges = biassec.lstrip('[').rstrip(']').split(',') x1_1 = int(ranges[0].split(':')[0]) - 1 x2_1 = int(ranges[0].split(':')[1]) - 1 y1_1 = int(ranges[1].split(':')[0]) - 1 y2_1 = int(ranges[1].split(':')[1]) - 1 if (status == 0 and hdu == 1): biassec, status = saltkey.get('BIASSEC',struct[hdu+1],infile,logfile) if (status == 0 and hdu == 1): ranges = biassec.lstrip('[').rstrip(']').split(',') x1_2 = int(ranges[0].split(':')[0]) - 1 x2_2 = int(ranges[0].split(':')[1]) - 1 y1_2 = int(ranges[1].split(':')[0]) - 1 y2_2 = int(ranges[1].split(':')[1]) - 1 if (status == 0 and hdu == 1): datasec,status = saltkey.get('DATASEC',struct[hdu],infile,logfile) if (status == 0 and hdu == 1): ranges = datasec.lstrip('[').rstrip(']').split(',') dx1_1 = int(ranges[0].split(':')[0]) - 1 dx2_1 = int(ranges[0].split(':')[1]) dy1_1 = int(ranges[1].split(':')[0]) - 1 dy2_1 = int(ranges[1].split(':')[1]) if (status == 0 and hdu == 1): datasec,status = saltkey.get('DATASEC',struct[hdu+1],infile,logfile) if (status == 0 and hdu == 1): ranges = datasec.lstrip('[').rstrip(']').split(',') dx1_2 = int(ranges[0].split(':')[0]) - 1 dx2_2 = int(ranges[0].split(':')[1]) dy1_2 = int(ranges[1].split(':')[0]) - 1 dy2_2 = int(ranges[1].split(':')[1]) if (status == 0 and dx2_1 - dx1_1 != dx2_2 - dx1_2): message = 'ERROR -- SALTSLOT: HDUs '+infile message += '['+str(hdu)+'] and '+infile+'['+str(hdu+1)+']' message += ' have different dimensions' status = saltprint.err(logfile,message) # read speed and gain of each exposure if (status == 0 and hdu == 1): gainset,status = saltkey.get('GAINSET',struct[0],infile,logfile) rospeed,status = saltkey.get('ROSPEED',struct[0],infile,logfile) if (rospeed == 'NONE'): saltprint.log(logfile," ",verbose) message = "ERROR -- SALTSLOT: Readout speed is 'NONE' in " message += "primary keywords of " + infile status = saltprint.err(logfile,message) # read raw images if (status == 0): imagedata1,status = saltio.readimage(struct,hdu,logfile) imagedata2,status = saltio.readimage(struct,hdu+1,logfile) # gain correction if (status == 0): for j in range(len(dbgain)): if (gainset == dbrate[j] and rospeed == dbspeed[j] and amplifier == int(dbamp[j])): try: gain1 = float(dbgain[j]) imagedata1 *= gain1 except: mesage = 'ERROR -- SALTSLOT: Cannot perform gain correction on image ' message += infile+'['+str(hdu)+']' status = saltprint.err(logfile,message) elif (gainset == dbrate[j] and rospeed == dbspeed[j] and amplifier + 1 == int(dbamp[j])): try: gain2 = float(dbgain[j]) imagedata2 *= gain2 except: mesage = 'ERROR -- SALTSLOT: Cannot perform gain correction on image ' message += infile+'['+str(hdu+1)+']' status = saltprint.err(logfile,message) # crosstalk correction if (status == 0): revimage1 = imagedata1 * float(xcoeff[amplifier]) revimage2 = imagedata2 * float(xcoeff[amplifier+1]) for j in range(dx2_1-dx1_1+1): imagedata1[:,j] -= revimage2[:,dx2_2-j-1] imagedata2[:,j] -= revimage1[:,dx2_1-j-1] # bias subtraction if (status == 0): overx_val_1 = [] overx_val_2 = [] for x in range(x1_1,x2_1+1): list_1 = imagedata1[y1_1:y2_1,x] * 1.0 overx_val_1.append(saltstat.median(list_1,logfile)) overlevel_1 = saltstat.median(overx_val_1,logfile) for x in range(x1_2,x2_2+1): list_2 = imagedata2[y1_2:y2_2,x] * 1.0 overx_val_2.append(saltstat.median(list_2,logfile)) overlevel_2 = saltstat.median(overx_val_2,logfile) imagedata1 -= overlevel_1 imagedata2 -= overlevel_2 # trim overscan if (status == 0): imagedata1 = imagedata1[dy1_1:dy2_1,dx1_1:dx2_1] imagedata2 = imagedata2[dy1_2:dy2_2,dx1_2:dx2_2] datasec = '[1:'+str(dx2_1-dx1_1)+',1:'+str(dy2_1-dy1_1)+']' status = saltkey.put('DATASEC',datasec,struct[hdu],infile,logfile) status = saltkey.rem('BIASSEC',struct[hdu],infile,logfile) datasec = '[1:'+str(dx2_2-dx1_2)+',1:'+str(dy2_2-dy1_2)+']' status = saltkey.put('DATASEC',datasec,struct[hdu+1],infile,logfile) status = saltkey.rem('BIASSEC',struct[hdu+1],infile,logfile) # log coefficient table if (status == 0): infilename = infile.split('/') infilename = infilename[len(infilename)-1] message = '%25s[%3d] %5.2f %4d %8.6f' % \ (infilename, hdu, gain1, overlevel_1, float(xcoeff[amplifier+1])) saltprint.log(logfile,message,verbose) message = '%25s[%3d] %5.2f %4d %8.6f' % \ (infilename, hdu+1, gain2, overlevel_2,float(xcoeff[amplifier])) saltprint.log(logfile,message,verbose) # update image in HDU structure if (status == 0): struct,status = saltio.writeimage(struct,hdu,imagedata1,logfile) struct,status = saltio.writeimage(struct,hdu+1,imagedata2,logfile) return struct, status
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None, geomfile=None,subover=True,trim=True,masbias=None, subbias=False, median=False, function='polynomial', order=5,rej_lo=3, rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log', verbose=True): """SALTCLEAN will provide basic CCD reductions for a set of data. It will sort the data, and first process the biases, flats, and then the science frames. It will record basic quality control information about each of the steps. """ plotover=False #start logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outpath=saltio.abspath(outpath) #does the gain database file exist if gaindb: dblist= saltio.readgaindb(gaindb) else: dblist=[] # does crosstalk coefficient data exist if xtalkfile: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict=None #does the mosaic file exist--raise error if no saltio.fileexists(geomfile) # Delete the obslog file if it already exists if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile) #read in the obsveration log or create it if os.path.isfile(obslogfile): msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes' raise SaltError(msg) else: headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') ccdtype=obsstruct.data.field('CCDTYPE') #set the bias list of objects biaslist=filename[ccdtype=='ZERO'] masterbias_dict={} for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=pyfits.open(img) bimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #create the list of flatfields and process them flatlist=filename[ccdtype=='FLAT'] masterflat_dict={} for img in infiles: if os.path.basename(img) in flatlist: #open the image struct=pyfits.open(img) fimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Flat frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,fimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list) #create the master flat frame for i in masterflat_dict.keys(): fkeys=masterflat_dict[i][0] flist=masterflat_dict[i][1:] mflatname=outpath+createmasterflatname(flist, fkeys) ffiles=','.join(flist) saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #process the science data for img in infiles: nimg=os.path.basename(img) if not nimg in flatlist or not nimg in biaslist: #open the image struct=pyfits.open(img) simg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the files--currently not in the proper format--will update when it is if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])): mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg)
def bias(struct,subover=True,trim=True, subbias=False, bstruct=None, median=False, function='polynomial',order=3,rej_lo=3,rej_hi=3,niter=10, plotover=False, log=None, verbose=True): """Bias subtracts the bias levels from a frame. It will fit and subtract the overscan region, trim the images, and subtract a master bias if required. struct--image structure subover--subtract the overscan region trim--trim the image subbias--subtract master bias bstruct--master bias image structure median--use the median instead of mean in image statistics function--form to fit to the overscan region order--order for the function rej_lo--sigma of low points to reject in the fit rej_hi--sigma of high points to reject in the fit niter--number of iterations log--saltio log for recording information verbose--whether to print to stdout """ infile=saltkey.getimagename(struct[0]) # how many extensions? nsciext = saltkey.get('NSCIEXT',struct[0]) nextend = saltkey.get('NEXTEND',struct[0]) nccd = saltkey.get('NCCDS',struct[0]) # how many amplifiers?--this is hard wired amplifiers = 2 * nccd #log the process if subover and log: message = '%28s %7s %5s %4s %6s' % \ ('HDU','Overscan','Order','RMS','Niter') log.message('\n --------------------------------------------------', with_header=False, with_stdout=verbose) log.message(message, with_header=False, with_stdout=verbose) log.message(' --------------------------------------------------', with_header=False, with_stdout=verbose) if (plotover): plt.figure(1) plt.axes([0.1,0.1,0.8,0.8]) plt.xlabel('CCD Column') plt.ylabel('Pixel Counts (e-)') plt.ion() #loop through the extensions and subtract the bias for i in range(1,nsciext+1): if struct[i].name=='SCI': #get the bias section biassec = saltkey.get('BIASSEC',struct[i]) y1,y2,x1,x2 = saltio.getSection(biassec, iraf_format=True) #get the data section datasec = saltkey.get('DATASEC',struct[i]) dy1,dy2, dx1, dx2 = saltio.getSection(datasec, iraf_format=True) #setup the overscan region if subover: yarr=np.arange(y1,y2, dtype=float) data=struct[i].data odata=struct[i].data[y1:y2,x1:x2] if median: odata=np.median((struct[i].data[y1:y2,x1:x2]),axis=1) olevel=np.median((struct[i].data[y1:y2,x1:x2])) saltkey.new('OVERSCAN','%f' % (olevel),'Overscan median value', struct[i]) else: odata=np.mean((struct[i].data[y1:y2,x1:x2]),axis=1) olevel=np.mean((struct[i].data[y1:y2,x1:x2])) saltkey.new('OVERSCAN','%f' % (olevel),'Overscan mean value', struct[i]) #fit the overscan region ifit=saltfit.interfit(yarr, odata, function=function, \ order=order, thresh=rej_hi, niter=niter) try: ifit.interfit() coeffs=ifit.coef ofit=ifit(yarr) omean, omed, osigma=saltstat.iterstat((odata-ofit), sig=3, niter=5) except ValueError: #catch the error if it is a zero array ofit=np.array(yarr)*0.0 osigma=0.0 except TypeError: #catch the error if it is a zero array ofit=np.array(yarr)*0.0 osigma=0.0 #if it hasn't been already, convert image to #double format struct[i].data = 1.0 * struct[i].data try: struct[i].header.remove('BZERO') struct[i].header.remove('BSCALE') except: pass #subtract the overscan region for j in range(len(struct[i].data[0])): struct[i].data[y1:y2,j] -= ofit #report the information if log: message = '%25s[%1d] %8.2f %3d %7.2f %3d' % \ (infile, i, olevel, order, osigma, niter) log.message(message, with_stdout=verbose, with_header=False) #add the statistics to the image header saltkey.new('OVERRMS','%f' % (osigma),'Overscan RMS value', struct[i]) #update the variance frame if saltkey.found('VAREXT', struct[i]): vhdu=saltkey.get('VAREXT', struct[i]) try: vdata=struct[vhdu].data #The bias level should not be included in the noise from the signal for j in range(len(struct[i].data[0])): vdata[y1:y2,j] -= ofit #add a bit to make sure that the minimum error is the rednoise rdnoise= saltkey.get('RDNOISE',struct[i]) vdata[vdata<rdnoise**2]=rdnoise**2 struct[vhdu].data=vdata+osigma**2 except Exception, e: msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e) raise SaltError(msg) #plot the overscan region if plotover: plt.plot(yarr, odata) plt.plot(yarr, ofit) #trim the data and update the headers if trim: struct[i].data=struct[i].data[dy1:dy2,dx1:dx2] datasec = '[1:'+str(dx2-dx1)+',1:'+str(dy2-dy1)+']' saltkey.put('DATASEC',datasec,struct[i]) #update the variance frame if saltkey.found('VAREXT', struct[i]): vhdu=saltkey.get('VAREXT', struct[i]) struct[vhdu].data=struct[vhdu].data[dy1:dy2,dx1:dx2] datasec = '[1:'+str(dx2-dx1)+',1:'+str(dy2-dy1)+']' saltkey.put('DATASEC',datasec,struct[vhdu]) #update the BPM frame if saltkey.found('BPMEXT', struct[i]): bhdu=saltkey.get('BPMEXT', struct[i]) struct[bhdu].data=struct[bhdu].data[dy1:dy2,dx1:dx2] datasec = '[1:'+str(dx2-dx1)+',1:'+str(dy2-dy1)+']' saltkey.put('DATASEC',datasec,struct[bhdu]) #subtract the master bias if necessary if subbias and bstruct: struct[i].data -= bstruct[i].data #update the variance frame if saltkey.found('VAREXT', struct[i]): vhdu=saltkey.get('VAREXT', struct[i]) try: vdata=struct[vhdu].data struct[vhdu].data=vdata+bstruct[vhdu].data except Exception, e: msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e) raise SaltError(msg)
def prepare(hdu): """Prepare HRS data to be similar to other SALT file formats This includes splitting each amplifier into multi-extension formats """ #set the detector detname=saltkey.get('DETNAM', hdu[0]) if detname=='08443-03-01' or detname=='HRDET': detector='hrdet' elif detname=='04434-23-02' or detname=='HBDET': detector='hbdet' else: raise SaltError('%s is not an HRS detector' % detnam) #get key parameters try: nccd=saltkey.get('CCDNAMPS', hdu[0]) except: nccd=saltkey.get('CCDAMPS', hdu[0]) xbin, ybin=saltkey.ccdbin(hdu[0]) gain=saltkey.get('GAIN', hdu[0]) gain=gain.split() rospeed=saltkey.get('ROSPEED', hdu[0]) ccdshape=hdu[0].data.shape #create a multiexention fits file phdu=pyfits.PrimaryHDU() phdu.header=hdu[0].header nhdu = pyfits.HDUList(phdu) #these keywords need to be added saltkey.new('DETMODE', value='Normal', comment='Detector mode', hdu=nhdu[0]) saltkey.new('NCCDS', value=1, comment='Detector mode', hdu=nhdu[0]) saltkey.new('GAINSET', value='SLOW', comment='Detector mode', hdu=nhdu[0]) value='OBJECT' if hdu[0].header['OBJECT']=='Bias': value='ZERO' if hdu[0].header['OBJECT']=='Arc': value='ARC' if hdu[0].header['OBJECT']=='Flat': value='FLAT' saltkey.new('CCDTYPE', value=value, comment='CCD Type', hdu=nhdu[0]) if nccd==1: nhdu.append(pyfits.ImageHDU(hdu[0].data)) j=1 saltkey.new('GAIN', value=float(gain[0]), comment='Nominal CCD gain (e/ADU)', hdu=nhdu[j]) saltkey.new('RDNOISE', value=0, comment='Nominal readout noise in e', hdu=nhdu[j]) saltkey.new('SATURATE', value=1, comment='Pixel saturation level in ADU', hdu=nhdu[j]) saltkey.new('XTALK', value=0.0, comment='Cross talk coefficient', hdu=nhdu[j]) detsize=getdetsize(ccdshape, xbin, ybin) ampshape=nhdu[j].data.shape ampsec=getdetsize(ccdshape, 1, 1) saltkey.new('DETSIZE', value=detsize, comment='Detector size', hdu=nhdu[j]) saltkey.new('BIASSEC', value=getbiassec(j, ampshape, xbin, ybin), comment='Bias section', hdu=nhdu[j]) saltkey.new('DATASEC', value=getdatasec(j, ampshape, xbin, ybin), comment='Data section', hdu=nhdu[j]) saltkey.new('AMPSEC', value=ampsec, comment='Amplifier section', hdu=nhdu[j]) saltkey.new('CCDSEC', value=detsize, comment='CCD section', hdu=nhdu[j]) saltkey.new('DETSEC', value=detsize, comment='Detector section', hdu=nhdu[j]) return nhdu for i in range(nccd): j=i+1 x1,x2,y1,y2=definesection(j, detector, ccdshape) nhdu.append(pyfits.ImageHDU(hdu[0].data[y1:y2,x1:x2])) #keywords that need to be added include #gain, rdnoise, xtalk, saturate, saltkey.new('GAIN', value=float(gain[i]), comment='Nominal CCD gain (e/ADU)', hdu=nhdu[j]) saltkey.new('RDNOISE', value=0, comment='Nominal readout noise in e', hdu=nhdu[j]) saltkey.new('SATURATE', value=1, comment='Pixel saturation level in ADU', hdu=nhdu[j]) saltkey.new('XTALK', value=0.0, comment='Cross talk coefficient', hdu=nhdu[j]) #DETSIZE, BIASSEC, DATASEC, #AMPSEC, CCDSEC, DETSEC ampsec='[%i:%i,%i:%i]' % (x1+1,x2,y1+1,y2) ampshape=nhdu[j].data.shape saltkey.new('DETSIZE', value=getdetsize(ccdshape, xbin, ybin), comment='Detector size', hdu=nhdu[j]) saltkey.new('BIASSEC', value=getbiassec(j, ampshape, xbin, ybin), comment='Bias section', hdu=nhdu[j]) saltkey.new('DATASEC', value=getdatasec(j, ampshape, xbin, ybin), comment='Data section', hdu=nhdu[j]) saltkey.new('AMPSEC', value=ampsec, comment='Amplifier section', hdu=nhdu[j]) saltkey.new('CCDSEC', value=getdetsize(ccdshape, xbin, ybin), comment='CCD section', hdu=nhdu[j]) saltkey.new('DETSEC', value=getdetsize(ccdshape, xbin, ybin), comment='Detector section', hdu=nhdu[j]) #BSCALE, BZERO #nhdu[j].header.set('BSCALE', value=1.0, comment='Val=BSCALE*pix+BZERO') #nhdu[j].header.set('BZERO', value=32768., comment='Val=BSCALE*pix+BZERO') #ATM1_1, ATM2_2, #not implimented return nhdu