def convertbin(inpath, fitsconfig, logfile, verbose): if len(glob.glob(inpath+'/*.bin')) > 0: saltbin2fit(inpath=inpath,outpath=inpath,cleanup=True,fitsconfig=fitsconfig,logfile=logfile,verbose=verbose) for bfile in glob.glob(inpath+'/*.bin'): saltio.delete(bfile) ffile=bfile.replace('bin', 'fits') slotreadtimefix(ffile, ffile, '', clobber=True, logfile=logfile, verbose=verbose)
def specsky(images,outimages,outpref, method='normal', section=None, function='polynomial', order=2, clobber=True,logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'') if method not in ['normal', 'fit']: msg='%s mode is not supported yet' % method raise SALTSpecError(msg) if section is None: section=saltio.getSection(section) msg='This mode is not supported yet' raise SALTSpecError(msg) else: section=saltio.getSection(section) # Identify the lines in each file for img, ofile in zip(infiles, outfiles): log.message('Subtracting sky spectrum in image %s into %s' % (img, ofile)) #open the images hdu=saltio.openfits(img) #sky subtract the array hdu=skysubtract(hdu, method=method, section=section, funct=function, order=order) #write out the image if clobber and os.path.isfile(ofile): saltio.delete(ofile) hdu.writeto(ofile)
def saltobslog(images, outfile, clobber=False, logfile='salt.log', verbose=True): """Create the observation log from the input files""" #start the logging with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) #create the header dictionary headerDict = obslog(infiles, log) #clobber the output if it exists if (clobber and os.path.isfile(outfile)): saltio.delete(outfile) #create the fits file struct = createobslogfits(headerDict) # close table file saltio.writefits(struct, outfile) #indicate the log was created log.message('\nSALTLOG -- created observation log ' + outfile)
def saltobslog(images,outfile,clobber=False,logfile='salt.log',verbose=True): """Create the observation log from the input files""" #start the logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) #create the header dictionary headerDict=obslog(infiles, log) #clobber the output if it exists if (clobber and os.path.isfile(outfile)): saltio.delete(outfile) #create the fits file struct=createobslogfits(headerDict) # close table file saltio.writefits(struct, outfile) #indicate the log was created log.message('\nSALTLOG -- created observation log ' + outfile)
def mkheader(file, keyword, value, comment): """create keyword with mkheader IRAF tool i.e. without opening the whole file""" try: tmpfile = saltsafeio.tmpfile('.', False) tmp = saltsafeio.openascii(tmpfile, 'w') tmp.write('%-8s= \'%-18s\' / %-s\n' % (keyword, value, comment)) saltsafeio.closeascii(tmp) iraf.noao.artdata.mkheader(file, tmpfile, append='y', verbose='n') saltsafeio.delete(tmpfile, False) except: raise SaltIOError('Cannot edit keyword ' + keyword + ' in ' + file)
def mkheader(file,keyword,value,comment): """create keyword with mkheader IRAF tool i.e. without opening the whole file""" try: tmpfile=saltsafeio.tmpfile('.',False) tmp=saltsafeio.openascii(tmpfile,'w') tmp.write('%-8s= \'%-18s\' / %-s\n' % (keyword,value,comment)) saltsafeio.closeascii(tmp) iraf.noao.artdata.mkheader(file,tmpfile,append='y',verbose='n') saltsafeio.delete(tmpfile,False) except: raise SaltIOError('Cannot edit keyword '+keyword+' in '+file)
def write_extract_fits(ofile, ap_list, clobber=False): """Write out the extracted spectrum to a FITS table. If the file already exists, this will not overwrite it. For each spectrum in ap_list, it will add another extension to the fits file. Each extension will have the first column as wavelength, the second column as counts, and the third column as sigma on the counts. ofile: Output file to write ap_list: List of extracted spectrum clobber: delete ofile if it already exists """ # delete the file if os.path.isfile(ofile) and clobber: saltio.delete(ofile) # create the primary array hdu = pyfits.PrimaryHDU() hdulist = pyfits.HDUList([hdu]) # create the columns and the for ap in ap_list: fvar = abs(ap.lvar) ** 0.5 # create the columns col1 = pyfits.Column( name='wavelength', format='D', unit='Angstroms', array=ap.wave) col2 = pyfits.Column( name='counts', format='D', unit='Counts', array=ap.ldata) col3 = pyfits.Column(name='counts_err', format='D', array=fvar) # add to the table tbhdu = pyfits.new_table([col1, col2, col3]) hdulist.append(tbhdu) # write it out hdulist.writeto(ofile) return
def specsky(images, outimages, outpref, method='normal', section=None, function='polynomial', order=2, clobber=True, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '') if method not in ['normal', 'fit']: msg = '%s mode is not supported yet' % method raise SALTSpecError(msg) if section is None: section = saltio.getSection(section) msg = 'This mode is not supported yet' raise SALTSpecError(msg) else: section = saltio.getSection(section) # Identify the lines in each file for img, ofile in zip(infiles, outfiles): log.message('Subtracting sky spectrum in image %s into %s' % (img, ofile)) # open the images hdu = saltio.openfits(img) # sky subtract the array hdu = skysubtract(hdu, method=method, section=section, funct=function, order=order) # write out the image if clobber and os.path.isfile(ofile): saltio.delete(ofile) hdu.writeto(ofile)
def write_extract_fits(ofile, ap_list, clobber=False): """Write out the extracted spectrum to a FITS table. If the file already exists, this will not overwrite it. For each spectrum in ap_list, it will add another extension to the fits file. Each extension will have the first column as wavelength, the second column as counts, and the third column as sigma on the counts. ofile: Output file to write ap_list: List of extracted spectrum clobber: delete ofile if it already exists """ # delete the file if os.path.isfile(ofile) and clobber: saltio.delete(ofile) # create the primary array hdu = pyfits.PrimaryHDU() hdulist = pyfits.HDUList([hdu]) # create the columns and the for ap in ap_list: fvar = abs(ap.lvar)**0.5 # create the columns col1 = pyfits.Column(name='wavelength', format='D', unit='Angstroms', array=ap.wave) col2 = pyfits.Column(name='counts', format='D', unit='Counts', array=ap.ldata) col3 = pyfits.Column(name='counts_err', format='D', array=fvar) # add to the table tbhdu = pyfits.new_table([col1, col2, col3]) hdulist.append(tbhdu) # write it out hdulist.writeto(ofile) return
def saltelsdata(propcode, obsdate, elshost, elsname, elsuser, elspass, sdbhost,sdbname,sdbuser, password, clobber,logfile,verbose): # set up proposers = [] propids = [] pids = [] with logging(logfile,debug) as log: # are the arguments defined if propcode.strip().upper()=='ALL': pids=saltmysql.getproposalcodes(str(obsdate), sdbhost,sdbname,sdbuser, password) else: pids = saltio.argunpack('propcode',propcode) #open the database els=saltmysql.connectelsview(elshost, elsname, elsuser, elspass) sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser, password) #create the values for the entire night #loop through the proposals for pid in pids: outfile='%s_%s_elsdata.fits' % (pid, obsdate) if clobber and os.path.isfile(outfile): saltio.delete(outfile) mintime, maxtime=determinetime(sdb, pid, obsdate) message='Extracting ELS data for %s from %s to %s' % (pid, mintime, maxtime) log.message(message, with_stdout=verbose) getelsdata(els, sdb, outfile, mintime, maxtime)
def slotback(images,outfits,extension,imgtype='image',subbacktype='median', sigback=3,mbin=7,sorder=3,niter=5,ampperccd=2,ignorexp=6, clobber=False,logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # set up the variables order=sorder plotfreq=0.01 ftime=plotfreq # is the input file specified? infiles = saltio.argunpack ('Input',images) # is the output file specified? saltio.filedefined('Output',outfits) #open the first file and check its charactistics struct=saltio.openfits(infiles[0]) # how many extensions? nextend=saltkey.get('NEXTEND',struct[0]) if nextend < extension: msg='Insufficient number of extensions in %s' % (infile) raise SaltIOError(msg) # how many amplifiers? amplifiers=saltkey.get('NCCDS',struct[0]) amplifiers = int(ampperccd*float(amplifiers)) if ampperccd>0: nframes = nextend/amplifiers nstep=amplifiers else: nframes = nextend nstep=1 ntotal=nframes*len(infiles) # image size naxis1=saltkey.get('NAXIS1',struct[extension]) naxis2=saltkey.get('NAXIS2',struct[extension]) # CCD binning ccdsum=saltkey.get('CCDSUM',struct[0]) binx=int(ccdsum.split(' ')[0]) biny=int(ccdsum.split(' ')[1]) # If a total file is to written out, create it and update it hdu = 1 # delete the file if clobber is on and the file exists if os.path.isfile(outfits): if clobber: saltio.delete(outfits) else: raise SaltIOError('Output fits file '+writenewfits+'already exists. Use Clobber=yes to overwrite file') # create the output file try: hdu = pyfits.PrimaryHDU() hdu.header=struct[0].header hdu.header['NCCDS']=1 hdu.header['NSCIEXT']=ntotal-ignorexp hdu.header['NEXTEND']=ntotal-ignorexp hduList = pyfits.HDUList(hdu) hduList.verify() hduList.writeto(outfits) except: raise SaltIOError('Could not create new fits file named '+writenewfits) # read it back in for updating hduList = saltio.openfits(outfits,mode='update') # set up the completeness tracker if verbose: j=0 x2=float(j)/float(ntotal) ctext='Percentage Complete: %3.2f' % x2 sys.stdout.write(ctext) sys.stdout.flush() for infile in infiles: struct=saltio.openfits(infile) # Skip through the frames and process each frame individually for i in range(nframes): if not (infile==infiles[0] and i < ignorexp): hdu=extension+i*nstep try: header=struct[hdu].header array=struct[hdu].data array=array*1.0 except Exception as e: msg='Unable to open extension %i in image %s because %s' % (hdu, infile, e) raise SaltIOError(msg) # start the analysis of each frame # gain and readout noise try: gain=float(header['GAIN']) except: gain=1 log.warning('Gain not specified in image header') try: rdnoise=float(header['RDNOISE']) except: rdnoise=0 log.warning('RDNOISE not specified in image header') # background subtraction if not subbacktype=='none': try: narray=subbackground(array, sigback, mbin, order, niter, subbacktype) except: log.warning('Image '+infile+' extention '+str(i)+' is blank, skipping') continue # create output array try: if imgtype=='background': oarray=narray-array else: oarray=narray except: oarray=array # print progress if verbose: x2=float(j)/float(ntotal) if x2 > ftime: ctext='\b\b\b\b\b %3.2f' % x2 sys.stdout.write(ctext) sys.stdout.flush() ftime += plotfreq # update the header values with the image name and extension number try: hdue = pyfits.ImageHDU(oarray) hdue.header=header hdue.header.update('ONAME',infile,'Original image name') hdue.header.update('OEXT',hdu,'Original extension number') except Exception as e: msg='SALTPHOT--WARNING: Could not update image in newfits file for %s ext %i because %s' \ % (infile, hdu, e) raise SaltIOError(msg) hduList.append(hdue) j +=1 # close FITS file saltio.closefits(struct) # close the output fits file: try: # write out the file hduList.flush() hduList.close() except Exception as e: raise SaltIOError('Failed to write %s because %s' % (outfits, e))
def make_mosaic(struct, gap, xshift, yshift, rotation, interp_type='linear', boundary='constant', constant=0, geotran=True, fill=False, cleanup=True, log=None, verbose=False): """Given a SALT image struct, combine each of the individual amplifiers and apply the geometric CCD transformations to the image """ # get the name of the file infile = saltkey.getimagename(struct[0], base=True) outpath = './' # identify instrument instrume, keyprep, keygain, keybias, keyxtalk, keyslot = \ saltkey.instrumid(struct) # how many amplifiers? nsciext = saltkey.get('NSCIEXT', struct[0]) nextend = saltkey.get('NEXTEND', struct[0]) nccds = saltkey.get('NCCDS', struct[0]) amplifiers = nccds * 2 if nextend > nsciext: varframe = True else: varframe = False # CCD geometry coefficients if (instrume == 'RSS' or instrume == 'PFIS'): xsh = [0., xshift[0], 0., xshift[1]] ysh = [0., yshift[0], 0., yshift[1]] rot = [0., rotation[0], 0., rotation[1]] elif instrume == 'SALTICAM': xsh = [0., xshift[0], 0.] ysh = [0., yshift[0], 0.] rot = [0., rotation[0], 0] # how many extensions? nextend = saltkey.get('NEXTEND', struct[0]) # CCD on-chip binning xbin, ybin = saltkey.ccdbin(struct[0]) # create temporary primary extension outstruct = [] outstruct.append(struct[0]) # define temporary FITS file store tiled CCDs tilefile = saltio.tmpfile(outpath) tilefile += 'tile.fits' if varframe: tilehdu = [None] * (3 * int(nsciext / 2) + 1) else: tilehdu = [None] * int(nsciext / 2 + 1) tilehdu[0] = fits.PrimaryHDU() #tilehdu[0].header = struct[0].header if log: log.message('', with_stdout=verbose) # iterate over amplifiers, stich them to produce file of CCD images for i in range(int(nsciext / 2)): hdu = i * 2 + 1 # amplifier = hdu%amplifiers # if (amplifier == 0): amplifier = amplifiers # read DATASEC keywords datasec1 = saltkey.get('DATASEC', struct[hdu]) datasec2 = saltkey.get('DATASEC', struct[hdu + 1]) xdsec1, ydsec1 = saltstring.secsplit(datasec1) xdsec2, ydsec2 = saltstring.secsplit(datasec2) # read images imdata1 = saltio.readimage(struct, hdu) imdata2 = saltio.readimage(struct, hdu + 1) # tile 2n amplifiers to yield n CCD images outdata = numpy.zeros((ydsec1[1] + abs(ysh[i + 1] / ybin), xdsec1[1] + xdsec2[1] + abs(xsh[i + 1] / xbin)), numpy.float32) # set up the variance frame if varframe: vardata = outdata.copy() vdata1 = saltio.readimage(struct, struct[hdu].header['VAREXT']) vdata2 = saltio.readimage(struct, struct[hdu + 1].header['VAREXT']) bpmdata = outdata.copy() bdata1 = saltio.readimage(struct, struct[hdu].header['BPMEXT']) bdata2 = saltio.readimage(struct, struct[hdu + 1].header['BPMEXT']) x1 = xdsec1[0] - 1 if x1 != 0: msg = 'The data in %s have not been trimmed prior to mosaicking.' \ % infile log.error(msg) if xsh[i + 1] < 0: x1 += abs(xsh[i + 1] / xbin) x2 = x1 + xdsec1[1] y1 = ydsec1[0] - 1 if ysh[i + 1] < 0: y1 += abs(ysh[i + 1] / ybin) y2 = y1 + ydsec1[1] outdata[y1:y2, x1:x2] =\ imdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] if varframe: vardata[y1:y2, x1:x2] =\ vdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] bpmdata[y1:y2, x1:x2] =\ bdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] x1 = x2 x2 = x1 + xdsec2[1] y1 = ydsec2[0] - 1 if ysh[i + 1] < 0: y1 += abs(ysh[i + 1] / ybin) y2 = y1 + ydsec2[1] outdata[y1:y2, x1:x2] =\ imdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] if varframe: vardata[y1:y2, x1:x2] =\ vdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] bpmdata[y1:y2, x1:x2] =\ bdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] # size of new image naxis1 = str(xdsec1[1] + xdsec2[1]) naxis2 = str(ydsec1[1]) # add image and keywords to HDU list tilehdu[i + 1] = fits.ImageHDU(outdata) tilehdu[i + 1].header = struct[hdu].header #tilehdu[ # i + 1].header['DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' if varframe: vext = i + 1 + int(nsciext / 2.) tilehdu[vext] = fits.ImageHDU(vardata) #tilehdu[vext].header = struct[struct[hdu].header['VAREXT']].header #tilehdu[vext].header[ # 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' bext = i + 1 + 2 * int(nsciext / 2.) tilehdu[bext] = fits.ImageHDU(bpmdata) #tilehdu[bext].header = struct[struct[hdu].header['BPMEXT']].header #tilehdu[bext].header[ # 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' # image tile log message #1 if log: message = os.path.basename(infile) + '[' + str(hdu) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> ' message += os.path.basename(tilefile) + '[' + str(i + 1) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + ']' log.message(message, with_stdout=verbose, with_header=False) message = os.path.basename(infile) + '[' + str(hdu + 1) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> ' message += os.path.basename(tilefile) + '[' + str(i + 1) + '][' message += str(xdsec1[1] + 1) + ':' + \ str(xdsec1[1] + xdsec2[1]) + ',' message += str(ydsec2[0]) + ':' + str(ydsec2[1]) + ']' log.message(message, with_stdout=verbose, with_header=False) # write temporary file of tiled CCDs hdulist = fits.HDUList(tilehdu) hdulist.writeto(tilefile) # iterate over CCDs, transform and rotate images yrot = [None] * 4 xrot = [None] * 4 tranfile = [' '] tranhdu = [0] if varframe: tranfile = [''] * (3 * int(nsciext / 2) + 1) tranhdu = [0] * (3 * int(nsciext / 2) + 1) else: tranfile = [''] * int(nsciext / 2 + 1) tranhdu = [0] * int(nsciext / 2 + 1) # this is hardwired for SALT where the second CCD is considered the # fiducial for hdu in range(1, int(nsciext / 2 + 1)): tranfile[hdu] = saltio.tmpfile(outpath) tranfile[hdu] += 'tran.fits' if varframe: tranfile[hdu + nccds] = saltio.tmpfile(outpath) + 'tran.fits' tranfile[hdu + 2 * nccds] = saltio.tmpfile(outpath) + 'tran.fits' ccd = hdu % nccds if (ccd == 0): ccd = nccds # correct rotation for CCD binning yrot[ccd] = rot[ccd] * ybin / xbin xrot[ccd] = rot[ccd] * xbin / ybin dxshift = xbin * int(float(int(gap) / xbin) + 0.5) - gap # transformation using geotran IRAF task # if (ccd == 1): if (ccd != 2): if geotran: message = '\nSALTMOSAIC -- geotran ' + tilefile + \ '[' + str(ccd) + '] ' + tranfile[hdu] message += ' \"\" \"\" xshift=' + \ str((xsh[ccd] + (2 - ccd) * dxshift) / xbin) + ' ' message += 'yshift=' + \ str(ysh[ccd] / ybin) + ' xrotation=' + str(xrot[ccd]) + ' ' message += 'yrotation=' + \ str(yrot[ccd]) + ' xmag=1 ymag=1 xmin=\'INDEF\'' message += 'xmax=\'INDEF\' ymin=\'INDEF\' ymax=\'INDEF\' ' message += 'ncols=\'INDEF\' ' message += 'nlines=\'INDEF\' verbose=\'no\' ' message += 'fluxconserve=\'yes\' nxblock=2048 ' message += 'nyblock=2048 interpolant=\'' + \ interp_type + '\' boundary=\'constant\' constant=0' log.message(message, with_stdout=verbose) yd, xd = tilehdu[ccd].data.shape ncols = 'INDEF' # ncols=xd+abs(xsh[ccd]/xbin) nlines = 'INDEF' # nlines=yd+abs(ysh[ccd]/ybin) geo_xshift = xsh[ccd] + (2 - ccd) * dxshift / xbin geo_yshift = ysh[ccd] / ybin iraf.images.immatch.geotran(tilefile + "[" + str(ccd) + "]", tranfile[hdu], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) if varframe: var_infile = tilefile + "[" + str(ccd + nccds) + "]" iraf.images.immatch.geotran(var_infile, tranfile[hdu + nccds], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) var2_infile = tilefile + "[" + str(ccd + 2 * nccds) + "]" iraf.images.immatch.geotran(var2_infile, tranfile[hdu + 2 * nccds], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) # open the file and copy the data to tranhdu tstruct = fits.open(tranfile[hdu]) tranhdu[hdu] = tstruct[0].data tstruct.close() if varframe: tranhdu[ hdu + nccds] = fits.open( tranfile[ hdu + nccds])[0].data tranhdu[ hdu + 2 * nccds] = fits.open( tranfile[ hdu + 2 * nccds])[0].data else: log.message( "Transform CCD #%i using dx=%s, dy=%s, rot=%s" % (ccd, xsh[ccd] / 2.0, ysh[ccd] / 2.0, xrot[ccd]), with_stdout=verbose, with_header=False) tranhdu[hdu] = geometric_transform( tilehdu[ccd].data, tran_func, prefilter=False, order=1, extra_arguments=( xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) tstruct = fits.PrimaryHDU(tranhdu[hdu]) tstruct.writeto(tranfile[hdu]) if varframe: tranhdu[hdu + nccds] = geometric_transform( tilehdu[hdu + 3].data, tran_func, prefilter=False, order=1, extra_arguments=( xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) tranhdu[hdu + 2 * nccds] = geometric_transform( tilehdu[hdu + 6].data, tran_func, prefilter=False, order=1, extra_arguments=( xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) else: log.message( "Transform CCD #%i using dx=%s, dy=%s, rot=%s" % (ccd, 0, 0, 0), with_stdout=verbose, with_header=False) tranhdu[hdu] = tilehdu[ccd].data if varframe: tranhdu[hdu + nccds] = tilehdu[ccd + nccds].data tranhdu[hdu + 2 * nccds] = tilehdu[ccd + 2 * nccds].data # open outfile if varframe: outlist = 4 * [None] else: outlist = 2 * [None] #outlist[0] = struct[0].copy() outlist[0] = fits.PrimaryHDU() outlist[0].header = struct[0].header naxis1 = int(gap / xbin * (nccds - 1)) naxis2 = 0 for i in range(1, nccds + 1): yw, xw = tranhdu[i].shape naxis1 += xw + int(abs(xsh[ccd] / xbin)) + 1 naxis2 = max(naxis2, yw) outdata = numpy.zeros((naxis2, naxis1), numpy.float32) outdata.shape = naxis2, naxis1 if varframe: vardata = outdata * 0 bpmdata = outdata * 0 + 1 # iterate over CCDs, stich them to produce a full image hdu = 0 totxshift = 0 for hdu in range(1, nccds + 1): # read DATASEC keywords ydsec, xdsec = tranhdu[hdu].shape # define size and shape of final image # tile CCDs to yield mosaiced image x1 = int((hdu - 1) * (xdsec + gap / xbin)) + int(totxshift) x2 = xdsec + x1 y1 = int(0) y2 = int(ydsec) outdata[y1:y2, x1:x2] = tranhdu[hdu] totxshift += int(abs(xsh[hdu] / xbin)) + 1 if varframe: vardata[y1:y2, x1:x2] = tranhdu[hdu + nccds] bpmdata[y1:y2, x1:x2] = tranhdu[hdu + 2 * nccds] # make sure to cover up all the gaps include bad areas if varframe: baddata = (outdata == 0) baddata = nd.maximum_filter(baddata, size=3) bpmdata[baddata] = 1 # fill in the gaps if requested if fill: if varframe: outdata = fill_gaps(outdata, 0) else: outdata = fill_gaps(outdata, 0) # add to the file outlist[1] = fits.ImageHDU(outdata) if varframe: outlist[2] = fits.ImageHDU(vardata,name='VAR') outlist[3] = fits.ImageHDU(bpmdata,name='BPM') # create the image structure outstruct = fits.HDUList(outlist) # update the head informaation # housekeeping keywords saltkey.put('NEXTEND', 2, outstruct[0]) saltkey.new('EXTNAME', 'SCI', 'Extension name', outstruct[1]) saltkey.new('EXTVER', 1, 'Extension number', outstruct[1]) if varframe: saltkey.new('VAREXT', 2, 'Variance frame extension', outstruct[1]) saltkey.new('BPMEXT', 3, 'BPM Extension', outstruct[1]) try: saltkey.copy(struct[1], outstruct[1], 'CCDSUM') except: pass # Add keywords associated with geometry saltkey.new('SGEOMGAP', gap, 'SALT Chip Gap', outstruct[0]) c1str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[0], yshift[0], rotation[0]) saltkey.new('SGEOM1', c1str, 'SALT Chip 1 Transform', outstruct[0]) c2str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[1], yshift[1], rotation[1]) saltkey.new('SGEOM2', c2str, 'SALT Chip 2 Transform', outstruct[0]) # WCS keywords saltkey.new('CRPIX1', 0, 'WCS: X reference pixel', outstruct[1]) saltkey.new('CRPIX2', 0, 'WCS: Y reference pixel', outstruct[1]) saltkey.new( 'CRVAL1', float(xbin), 'WCS: X reference coordinate value', outstruct[1]) saltkey.new( 'CRVAL2', float(ybin), 'WCS: Y reference coordinate value', outstruct[1]) saltkey.new('CDELT1', float(xbin), 'WCS: X pixel size', outstruct[1]) saltkey.new('CDELT2', float(ybin), 'WCS: Y pixel size', outstruct[1]) saltkey.new('CTYPE1', 'pixel', 'X type', outstruct[1]) saltkey.new('CTYPE2', 'pixel', 'Y type', outstruct[1]) # cleanup temporary files if cleanup: for tfile in tranfile: if os.path.isfile(tfile): saltio.delete(tfile) if os.path.isfile(tilefile): status = saltio.delete(tilefile) # return the file return outstruct
def saltfpringfit(images, outfile, section=None, bthresh=5, niter=5, displayimage=True, clobber=True,logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # read in the section if section is None: section=saltio.getSection(section) msg='This mode is not supported yet' raise SaltError(msg) else: section=saltio.getSection(section) print section # open each raw image file for img in infiles: #open the fits file struct=saltio.openfits(img) data=struct[0].data #only keep the bright pixels y1,y2,x1,x2=section bmean, bmedian, bstd=iterstat(data[y1:y2,x1:x2], sig=bthresh, niter=niter, verbose=False) message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \ ('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd) log.message(message, with_stdout=verbose) mdata=data*(data-bmean>bthresh*bstd) #prepare the first guess for the image ring_list=findrings(data, thresh=5, niter=5, minsize=10) if displayimage: regfile=img.replace('.fits', '.reg') print regfile if clobber and os.path.isfile(regfile): fout=saltio.delete(regfile) fout=open(regfile, 'w') fout.write("""# Region file format: DS9 version 4.1 # Filename: %s global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 physical """ % img) for ring in ring_list: print ring fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad)) fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad-5*ring.sigma)) fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad+5*ring.sigma)) fout.close() display(img, catname=regfile, rformat='reg') #write out the result for viewing struct[0].data=mdata saltio.writefits(struct, 'out.fits', clobber=True) message = 'Ring Parameters' log.message(message)
def vid2fits(inhead, inbin,outfile, config): """ Convert bin files made during the video process to regular fits files Format python vid2fits.py inhead inbin outfits config Returns """ #Check that the input files exists saltio.fileexists(inhead) saltio.fileexists(inbin) saltio.fileexists(config) #if output file exists, then delete if os.path.isfile(outfile): saltio.delete(outfile) #read in and process the config file condict=fitsconfig(config) #read in the header information infits=saltio.openfits(inhead) inheader = infits['Primary'].header instrume=inheader['INSTRUME'] detswv=softwareversion(inheader['DETSWV']) #create a new image and copy the header to it try: hdu = fits.PrimaryHDU() hdu.header=inheader hduList = fits.HDUList(hdu) #hduList.verify() hduList.writeto(outfile, output_verify='ignore') except: message = 'ERROR -- VID2FIT: Could not create new fits file' raise SaltError(message) #Now open up the file that you just made and update it from here on hduList = fits.open(outfile, mode='update') #open the binary file bindata = saltio.openbinary(inbin,'rb') #read in header information from binary file #some constants that are needed for reading in the binary data sizeofinteger=struct.calcsize('i') sizeofunsignshort=struct.calcsize('H') sizeofdouble=struct.calcsize('d') sizeoffloat =struct.calcsize('f') #read in the number of exposures, geometry of image (width and height) and number of amps nframes= saltio.readbinary(bindata,sizeofinteger,"=i") if detswv<=4.78 and instrume=='SALTICAM': fwidth= saltio.readbinary(bindata,sizeofinteger,"=i") fheight= saltio.readbinary(bindata,sizeofinteger, "=i") elif (detswv>=7.01 and instrume=='SALTICAM') or (detswv>=4.37 and instrume=='RSS'): fwidth= saltio.readbinary(bindata,sizeofunsignshort,"=H") fheight= saltio.readbinary(bindata,sizeofunsignshort, "=H") pbcols=saltio.readbinary(bindata,sizeofunsignshort, "=H") pbrows=saltio.readbinary(bindata,sizeofunsignshort,"=H") else: message='VID2FITS--Detector Software version %s is not supported' % detswv raise SaltError(message) nelements=fwidth*fheight namps=saltio.readbinary(bindata,sizeofinteger,"=i") #read in the gain gain = numpy.zeros(namps,dtype=float) for i in range(namps): gain[i]=saltio.readbinary(bindata,sizeofdouble,"=d") #read in the rdnoise rdnoise = numpy.zeros(namps, dtype=float) for i in range(namps): rdnoise[i]=saltio.readbinary(bindata,sizeofdouble,"=d") #set the scale parameters bzero=32768 bscale=1 otime=0 #start the loop to read in the data for i in range(nframes): #read in the start of the data,time starttime= saltio.readbinary(bindata,sizeofdouble,"=d") date_obs, time_obs= ascii_time(starttime) #read in the exposure time exptime= saltio.readbinary(bindata,sizeofdouble,"=d") #read in the dead time in milliseconds if (detswv>=7.01 and instrume=='SALTICAM') or (detswv>=4.37 and instrume=='RSS'): deadtime= saltio.readbinary(bindata,sizeofinteger,"=i") else: deadtime=None #read in the dead time if (detswv>=7.01 and instrume=='SALTICAM') or (detswv>=4.37 and instrume=='RSS'): framecnt= saltio.readbinary(bindata,sizeofinteger,"=i") else: framecnt=None otime=starttime #read in the data shape = (fheight,fwidth) imdata = numpy.fromfile(bindata,dtype=numpy.ushort,count=nelements) imdata = imdata.reshape(shape) #for each amplifier write it to the image if namps > 0: awidth=fwidth/namps for j in range(namps): ###create the new extension ### #cut each image by the number of amplifiers y1=j*awidth y2=y1+awidth data = imdata[:,y1:y2].astype(numpy.int16) hdue = fits.ImageHDU(data) hdue.scale('int16','',bzero=bzero) #set the header values datasec,detsec,ccdsec,ampsec,biassec= \ create_header_values(condict,hdu,j,fheight) #fill in the header data hdue = write_ext_header(hdue,outfile,hdu,time_obs,date_obs,bscale,bzero, \ exptime,gain[j],rdnoise[j],datasec,detsec,ccdsec,ampsec, \ biassec, deadtime=deadtime, framecnt=framecnt ) #append the extension to the image hduList.append(hdue) try: hduList.flush() hduList.close() except Exception, e: message = 'ERROR: VID2BIN -- Fail to convert %s due to %s' % (outfile, e) raise SaltError(message)
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None, subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log',verbose=True): """Convert MEF HRS data into a single image. If variance frames and BPMs, then convert them to the same format as well. Returns an MEF image but that is combined into a single frame """ with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outpath=saltio.abspath(outpath) if saltio.checkfornone(obslogfile) is None: raise SaltError('Obslog file is required') # Delete the obslog file if it already exists if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile): if os.path.isfile(obslogfile): saltio.delete(obslogfile) #read in the obsveration log or create it headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) else: obsstruct=saltio.openfits(obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') ccdtype=obsstruct.data.field('OBJECT') biaslist=filename[ccdtype=='Bias'] masterbias_dict={} if log: log.message('Processing Bias Frames') for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=pyfits.open(img) bimg=outpath+'bgph'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose, with_header=False) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=False, imstack=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list) #create the master bias frame for i in list(masterbias_dict.keys()): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys, x1=5, x2=13) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #apply full reductions to the science data for img in infiles: nimg=os.path.basename(img) if not nimg in biaslist: #open the image struct=pyfits.open(img) simg=outpath+'mbgph'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #get master bias frame masterbias=get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list) if masterbias: subbias=True bstruct=saltio.openfits(masterbias) else: subbias=False bstruct=None #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=subbias, imstack=True, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) return
def saltfpringfit(images, outfile, section=None, bthresh=5, niter=5, displayimage=True, clobber=True, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # read in the section if section is None: section = saltio.getSection(section) msg = 'This mode is not supported yet' raise SaltError(msg) else: section = saltio.getSection(section) print section # open each raw image file for img in infiles: #open the fits file struct = saltio.openfits(img) data = struct[0].data #only keep the bright pixels y1, y2, x1, x2 = section bmean, bmedian, bstd = iterstat(data[y1:y2, x1:x2], sig=bthresh, niter=niter, verbose=False) message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \ ('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd) log.message(message, with_stdout=verbose) mdata = data * (data - bmean > bthresh * bstd) #prepare the first guess for the image ring_list = findrings(data, thresh=5, niter=5, minsize=10) if displayimage: regfile = img.replace('.fits', '.reg') print regfile if clobber and os.path.isfile(regfile): fout = saltio.delete(regfile) fout = open(regfile, 'w') fout.write("""# Region file format: DS9 version 4.1 # Filename: %s global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 physical """ % img) for ring in ring_list: print ring fout.write('circle(%f, %f, %f)\n' % (ring.xc, ring.yc, ring.prad)) fout.write('circle(%f, %f, %f)\n' % (ring.xc, ring.yc, ring.prad - 5 * ring.sigma)) fout.write('circle(%f, %f, %f)\n' % (ring.xc, ring.yc, ring.prad + 5 * ring.sigma)) fout.close() display(img, catname=regfile, rformat='reg') #write out the result for viewing struct[0].data = mdata saltio.writefits(struct, 'out.fits', clobber=True) message = 'Ring Parameters' log.message(message)
def make_mosaic(struct, gap, xshift, yshift, rotation, interp_type='linear', boundary='constant', constant=0, geotran=True, fill=False, cleanup=True, log=None, verbose=False): """Given a SALT image struct, combine each of the individual amplifiers and apply the geometric CCD transformations to the image """ # get the name of the file infile = saltkey.getimagename(struct[0], base=True) outpath = './' # identify instrument instrume, keyprep, keygain, keybias, keyxtalk, keyslot = \ saltkey.instrumid(struct) # how many amplifiers? nsciext = saltkey.get('NSCIEXT', struct[0]) nextend = saltkey.get('NEXTEND', struct[0]) nccds = saltkey.get('NCCDS', struct[0]) amplifiers = nccds * 2 if nextend > nsciext: varframe = True else: varframe = False # CCD geometry coefficients if (instrume == 'RSS' or instrume == 'PFIS'): xsh = [0., xshift[0], 0., xshift[1]] ysh = [0., yshift[0], 0., yshift[1]] rot = [0., rotation[0], 0., rotation[1]] elif instrume == 'SALTICAM': xsh = [0., xshift[0], 0.] ysh = [0., yshift[0], 0.] rot = [0., rotation[0], 0] # how many extensions? nextend = saltkey.get('NEXTEND', struct[0]) # CCD on-chip binning xbin, ybin = saltkey.ccdbin(struct[0]) # create temporary primary extension outstruct = [] outstruct.append(struct[0]) # define temporary FITS file store tiled CCDs tilefile = saltio.tmpfile(outpath) tilefile += 'tile.fits' if varframe: tilehdu = [None] * (3 * int(nsciext / 2) + 1) else: tilehdu = [None] * int(nsciext / 2 + 1) tilehdu[0] = fits.PrimaryHDU() #tilehdu[0].header = struct[0].header if log: log.message('', with_stdout=verbose) # iterate over amplifiers, stich them to produce file of CCD images for i in range(int(nsciext / 2)): hdu = i * 2 + 1 # amplifier = hdu%amplifiers # if (amplifier == 0): amplifier = amplifiers # read DATASEC keywords datasec1 = saltkey.get('DATASEC', struct[hdu]) datasec2 = saltkey.get('DATASEC', struct[hdu + 1]) xdsec1, ydsec1 = saltstring.secsplit(datasec1) xdsec2, ydsec2 = saltstring.secsplit(datasec2) # read images imdata1 = saltio.readimage(struct, hdu) imdata2 = saltio.readimage(struct, hdu + 1) # tile 2n amplifiers to yield n CCD images outdata = numpy.zeros( (int(ydsec1[1] + abs(ysh[i + 1] / ybin)), int(xdsec1[1] + xdsec2[1] + abs(xsh[i + 1] / xbin))), numpy.float32) # set up the variance frame if varframe: vardata = outdata.copy() vdata1 = saltio.readimage(struct, struct[hdu].header['VAREXT']) vdata2 = saltio.readimage(struct, struct[hdu + 1].header['VAREXT']) bpmdata = outdata.copy() bdata1 = saltio.readimage(struct, struct[hdu].header['BPMEXT']) bdata2 = saltio.readimage(struct, struct[hdu + 1].header['BPMEXT']) x1 = xdsec1[0] - 1 if x1 != 0: msg = 'The data in %s have not been trimmed prior to mosaicking.' \ % infile log.error(msg) if xsh[i + 1] < 0: x1 += int(abs(xsh[i + 1] / xbin)) x2 = x1 + xdsec1[1] y1 = ydsec1[0] - 1 if ysh[i + 1] < 0: y1 += int(abs(ysh[i + 1] / ybin)) y2 = y1 + ydsec1[1] outdata[y1:y2, x1:x2] =\ imdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] if varframe: vardata[y1:y2, x1:x2] =\ vdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] bpmdata[y1:y2, x1:x2] =\ bdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] x1 = x2 x2 = x1 + xdsec2[1] y1 = ydsec2[0] - 1 if ysh[i + 1] < 0: y1 += abs(ysh[i + 1] / ybin) y2 = y1 + ydsec2[1] outdata[y1:y2, x1:x2] =\ imdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] if varframe: vardata[y1:y2, x1:x2] =\ vdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] bpmdata[y1:y2, x1:x2] =\ bdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]] # size of new image naxis1 = str(xdsec1[1] + xdsec2[1]) naxis2 = str(ydsec1[1]) # add image and keywords to HDU list tilehdu[i + 1] = fits.ImageHDU(outdata) tilehdu[i + 1].header = struct[hdu].header #tilehdu[ # i + 1].header['DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' if varframe: vext = i + 1 + int(nsciext / 2.) tilehdu[vext] = fits.ImageHDU(vardata) #tilehdu[vext].header = struct[struct[hdu].header['VAREXT']].header #tilehdu[vext].header[ # 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' bext = i + 1 + 2 * int(nsciext / 2.) tilehdu[bext] = fits.ImageHDU(bpmdata) #tilehdu[bext].header = struct[struct[hdu].header['BPMEXT']].header #tilehdu[bext].header[ # 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']' # image tile log message #1 if log: message = os.path.basename(infile) + '[' + str(hdu) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> ' message += os.path.basename(tilefile) + '[' + str(i + 1) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + ']' log.message(message, with_stdout=verbose, with_header=False) message = os.path.basename(infile) + '[' + str(hdu + 1) + '][' message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ',' message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> ' message += os.path.basename(tilefile) + '[' + str(i + 1) + '][' message += str(xdsec1[1] + 1) + ':' + \ str(xdsec1[1] + xdsec2[1]) + ',' message += str(ydsec2[0]) + ':' + str(ydsec2[1]) + ']' log.message(message, with_stdout=verbose, with_header=False) # write temporary file of tiled CCDs hdulist = fits.HDUList(tilehdu) hdulist.writeto(tilefile) # iterate over CCDs, transform and rotate images yrot = [None] * 4 xrot = [None] * 4 tranfile = [' '] tranhdu = [0] if varframe: tranfile = [''] * (3 * int(nsciext / 2) + 1) tranhdu = [0] * (3 * int(nsciext / 2) + 1) else: tranfile = [''] * int(nsciext / 2 + 1) tranhdu = [0] * int(nsciext / 2 + 1) # this is hardwired for SALT where the second CCD is considered the # fiducial for hdu in range(1, int(nsciext / 2 + 1)): tranfile[hdu] = saltio.tmpfile(outpath) tranfile[hdu] += 'tran.fits' if varframe: tranfile[hdu + nccds] = saltio.tmpfile(outpath) + 'tran.fits' tranfile[hdu + 2 * nccds] = saltio.tmpfile(outpath) + 'tran.fits' ccd = hdu % nccds if (ccd == 0): ccd = nccds # correct rotation for CCD binning yrot[ccd] = rot[ccd] * ybin / xbin xrot[ccd] = rot[ccd] * xbin / ybin dxshift = xbin * int(float(int(gap) / xbin) + 0.5) - gap # transformation using geotran IRAF task # if (ccd == 1): if (ccd != 2): if geotran: message = '\nSALTMOSAIC -- geotran ' + tilefile + \ '[' + str(ccd) + '] ' + tranfile[hdu] message += ' \"\" \"\" xshift=' + \ str((xsh[ccd] + (2 - ccd) * dxshift) / xbin) + ' ' message += 'yshift=' + \ str(ysh[ccd] / ybin) + ' xrotation=' + str(xrot[ccd]) + ' ' message += 'yrotation=' + \ str(yrot[ccd]) + ' xmag=1 ymag=1 xmin=\'INDEF\'' message += 'xmax=\'INDEF\' ymin=\'INDEF\' ymax=\'INDEF\' ' message += 'ncols=\'INDEF\' ' message += 'nlines=\'INDEF\' verbose=\'no\' ' message += 'fluxconserve=\'yes\' nxblock=2048 ' message += 'nyblock=2048 interpolant=\'' + \ interp_type + '\' boundary=\'constant\' constant=0' log.message(message, with_stdout=verbose) yd, xd = tilehdu[ccd].data.shape ncols = 'INDEF' # ncols=xd+abs(xsh[ccd]/xbin) nlines = 'INDEF' # nlines=yd+abs(ysh[ccd]/ybin) geo_xshift = xsh[ccd] + (2 - ccd) * dxshift / xbin geo_yshift = ysh[ccd] / ybin iraf.images.immatch.geotran(tilefile + "[" + str(ccd) + "]", tranfile[hdu], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) if varframe: var_infile = tilefile + "[" + str(ccd + nccds) + "]" iraf.images.immatch.geotran(var_infile, tranfile[hdu + nccds], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) var2_infile = tilefile + "[" + str(ccd + 2 * nccds) + "]" iraf.images.immatch.geotran(var2_infile, tranfile[hdu + 2 * nccds], "", "", xshift=geo_xshift, yshift=geo_yshift, xrotation=xrot[ccd], yrotation=yrot[ccd], xmag=1, ymag=1, xmin='INDEF', xmax='INDEF', ymin='INDEF', ymax='INDEF', ncols=ncols, nlines=nlines, verbose='no', fluxconserve='yes', nxblock=2048, nyblock=2048, interpolant="linear", boundary="constant", constant=0) # open the file and copy the data to tranhdu tstruct = fits.open(tranfile[hdu]) tranhdu[hdu] = tstruct[0].data tstruct.close() if varframe: tranhdu[hdu + nccds] = fits.open(tranfile[hdu + nccds])[0].data tranhdu[hdu + 2 * nccds] = fits.open( tranfile[hdu + 2 * nccds])[0].data else: log.message("Transform CCD #%i using dx=%s, dy=%s, rot=%s" % (ccd, xsh[ccd] / 2.0, ysh[ccd] / 2.0, xrot[ccd]), with_stdout=verbose, with_header=False) tranhdu[hdu] = geometric_transform( tilehdu[ccd].data, tran_func, prefilter=False, order=1, extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) tstruct = fits.PrimaryHDU(tranhdu[hdu]) tstruct.writeto(tranfile[hdu]) if varframe: tranhdu[hdu + nccds] = geometric_transform( tilehdu[hdu + 3].data, tran_func, prefilter=False, order=1, extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) tranhdu[hdu + 2 * nccds] = geometric_transform( tilehdu[hdu + 6].data, tran_func, prefilter=False, order=1, extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1, xrot[ccd], yrot[ccd])) else: log.message("Transform CCD #%i using dx=%s, dy=%s, rot=%s" % (ccd, 0, 0, 0), with_stdout=verbose, with_header=False) tranhdu[hdu] = tilehdu[ccd].data if varframe: tranhdu[hdu + nccds] = tilehdu[ccd + nccds].data tranhdu[hdu + 2 * nccds] = tilehdu[ccd + 2 * nccds].data # open outfile if varframe: outlist = 4 * [None] else: outlist = 2 * [None] #outlist[0] = struct[0].copy() outlist[0] = fits.PrimaryHDU() outlist[0].header = struct[0].header naxis1 = int(gap / xbin * (nccds - 1)) naxis2 = 0 for i in range(1, nccds + 1): yw, xw = tranhdu[i].shape naxis1 += xw + int(abs(xsh[ccd] / xbin)) + 1 naxis2 = max(naxis2, yw) outdata = numpy.zeros((naxis2, naxis1), numpy.float32) outdata.shape = naxis2, naxis1 if varframe: vardata = outdata * 0 bpmdata = outdata * 0 + 1 # iterate over CCDs, stich them to produce a full image hdu = 0 totxshift = 0 for hdu in range(1, nccds + 1): # read DATASEC keywords ydsec, xdsec = tranhdu[hdu].shape # define size and shape of final image # tile CCDs to yield mosaiced image x1 = int((hdu - 1) * (xdsec + gap / xbin)) + int(totxshift) x2 = xdsec + x1 y1 = int(0) y2 = int(ydsec) outdata[y1:y2, x1:x2] = tranhdu[hdu] totxshift += int(abs(xsh[hdu] / xbin)) + 1 if varframe: vardata[y1:y2, x1:x2] = tranhdu[hdu + nccds] bpmdata[y1:y2, x1:x2] = tranhdu[hdu + 2 * nccds] # make sure to cover up all the gaps include bad areas if varframe: baddata = (outdata == 0) baddata = nd.maximum_filter(baddata, size=3) bpmdata[baddata] = 1 # fill in the gaps if requested if fill: if varframe: outdata = fill_gaps(outdata, 0) else: outdata = fill_gaps(outdata, 0) # add to the file outlist[1] = fits.ImageHDU(outdata) if varframe: outlist[2] = fits.ImageHDU(vardata, name='VAR') outlist[3] = fits.ImageHDU(bpmdata, name='BPM') # create the image structure outstruct = fits.HDUList(outlist) # update the head informaation # housekeeping keywords saltkey.put('NEXTEND', 2, outstruct[0]) saltkey.new('EXTNAME', 'SCI', 'Extension name', outstruct[1]) saltkey.new('EXTVER', 1, 'Extension number', outstruct[1]) if varframe: saltkey.new('VAREXT', 2, 'Variance frame extension', outstruct[1]) saltkey.new('BPMEXT', 3, 'BPM Extension', outstruct[1]) try: saltkey.copy(struct[1], outstruct[1], 'CCDSUM') except: pass # Add keywords associated with geometry saltkey.new('SGEOMGAP', gap, 'SALT Chip Gap', outstruct[0]) c1str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[0], yshift[0], rotation[0]) saltkey.new('SGEOM1', c1str, 'SALT Chip 1 Transform', outstruct[0]) c2str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[1], yshift[1], rotation[1]) saltkey.new('SGEOM2', c2str, 'SALT Chip 2 Transform', outstruct[0]) # WCS keywords saltkey.new('CRPIX1', 0, 'WCS: X reference pixel', outstruct[1]) saltkey.new('CRPIX2', 0, 'WCS: Y reference pixel', outstruct[1]) saltkey.new('CRVAL1', float(xbin), 'WCS: X reference coordinate value', outstruct[1]) saltkey.new('CRVAL2', float(ybin), 'WCS: Y reference coordinate value', outstruct[1]) saltkey.new('CDELT1', float(xbin), 'WCS: X pixel size', outstruct[1]) saltkey.new('CDELT2', float(ybin), 'WCS: Y pixel size', outstruct[1]) saltkey.new('CTYPE1', 'pixel', 'X type', outstruct[1]) saltkey.new('CTYPE2', 'pixel', 'Y type', outstruct[1]) # cleanup temporary files if cleanup: for tfile in tranfile: if os.path.isfile(tfile): saltio.delete(tfile) if os.path.isfile(tilefile): status = saltio.delete(tilefile) # return the file return outstruct
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None, geomfile=None,subover=True,trim=True,masbias=None, subbias=False, median=False, function='polynomial', order=5,rej_lo=3, rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='', clobber=False, cleanup=True, logfile='salt.log', verbose=True): """SALTADVANCE provides advanced data reductions for a set of data. It will sort the data, and first process the biases, flats, and then the science frames. It will record basic quality control information about each of the steps. """ plotover=False #start logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) infiles.sort() # create list of output files outpath=saltio.abspath(outpath) #log into the database sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password) #does the gain database file exist if gaindb: dblist= saltio.readgaindb(gaindb) else: dblist=[] # does crosstalk coefficient data exist if xtalkfile: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict=None #does the mosaic file exist--raise error if no saltio.fileexists(geomfile) # Delete the obslog file if it already exists if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile) #read in the obsveration log or create it if os.path.isfile(obslogfile): msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes' raise SaltError(msg) else: headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') obsmode=obsstruct.data.field('OBSMODE') ccdtype=obsstruct.data.field('CCDTYPE') propcode=obsstruct.data.field('PROPID') masktype=obsstruct.data.field('MASKTYP') #set the bias list of objects biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')] masterbias_dict={} for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=fits.open(img) bimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #create the list of flatfields and process them flatlist=filename[ccdtype=='FLAT'] masterflat_dict={} for img in infiles: if os.path.basename(img) in flatlist: #open the image struct=fits.open(img) fimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Flat frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,fimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list) #create the master flat frame for i in masterflat_dict.keys(): fkeys=masterflat_dict[i][0] flist=masterflat_dict[i][1:] mflatname=outpath+createmasterflatname(flist, fkeys) ffiles=','.join(flist) saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #process the arc data arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')] for i, img in enumerate(infiles): nimg=os.path.basename(img) if nimg in arclist: #open the image struct=fits.open(img) simg=outpath+'bxgp'+os.path.basename(img) obsdate=os.path.basename(img)[1:9] #print the message if log: message='Processing ARC frame %s' % img log.message(message, with_stdout=verbose) struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the images mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg) #measure the arcdata arcimage=outpath+'mbxgp'+nimg dbfile=outpath+obsdate+'_specid.db' lamp = obsstruct.data.field('LAMPID')[i] lamp = lamp.replace(' ', '') lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp) print arcimage, lampfile, os.getcwd() specidentify(arcimage, lampfile, dbfile, guesstype='rss', guessfile='', automethod='Matchlines', function='legendre', order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3, startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile, verbose=verbose) try: ximg = outpath+'xmbxgp'+os.path.basename(arcimage) specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, conserve=True, nearest=True, clobber=True, logfile=logfile, verbose=verbose) except: pass #process the science data for i, img in enumerate(infiles): nimg=os.path.basename(img) if not (nimg in flatlist or nimg in biaslist or nimg in arclist): #open the image struct=fits.open(img) if struct[0].header['PROPID'].count('CAL_GAIN'): continue simg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #Check to see if it is RSS 2x2 and add bias subtraction instrume=saltkey.get('INSTRUME', struct[0]).strip() gainset = saltkey.get('GAINSET', struct[0]) rospeed = saltkey.get('ROSPEED', struct[0]) target = saltkey.get('OBJECT', struct[0]).strip() exptime = saltkey.get('EXPTIME', struct[0]) obsmode = saltkey.get('OBSMODE', struct[0]).strip() detmode = saltkey.get('DETMODE', struct[0]).strip() masktype = saltkey.get('MASKTYP', struct[0]).strip() xbin, ybin = saltkey.ccdbin( struct[0], img) obsdate=os.path.basename(img)[1:9] bstruct=None crtype=None thresh=5 mbox=11 bthresh=5.0, flux_ratio=0.2 bbox=25 gain=1.0 rdnoise=5.0 fthresh=5.0 bfactor=2 gbox=3 maxiter=5 subbias=False if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW': bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin) if os.path.exists(bfile): bstruct=fits.open(bfile) subbias=True if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5: crtype='edge' thresh=5 mbox=11 bthresh=5.0, flux_ratio=0.2 bbox=25 gain=1.0 rdnoise=5.0 fthresh=5.0 bfactor=2 gbox=3 maxiter=3 #process the image struct=clean(struct, createvar=True, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \ bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise, bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter, log=log, verbose=verbose) #update the database updatedq(os.path.basename(img), struct, sdb) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the files--currently not in the proper format--will update when it is if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])): mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg) #if the file is spectroscopic mode, apply the wavelength correction if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT': dbfile=outpath+obsdate+'_specid.db' try: ximg = outpath+'xmbxgp'+os.path.basename(img) specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line', function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None, nw=None, blank=0.0, conserve=True, nearest=True, clobber=True, logfile=logfile, verbose=verbose) except Exception, e: log.message('%s' % e) #clean up the results if cleanup: #clean up the bias frames for i in masterbias_dict.keys(): blist=masterbias_dict[i][1:] for b in blist: saltio.delete(b) #clean up the flat frames for i in masterflat_dict.keys(): flist=masterflat_dict[i][1:] for f in flist: saltio.delete(f)
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None, geomfile=None,subover=True,trim=True,masbias=None, subbias=False, median=False, function='polynomial', order=5,rej_lo=3, rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log', verbose=True): """SALTCLEAN will provide basic CCD reductions for a set of data. It will sort the data, and first process the biases, flats, and then the science frames. It will record basic quality control information about each of the steps. """ plotover=False #start logging with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) # create list of output files outpath=saltio.abspath(outpath) #does the gain database file exist if gaindb: dblist= saltio.readgaindb(gaindb) else: dblist=[] # does crosstalk coefficient data exist if xtalkfile: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict=None #does the mosaic file exist--raise error if no saltio.fileexists(geomfile) # Delete the obslog file if it already exists if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile) #read in the obsveration log or create it if os.path.isfile(obslogfile): msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes' raise SaltError(msg) else: headerDict=obslog(infiles, log) obsstruct=createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) #create the list of bias frames and process them filename=obsstruct.data.field('FILENAME') detmode=obsstruct.data.field('DETMODE') ccdtype=obsstruct.data.field('CCDTYPE') #set the bias list of objects biaslist=filename[ccdtype=='ZERO'] masterbias_dict={} for img in infiles: if os.path.basename(img) in biaslist: #open the image struct=pyfits.open(img) bimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Zero frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys=masterbias_dict[i][0] blist=masterbias_dict[i][1:] mbiasname=outpath+createmasterbiasname(blist, bkeys) bfiles=','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #create the list of flatfields and process them flatlist=filename[ccdtype=='FLAT'] masterflat_dict={} for img in infiles: if os.path.basename(img) in flatlist: #open the image struct=pyfits.open(img) fimg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing Flat frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,fimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list) #create the master flat frame for i in masterflat_dict.keys(): fkeys=masterflat_dict[i][0] flist=masterflat_dict[i][1:] mflatname=outpath+createmasterflatname(flist, fkeys) ffiles=','.join(flist) saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #process the science data for img in infiles: nimg=os.path.basename(img) if not nimg in flatlist or not nimg in biaslist: #open the image struct=pyfits.open(img) simg=outpath+'bxgp'+os.path.basename(img) #print the message if log: message='Processing science frame %s' % img log.message(message, with_stdout=verbose) #process the image struct=clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0]) saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0]) # write FITS file saltio.writefits(struct,simg, clobber=clobber) saltio.closefits(struct) #mosaic the files--currently not in the proper format--will update when it is if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])): mimg=outpath+'mbxgp'+os.path.basename(img) saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile, interp=interp,cleanup=True,clobber=clobber,logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg)
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None, subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log', verbose=True): """Convert MEF HRS data into a single image. If variance frames and BPMs, then convert them to the same format as well. Returns an MEF image but that is combined into a single frame """ with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outpath = saltio.abspath(outpath) if saltio.checkfornone(obslogfile) is None: raise SaltError('Obslog file is required') # Delete the obslog file if it already exists if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile): if os.path.isfile(obslogfile): saltio.delete(obslogfile) #read in the obsveration log or create it headerDict = obslog(infiles, log) obsstruct = createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) else: obsstruct = saltio.openfits(obslogfile) #create the list of bias frames and process them filename = obsstruct.data.field('FILENAME') detmode = obsstruct.data.field('DETMODE') ccdtype = obsstruct.data.field('OBJECT') biaslist = filename[ccdtype == 'Bias'] masterbias_dict = {} if log: log.message('Processing Bias Frames') for img in infiles: if os.path.basename(img) in biaslist: #open the image struct = pyfits.open(img) bimg = outpath + 'bgph' + os.path.basename(img) #print the message if log: message = 'Processing Zero frame %s' % img log.message(message, with_stdout=verbose, with_header=False) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=False, imstack=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict = compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys = masterbias_dict[i][0] blist = masterbias_dict[i][1:] mbiasname = outpath + createmasterbiasname( blist, bkeys, x1=5, x2=13) bfiles = ','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #apply full reductions to the science data for img in infiles: nimg = os.path.basename(img) if not nimg in biaslist: #open the image struct = pyfits.open(img) simg = outpath + 'mbgph' + os.path.basename(img) #print the message if log: message = 'Processing science frame %s' % img log.message(message, with_stdout=verbose) #get master bias frame masterbias = get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list) if masterbias: subbias = True bstruct = saltio.openfits(masterbias) else: subbias = False bstruct = None #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, subover=subover, trim=trim, subbias=subbias, imstack=True, bstruct=bstruct, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'HPREPARE', 'Images have been prepared', hist) saltkey.new('HGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) #saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0]) saltkey.new('HBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, simg, clobber=clobber) saltio.closefits(struct) return
def saltclean(images, outpath, obslogfile=None, gaindb=None, xtalkfile=None, geomfile=None, subover=True, trim=True, masbias=None, subbias=False, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, interp='linear', clobber=False, logfile='salt.log', verbose=True): """SALTCLEAN will provide basic CCD reductions for a set of data. It will sort the data, and first process the biases, flats, and then the science frames. It will record basic quality control information about each of the steps. """ plotover = False #start logging with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) # create list of output files outpath = saltio.abspath(outpath) #does the gain database file exist if gaindb: dblist = saltio.readgaindb(gaindb) else: dblist = [] # does crosstalk coefficient data exist if xtalkfile: xtalkfile = xtalkfile.strip() xdict = saltio.readxtalkcoeff(xtalkfile) else: xdict = None #does the mosaic file exist--raise error if no saltio.fileexists(geomfile) # Delete the obslog file if it already exists if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile) #read in the obsveration log or create it if os.path.isfile(obslogfile): msg = 'The observing log already exists. Please either delete it or run saltclean with clobber=yes' raise SaltError(msg) else: headerDict = obslog(infiles, log) obsstruct = createobslogfits(headerDict) saltio.writefits(obsstruct, obslogfile) #create the list of bias frames and process them filename = obsstruct.data.field('FILENAME') detmode = obsstruct.data.field('DETMODE') ccdtype = obsstruct.data.field('CCDTYPE') #set the bias list of objects biaslist = filename[ccdtype == 'ZERO'] masterbias_dict = {} for img in infiles: if os.path.basename(img) in biaslist: #open the image struct = fits.open(img) bimg = outpath + 'bxgp' + os.path.basename(img) #print the message if log: message = 'Processing Zero frame %s' % img log.message(message, with_stdout=verbose) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) saltkey.new('SXTALK', time.asctime(time.localtime()), 'Images have been xtalk corrected', struct[0]) saltkey.new('SBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, bimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterbias_dict = compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list) #create the master bias frame for i in masterbias_dict.keys(): bkeys = masterbias_dict[i][0] blist = masterbias_dict[i][1:] mbiasname = outpath + createmasterbiasname(blist, bkeys) bfiles = ','.join(blist) saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #create the list of flatfields and process them flatlist = filename[ccdtype == 'FLAT'] masterflat_dict = {} for img in infiles: if os.path.basename(img) in flatlist: #open the image struct = fits.open(img) fimg = outpath + 'bxgp' + os.path.basename(img) #print the message if log: message = 'Processing Flat frame %s' % img log.message(message, with_stdout=verbose) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) saltkey.new('SXTALK', time.asctime(time.localtime()), 'Images have been xtalk corrected', struct[0]) saltkey.new('SBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, fimg, clobber=clobber) saltio.closefits(struct) #add files to the master bias list masterflat_dict = compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list) #create the master flat frame for i in masterflat_dict.keys(): fkeys = masterflat_dict[i][0] flist = masterflat_dict[i][1:] mflatname = outpath + createmasterflatname(flist, fkeys) ffiles = ','.join(flist) saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False, weight=False, blank=0, scale=None, statsec=None, lthresh=3, \ hthresh=3, clobber=False, logfile=logfile,verbose=verbose) #process the science data for img in infiles: nimg = os.path.basename(img) #print nimg, nimg in flatlist, nimg in biaslist if not (nimg in biaslist): #open the image struct = fits.open(img) simg = outpath + 'bxgp' + os.path.basename(img) #print the message if log: message = 'Processing science frame %s' % img log.message(message, with_stdout=verbose) #process the image struct = clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False, bstruct=None, median=median, function=function, order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log, verbose=verbose) #write the file out # housekeeping keywords fname, hist = history( level=1, wrap=False, exclude=['images', 'outimages', 'outpref']) saltkey.housekeeping(struct[0], 'SPREPARE', 'Images have been prepared', hist) saltkey.new('SGAIN', time.asctime(time.localtime()), 'Images have been gain corrected', struct[0]) saltkey.new('SXTALK', time.asctime(time.localtime()), 'Images have been xtalk corrected', struct[0]) saltkey.new('SBIAS', time.asctime(time.localtime()), 'Images have been de-biased', struct[0]) # write FITS file saltio.writefits(struct, simg, clobber=clobber) saltio.closefits(struct) #mosaic the files--currently not in the proper format--will update when it is if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])): mimg = outpath + 'mbxgp' + os.path.basename(img) saltmosaic(images=simg, outimages=mimg, outpref='', geomfile=geomfile, interp=interp, cleanup=True, clobber=clobber, logfile=logfile, verbose=verbose) #remove the intermediate steps saltio.delete(simg)
def slotback(images, outfits, extension, imgtype='image', subbacktype='median', sigback=3, mbin=7, sorder=3, niter=5, ampperccd=2, ignorexp=6, clobber=False, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # set up the variables order = sorder plotfreq = 0.01 ftime = plotfreq # is the input file specified? infiles = saltio.argunpack('Input', images) # is the output file specified? saltio.filedefined('Output', outfits) #open the first file and check its charactistics struct = saltio.openfits(infiles[0]) # how many extensions? nextend = saltkey.get('NEXTEND', struct[0]) if nextend < extension: msg = 'Insufficient number of extensions in %s' % (infile) raise SaltIOError(msg) # how many amplifiers? amplifiers = saltkey.get('NCCDS', struct[0]) amplifiers = int(ampperccd * float(amplifiers)) if ampperccd > 0: nframes = nextend / amplifiers nstep = amplifiers else: nframes = nextend nstep = 1 ntotal = nframes * len(infiles) # image size naxis1 = saltkey.get('NAXIS1', struct[extension]) naxis2 = saltkey.get('NAXIS2', struct[extension]) # CCD binning ccdsum = saltkey.get('CCDSUM', struct[0]) binx = int(ccdsum.split(' ')[0]) biny = int(ccdsum.split(' ')[1]) # If a total file is to written out, create it and update it hdu = 1 # delete the file if clobber is on and the file exists if os.path.isfile(outfits): if clobber: saltio.delete(outfits) else: raise SaltIOError( 'Output fits file ' + writenewfits + 'already exists. Use Clobber=yes to overwrite file') # create the output file try: hdu = pyfits.PrimaryHDU() hdu.header = struct[0].header hdu.header['NCCDS'] = 1 hdu.header['NSCIEXT'] = ntotal - ignorexp hdu.header['NEXTEND'] = ntotal - ignorexp hduList = pyfits.HDUList(hdu) hduList.verify() hduList.writeto(outfits) except: raise SaltIOError('Could not create new fits file named ' + writenewfits) # read it back in for updating hduList = saltio.openfits(outfits, mode='update') # set up the completeness tracker if verbose: j = 0 x2 = float(j) / float(ntotal) ctext = 'Percentage Complete: %3.2f' % x2 sys.stdout.write(ctext) sys.stdout.flush() for infile in infiles: struct = saltio.openfits(infile) # Skip through the frames and process each frame individually for i in range(nframes): if not (infile == infiles[0] and i < ignorexp): hdu = extension + i * nstep try: header = struct[hdu].header array = struct[hdu].data array = array * 1.0 except Exception, e: msg = 'Unable to open extension %i in image %s because %s' % ( hdu, infile, e) raise SaltIOError(msg) # start the analysis of each frame # gain and readout noise try: gain = float(header['GAIN']) except: gain = 1 log.warning('Gain not specified in image header') try: rdnoise = float(header['RDNOISE']) except: rdnoise = 0 log.warning('RDNOISE not specified in image header') # background subtraction if not subbacktype == 'none': try: narray = subbackground(array, sigback, mbin, order, niter, subbacktype) except: log.warning('Image ' + infile + ' extention ' + str(i) + ' is blank, skipping') continue # create output array try: if imgtype == 'background': oarray = narray - array else: oarray = narray except: oarray = array # print progress if verbose: x2 = float(j) / float(ntotal) if x2 > ftime: ctext = '\b\b\b\b\b %3.2f' % x2 sys.stdout.write(ctext) sys.stdout.flush() ftime += plotfreq # update the header values with the image name and extension number try: hdue = pyfits.ImageHDU(oarray) hdue.header = header hdue.header.update('ONAME', infile, 'Original image name') hdue.header.update('OEXT', hdu, 'Original extension number') except Exception, e: msg='SALTPHOT--WARNING: Could not update image in newfits file for %s ext %i because %s' \ % (infile, hdu, e) raise SaltIOError(msg) hduList.append(hdue) j += 1 # close FITS file saltio.closefits(struct)
def saltfpringfind(images, method=None, section=None, thresh=5, minsize=10, niter=5, conv=0.05, displayimage=True, clobber=False, logfile='salt.log', verbose=True): with logging(logfile, debug) as log: # Check the input images infiles = saltio.argunpack('Input', images) #check the method method = saltio.checkfornone(method) # read in the section section = saltio.checkfornone(section) if section is None: pass else: section = saltio.getSection(section) # open each raw image file for img in infiles: #open the fits file struct = saltio.openfits(img) data = struct[0].data #determine the background value for the image if section is None: #if section is none, just use all pixels greater than zero bdata = data[data > 0] else: y1, y2, x1, x2 = section bdata = data[y1:y2, x1:x2] bmean, bmedian, bstd = iterstat(bdata, sig=thresh, niter=niter, verbose=False) message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \ ('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd) log.message(message, with_stdout=verbose) mdata = data * (data - bmean > thresh * bstd) #prepare the first guess for the image ring_list = findrings(data, thresh=thresh, niter=niter, minsize=minsize) #if specified, find the center of the ring if method is not None: for i in range(len(ring_list)): ring_list[i] = findcenter(data, ring_list[i], method, niter=niter, conv=conv) #if one peak: no rings. If two peaks: one ring, if two peaks: four rings if len(ring_list) == 1: msg = "One ring dected in image" else: msg = "%i rings found in image" % len(ring_list) log.message(message, with_stdout=verbose) if displayimage: regfile = img.replace('.fits', '.reg') if clobber and os.path.isfile(regfile): fout = saltio.delete(regfile) fout = open(regfile, 'w') fout.write("""# Region file format: DS9 version 4.1 # Filename: %s global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 physical """ % img) for ring in ring_list: fout.write('circle(%f, %f, %f)\n' % (ring.xc, ring.yc, ring.prad)) fout.write('circle(%f, %f, %f)\n' % (ring.xc, ring.yc, ring.prad - 3 * ring.sigma)) fout.write('circle(%f, %f, %f)\n' % (ring.xc, ring.yc, ring.prad + 3 * ring.sigma)) fout.close() display(img, catname=regfile, rformat='reg') message = 'Ring Parameters\n%30s %6s %6s %6s\n' % ('Image', 'XC', 'YC', 'Radius') log.message(message, with_stdout=verbose) for ring in ring_list: msg = '%30s %6.2f %6.2f %6.2f\n' % (img, ring.xc, ring.yc, ring.prad) log.message(msg, with_header=False, with_stdout=verbose)
def saltfpringfind(images, method=None, section=None, thresh=5, minsize=10, niter=5, conv=0.05, displayimage=True, clobber=False, logfile='salt.log',verbose=True): with logging(logfile,debug) as log: # Check the input images infiles = saltio.argunpack ('Input',images) #check the method method=saltio.checkfornone(method) # read in the section section=saltio.checkfornone(section) if section is None: pass else: section=saltio.getSection(section) # open each raw image file for img in infiles: #open the fits file struct=saltio.openfits(img) data=struct[0].data #determine the background value for the image if section is None: #if section is none, just use all pixels greater than zero bdata=data[data>0] else: y1,y2,x1,x2=section bdata=data[y1:y2,x1:x2] bmean, bmedian, bstd=iterstat(bdata, sig=thresh, niter=niter, verbose=False) message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \ ('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd) log.message(message, with_stdout=verbose) mdata=data*(data-bmean>thresh*bstd) #prepare the first guess for the image ring_list=findrings(data, thresh=thresh, niter=niter, minsize=minsize) #if specified, find the center of the ring if method is not None: for i in range(len(ring_list)): ring_list[i]=findcenter(data, ring_list[i], method, niter=niter, conv=conv) #if one peak: no rings. If two peaks: one ring, if two peaks: four rings if len(ring_list)==1: msg="One ring dected in image" else: msg="%i rings found in image" % len(ring_list) log.message(message, with_stdout=verbose) if displayimage: regfile=img.replace('.fits', '.reg') if clobber and os.path.isfile(regfile): fout=saltio.delete(regfile) fout=open(regfile, 'w') fout.write("""# Region file format: DS9 version 4.1 # Filename: %s global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1 physical """ % img) for ring in ring_list: fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad)) fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad-3*ring.sigma)) fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad+3*ring.sigma)) fout.close() display(img, catname=regfile, rformat='reg') message = 'Ring Parameters\n%30s %6s %6s %6s\n' % ('Image', 'XC', 'YC', 'Radius') log.message(message, with_stdout=verbose) for ring in ring_list: msg='%30s %6.2f %6.2f %6.2f\n' % (img, ring.xc, ring.yc, ring.prad) log.message(msg, with_header=False, with_stdout=verbose)
def slotphot(images,outfile,srcfile,newfits=None,phottype='square', subbacktype='median',sigback=3,mbin=7,sorder=3,niter=5,sigdet=5, contpix=10,ampperccd=2,ignorexp=6,driftlimit=10.,finddrift=True, outtype='ascii',reltime=True,clobber=True,logfile='salt.log', verbose=True): """Perform photometry on listed SALT slotmode *images*.""" with logging(logfile,debug) as log: # set up the variables entries = [] vig_lo = {} vig_hi = {} amp = {} x = {} y = {} x_o = {} y_o = {} r = {} br1 = {} br2 = {} hour = 0 min = 0 sec = 0. time0 = 0. nframes = 0 bin=mbin order=sorder # is the input file specified? saltsafeio.filedefined('Input',images) # if the input file is a list, does it exist? if images[0] == '@': saltsafeio.listexists('Input',images) # parse list of input files infiles=saltsafeio.listparse('Raw image',images,'','','') # check input files exist saltsafeio.filesexist(infiles,'','r') # is the output file specified? saltsafeio.filedefined('Output',outfile) # check output file does not exist, optionally remove it if it does exist if os.path.exists(outfile) and clobber: os.remove(outfile) elif os.path.exists(outfile) and not clobber: raise SaltIOError('File '+outfile+' already exists, use clobber=y') # open output ascii file if outtype=='ascii': try: lc = open(outfile,'a') except: raise SaltIOError('Cannot open ouput file '+outfile) # is the extraction region defintion file specified? saltsafeio.filedefined('Extraction region defintion',srcfile) # check extraction region defintion file exists srcfile = srcfile.strip() saltsafeio.fileexists(srcfile) # read extraction region defintion file amp, x, y, x_o, y_o, r, br1, br2=slottool.readsrcfile(srcfile) # set the writenewfits parameter if not newfits or newfits=='none': writenewfits=False else: writenewfits=newfits # get time of first exposure and basic information about the observations infile=infiles[0] struct=saltsafeio.openfits(infile) # identify instrument instrume,keyprep,keygain,keybias,keyxtalk,keyslot=saltsafekey.instrumid(struct,infile) # how many extensions? nextend=saltsafekey.get('NEXTEND',struct[0],infile) if nextend < amp['comparison']: msg='Insufficient number of extensions in %s' % (infile) raise SaltIOError(msg) # how many amplifiers? amplifiers=saltsafekey.get('NCCDS',struct[0],infile) amplifiers = int(ampperccd*float(amplifiers)) if ampperccd>0: nframes = int(nextend/amplifiers) nstep=amplifiers else: nframes = nextend nstep=1 ntotal=nframes*len(infiles) # image size naxis1=saltsafekey.get('NAXIS1',struct[amp['comparison']],infile) naxis2=saltsafekey.get('NAXIS2',struct[amp['comparison']],infile) # CCD binning ccdsum=saltsafekey.get('CCDSUM',struct[0],infile) binx=int(ccdsum.split(' ')[0]) biny=int(ccdsum.split(' ')[1]) # Identify the time of the observations ext = 1 try: time0=slottool.getobstime(struct[ext], infile+'['+str(ext)+']') dateobs=saltsafekey.get('DATE-OBS',struct[ext],infile) dateobs=dateobs.replace('-','/') except: raise SaltIOError('No time or obsdate in first image') # If a total file is to be written out, create it and update it if writenewfits: if os.path.isfile(writenewfits): if clobber: saltsafeio.delete(writenewfits) else: raise SaltIOError('Newfits file exists, use clobber') try: hdu=pyfits.PrimaryHDU() hdu.header=struct[0].header hdu.header['NCCDS']=1 hdu.header['NSCIEXT']=ntotal-ignorexp hdu.header['NEXTEND']=ntotal-ignorexp hduList=pyfits.HDUList(hdu) hduList.verify() hduList.writeto(writenewfits) except: raise SaltIOError('Could not create newfits file, '+writenewfits) # Close image file saltsafeio.closefits(struct) # Read newfits file back in for updating if writenewfits: try: hduList=pyfits.open(writenewfits,mode='update') except: raise SaltIOError('Cannot open newfits file '+writenewfits+' for updating.') # set up the arrays j=0 time=np.zeros(ntotal ,dtype='float')-1.0 dx=np.zeros(ntotal ,dtype='float') dy=np.zeros(ntotal ,dtype='float') tflux=np.zeros(ntotal ,dtype='float') terr =np.zeros(ntotal ,dtype='float') cflux=np.zeros(ntotal ,dtype='float') cerr =np.zeros(ntotal ,dtype='float') ratio=np.zeros(ntotal ,dtype='float') rerr =np.zeros(ntotal ,dtype='float') tgt_x=np.zeros(ntotal ,dtype='float') tgt_y=np.zeros(ntotal ,dtype='float') cmp_x=np.zeros(ntotal ,dtype='float') cmp_y=np.zeros(ntotal ,dtype='float') p_one=100./ntotal # One percent p_old=-1 # Previous completed percentage p_new=0 # New completed percentage p_n=1 # Counter number for infile in infiles: # Log if verbose: log.message('Starting photometry on file '+infile, with_stdout=False) struct=pyfits.open(infile) # Skip through the frames and process each frame individually for i in range(nframes): # Show progress if verbose: p_new=int(p_n*p_one) p_n+=1 if p_new!=p_old: ctext='Percentage Complete: %d\r' % p_new sys.stdout.write(ctext) sys.stdout.flush() p_old=p_new if not (infile==infiles[0] and i < ignorexp): ext=amp['comparison']+i*nstep try: header=struct[ext].header array=struct[ext].data array=array*1.0 except: msg='Unable to open extension %i in image %s' % (ext, infile) raise SaltIOError(msg) # starti the analysis of each frame # get the time time[j]=slottool.getobstime(struct[ext],infile+'['+str(ext)+']') # gain and readout noise try: gain=float(header['GAIN']) except: gain=1 raise SaltIOError('Gain not specified in image header') try: rdnoise=float(header['RDNOISE']) except: rdnoise=0 raise SaltIOError('RDNOISE not specified in image header') # background subtraction if not subbacktype=='none': try: array=subbackground(array, sigback, bin, order, niter, subbacktype) except SaltError: log.warning('Image '+infile+' extention '+str(ext)+' is blank, skipping') continue # x-y fit to the comparison star and update the x,y values if finddrift: carray, fx,fy=slottool.finddrift(array, x['comparison'], y['comparison'], r['comparison'], naxis1, naxis2, sigdet, contpix, sigback, driftlimit, niter) if fx > -1 and fy > -1: if fx < naxis1 and fy < naxis2: dx[j]=x['comparison']-fx dy[j]=y['comparison']-fy x['comparison']=fx y['comparison']=fy x['target']=x['target']-dx[j] y['target']=y['target']-dy[j] else: dx[j]=0 dy[j]=0 x['comparison']=x_o['comparison'] y['comparison']=y_o['comparison'] x['target']=x_o['target'] y['target']=y_o['target'] else: msg='No comparison object found in image file ' + infile+' on extension %i skipping.' % ext log.warning(msg) pass # do photometry try: tflux[j],terr[j],cflux[j],cerr[j],ratio[j],rerr[j]=slottool.dophot(phottype, array, x, y, r, br1, br2, gain, rdnoise, naxis1, naxis2) except SaltError, e: msg='Could not do photometry on extension %i in image %s because %s skipping.' % (ext, infile, e) log.warning(msg) tgt_x[j]=x['target'] tgt_y[j]=y['target'] cmp_x[j]=x['comparison'] cmp_y[j]=y['comparison'] # record results # TODO! This should be removed in favor of the write all to disk in the end if outtype=='ascii': slottool.writedataout(lc, j+1, time[j], x, y, tflux[j], terr[j], cflux[j],cerr[j],ratio[j],rerr[j],time0, reltime) # write newfits file if writenewfits: # add original name and extension number to header try: hdue=pyfits.ImageHDU(array) hdue.header=header hdue.header.update('ONAME',infile,'Original image name') hdue.header.update('OEXT',ext,'Original extension number') hduList.append(hdue) except: log.warning('Could not update image in newfits '+infile+' '+str(ext)) # increment counter j+=1 # close FITS file saltsafeio.closefits(struct) # close newfits file if writenewfits: try: hduList.flush() hduList.close() except: raise SaltIOError('Cannot close newfits file.') # write to output if outtype=='ascii': # close output ascii file try: lc.close() except: raise SaltIOError('Cannot close ouput file ' + outfile) elif outtype=='fits': print 'writing fits' try: c1=pyfits.Column(name='index',format='D',array=np.arange(ntotal)) if reltime: c2=pyfits.Column(name='time',format='D',array=time-time0) else: c2=pyfits.Column(name='time',format='D',array=time) c3=pyfits.Column(name='tgt_x',format='D',array=tgt_x) c4=pyfits.Column(name='tgt_y',format='D',array=tgt_y) c5=pyfits.Column(name='tgt_flux',format='D',array=tflux) c6=pyfits.Column(name='tgt_err',format='D',array=terr) c7=pyfits.Column(name='cmp_x',format='D',array=cmp_x) c8=pyfits.Column(name='cmp_y',format='D',array=cmp_y) c9=pyfits.Column(name='cmp_flux',format='D',array=cflux) c10=pyfits.Column(name='cmp_err',format='D',array=cerr) c11=pyfits.Column(name='flux_ratio',format='D',array=ratio) c12=pyfits.Column(name='flux_ratio_err',format='D',array=rerr) tbhdu=pyfits.new_table([c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12]) # Add header information tbhdu.header.update('RELTIME',str(reltime),'Time relative to first datapoint or absolute.') tbhdu.writeto(outfile) print 'fits written to ',outfile except: raise SaltIOError('Could not write to fits table.')