def clean(struct, createvar=False, badpixelstruct=None, mult=True, \
subover=True, trim=True, subbias=False, bstruct=None, imstack=False,
median=False, \
function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, \
log=None, verbose=True):
"""Clean HRS data and files. This includes the following steps currently:
* overscan correction
* bias subtraction
* gain correction
* mosaic correction
"""
infile = struct
tfile = struct[0]._file
#prepare the HRS files
struct = hrsprepare(struct)
struct[0]._file = tfile
ampccd = len(struct) - 1
#prepare the files
struct = prepare(struct,
createvar=createvar,
badpixelstruct=badpixelstruct,
namps=ampccd)
#gain correct the files
usedb = False
struct = gain(struct,
mult=mult,
usedb=usedb,
dblist=None,
ampccd=ampccd,
log=log,
verbose=verbose)
#overscan and bias correction
struct = bias(struct,
subover=subover,
trim=trim,
subbias=subbias,
bstruct=bstruct,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=False,
log=log,
verbose=verbose)
#stack the data if requested
if imstack and ampccd > 1:
struct = stack(struct)
#struct=salt2iraf(struct)
return struct
def clean(struct, createvar=False, badpixelstruct=None, mult=True, dblist=None, ampccd=2,
xdict=[], subover=True,trim=True, subbias=False, bstruct=None,
median=False, function='polynomial',order=3,rej_lo=3,rej_hi=3,niter=10,
crtype=None,thresh=5,mbox=3, bbox=11, \
bthresh=3, flux_ratio=0.2, gain=1, rdnoise=5, bfactor=2, fthresh=5,\
gbox=3, maxiter=5, \
plotover=False, log=None, verbose=True):
infile=struct
#prepare the files
struct=prepare(struct, createvar=False, badpixelstruct=None)
#bias correct the file
struct=bias(struct,subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function,
order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter,
plotover=plotover, log=log, verbose=verbose)
#reset the names in the structures
for i in range(1,len(struct)):
struct[i].name=struct[i].header['EXTNAME']
#add the variance and bad pixel frame
nextend=len(struct)-1
nsciext=nextend
if createvar:
#create the inv. variance frames
for i in range(1, nsciext+1):
hdu=CreateVariance(struct[i], i, nextend+i)
try:
pass
except Exception, e:
msg='Cannot create variance frame in extension %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header['VAREXT'] = (nextend+i, 'Extension for Variance Frame')
struct.append(hdu)
nextend+=nsciext
#create the badpixelframes
for i in range(1, nsciext+1):
try:
hdu=createbadpixel(struct, badpixelstruct, i, nextend+i)
except Exception, e:
msg='Could not create bad pixel extension in ext %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header['BPMEXT'] = (nextend+i, 'Extension for Bad Pixel Mask')
struct.append(hdu)
def clean(struct, createvar=False, badpixelstruct=None, mult=True, \
subover=True, trim=True, subbias=False, bstruct=None, imstack=False,
median=False, \
function='polynomial', order=5, rej_lo=3, rej_hi=3, niter=5, \
log=None, verbose=True):
"""Clean HRS data and files. This includes the following steps currently:
* overscan correction
* bias subtraction
* gain correction
* mosaic correction
"""
infile=struct
tfile=struct[0]._file
#prepare the HRS files
struct=hrsprepare(struct)
struct[0]._file=tfile
ampccd=len(struct)-1
#prepare the files
struct=prepare(struct, createvar=createvar, badpixelstruct=badpixelstruct, namps=ampccd)
#gain correct the files
usedb=False
struct=gain(struct, mult=mult,usedb=usedb, dblist=None, ampccd=ampccd, log=log, verbose=verbose)
#overscan and bias correction
struct=bias(struct,subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function,
order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter,
plotover=False, log=log, verbose=verbose)
#stack the data if requested
if imstack and ampccd>1:
struct=stack(struct)
#struct=salt2iraf(struct)
return struct
if log: log.warning(msg)
xcoeff = xdict[xdict.keys()[-1]]
else:
xcoeff = []
struct = xtalk(struct, xcoeff, log=log, verbose=verbose)
#bias correct the files
if saltkey.fastmode(saltkey.get('DETMODE', struct[0])): order = 1
struct = bias(struct,
subover=subover,
trim=trim,
subbias=subbias,
bstruct=bstruct,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#mosaic correct the files
return struct
def createmasterbiasname(infiles, biaskeys):
"""Create the name for the master bias file based on its parameters. The format for
hte name is
def imred(infilelist, prodir, bpmfile=None, crthresh='', gaindb = None, cleanup=True):
#get the name of the files
infiles=','.join(['%s' % x for x in infilelist])
#get the current date for the files
obsdate=os.path.basename(infilelist[0])[1:9]
print "Observation Date: ",obsdate
#set up some files that will be needed
logfile='im'+obsdate+'.log'
flatimage='FLAT%s.fits' % (obsdate)
dbfile='spec%s.db' % obsdate
geomline = datedline(datadir+'RSSgeom.dat',obsdate)
if len(geomline) == 0:
print 'Invalid geometry file, ',datadir+'RSSgeom.dat',', exitting'
exit()
geomfile=obsdate+'_geom.txt'
open(geomfile,'w').write(geomline)
#create the observation log
# obs_dict=obslog(infilelist)
verbose=True
with logging(logfile, debug) as log:
log.message('Pysalt Version: '+pysalt.verno, with_header=False)
#prepare the data
for img in infilelist:
hdu = pyfits.open(img)
# for backwards compatibility
hdu = remove_duplicate_keys(hdu)
if not 'XTALK' in hdu[1].header:
hdu[1].header['XTALK']=1474
hdu[2].header['XTALK']=1474
hdu[3].header['XTALK']=1166
hdu[4].header['XTALK']=1111
hdu[5].header['XTALK']=1377
hdu[6].header['XTALK']=1377
img = os.path.basename(img)
hdu = prepare(hdu, createvar=False, badpixelstruct=None)
if not cleanup: hdu.writeto('p'+img, overwrite=True)
hdu = bias(hdu,subover=True, trim=True, subbias=False,
bstruct=None, median=False, function='polynomial',
order=5, rej_lo=5.0, rej_hi=5.0, niter=10,
plotover=False, log=log, verbose=verbose)
if not cleanup: hdu.writeto('bp'+img, overwrite=True)
# put windowed data into full image
exts = len(hdu)
if exts > 7:
rows, cols = hdu[1].data.shape
cbin, rbin = [int(x) for x in hdu[0].header['CCDSUM'].split(" ")]
ampsecO = hdu[1].header["AMPSEC"].strip("[]").split(",")
ampsecE = hdu[7].header["AMPSEC"].strip("[]").split(",")
rO = int((float(ampsecO[1].split(":")[0]) - 1.)/rbin)
rE = int((float(ampsecE[1].split(":")[0]) - 1.)/rbin)
keylist = ['BIASSEC','DATASEC','AMPSEC','CCDSEC','DETSEC']
oldlist = [hdu[1].header[key].strip("[]").split(",")[1] for key in keylist]
newlist = 2*['1:'+str(int(0.5+4102/rbin))]+3*[str(int(rbin/2))+':4102']
for amp in range(6):
hduO = hdu[amp+1].copy()
hdu[amp+1].data = np.zeros((4102/rbin,cols))
hdu[amp+1].data[rO:rO+rows] = hduO.data
hdu[amp+1].data[rE:rE+rows] = hdu[amp+7].data
hdu[amp+1].update_header
for k,key in enumerate(keylist):
hdu[amp+1].header[key] = \
hdu[amp+1].header[key].replace(oldlist[k],newlist[k])
del hdu[7:]
hdu[0].header['NSCIEXT'] = 6
badpixelstruct = saltio.openfits(bpmfile)
hdu = add_variance(hdu, badpixelstruct)
#gain correct the data
if gaindb:
usedb = True
dblist = saltio.readgaindb(gaindb.strip())
else:
usedb = False
dblist = ''
hdu = gain(hdu, mult=True, usedb=usedb, dblist=dblist, log=log, verbose=verbose)
if not cleanup: hdu.writeto('gbp'+img, overwrite=True)
#cross talk correct the data
hdu=xtalk(hdu, [], log=log, verbose=verbose)
#cosmic ray clean the data
#only clean the object data
if crthresh=='':
thresh = 5.0
if hdu[0].header['GRATING'].strip()=='PG0300': thresh = 7.0
else: thresh=crthresh
if hdu[0].header['CCDTYPE']=='OBJECT' and \
hdu[0].header['LAMPID']=='NONE' and \
hdu[0].header['INSTRUME']=='RSS':
if crthresh != False:
log.message('Cleaning CR using thresh={}'.format(thresh))
hdu = multicrclean(hdu, crtype='edge', thresh=thresh, mbox=11, bthresh=5.0,
flux_ratio=0.2, bbox=25, gain=1.0, rdnoise=5.0, fthresh=5.0, bfactor=2,
gbox=3, maxiter=5, log=log, verbose=verbose)
for ext in range(13,19): hdu[ext].data = hdu[ext].data.astype('uint8')
hdu[0].header.add_history('CRCLEAN: multicrclean, thresh = ',thresh)
else:
hdu[0].header.add_history('CRCLEAN: None')
hdu.writeto('xgbp'+img, overwrite=True)
hdu.close()
#mosaic the data
#khn: attempt to use most recent previous geometry to obsdate.
#geomfile=iraf.osfn("pysalt$data/rss/RSSgeom.dat")
try:
saltmosaic('xgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
except:
saltmosaic('xgbpP*fits', '', 'm', geomfile, interp='linear', cleanup=True, geotran=True, clobber=True, logfile=logfile, verbose=True)
#khn: fix mosaiced VAR and BPM extensions
#khn: fix mosaiced bpm missing some of gap
for img in infilelist:
filename = 'mxgbp'+os.path.basename(img)
hdu = pyfits.open(filename, 'update')
hdu[2].header['EXTNAME'] = 'VAR'
hdu[3].header['EXTNAME'] = 'BPM'
bpm_rc = (hdu[3].data>0).astype('uint8')
zeroscicol = hdu['SCI'].data.sum(axis=0) == 0
bpmgapcol = bpm_rc.mean(axis=0) == 1
addbpmcol = zeroscicol & ~bpmgapcol
addbpmcol[np.argmax(addbpmcol)-4:np.argmax(addbpmcol)] = True # allow for chip tilt
bpm_rc[:,addbpmcol] = 1
hdu[3].data = bpm_rc
hdu.writeto(filename,overwrite=True)
#clean up the images
if cleanup:
for f in glob.glob('p*fits'): os.remove(f)
for f in glob.glob('bp*fits'): os.remove(f)
for f in glob.glob('gbp*fits'): os.remove(f)
for f in glob.glob('xgbp*fits'): os.remove(f)
def quickclean(filename, interp='linear', cleanup=True, clobber=False, logfile='saltclean.log', verbose=True):
"""Start the process to reduce the data and produce a single mosaicked image"""
print filename
#create the input file name
status=0
infile=os.path.basename(filename)
rawpath=os.path.dirname(filename)
outpath='./'
outfile=outpath+'mbxp'+infile
print infile, rawpath, outpath
#check to see if it exists and return if clobber is no
if os.path.isfile(outfile) and not clobber: return
#set up the files needed
if infile[0]=='P':
gaindb = iraf.osfn('pysalt$data/rss/RSSamps.dat')
xtalkfile = iraf.osfn('pysalt$data/rss/RSSxtalk.dat')
geomfile = iraf.osfn('pysalt$data/rss/RSSgeom.dat')
elif infile[0]=='S':
gaindb = iraf.osfn('pysalt$data/scam/SALTICAMamps.dat')
xtalkfile = iraf.osfn('pysalt$data/scam/SALTICAMxtalk.dat')
geomfile = iraf.osfn('pysalt$data/scam/SALTICAMgeom.dat')
#verify the file
struct=saltio.openfits(rawpath+'/'+infile)
struct.verify('exception')
#check to see if detmode is there
if not saltkey.found('DETMODE', struct[0]):
return
#reduce the file
struct=prepare(struct, createvar=False, badpixelstruct=None)
#reset the names in the structures
for i in range(1,len(struct)):
struct[i].name=struct[i].header['EXTNAME']
#gain correct the files
usedb=True
dblist= saltio.readgaindb(gaindb)
log=open(logfile, 'a')
ampccd = struct[0].header['NAMPS'] / struct[0].header['NCCDS']
struct=gain(struct, mult=True,usedb=usedb, dblist=dblist, ampccd=ampccd, log=None, verbose=verbose)
struct=bias(struct, subover=True,trim=True,subbias=False,
median=False,function='polynomial',order=5,rej_lo=3,rej_hi=3,niter=10,
plotover=False,log=None, verbose=verbose)
if struct[0].header['CCDTYPE']=='OBJECT' and struct[0].header['EXPTIME']>90:
struct = multicrclean(struct, crtype='median', thresh=5, mbox=5, bbox=25, bthresh=5, flux_ratio=0.2, \
gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=5, log=None, verbose=verbose)
pinfile=outpath+'bxp'+infile
saltio.writefits(struct, pinfile, clobber)
saltred.saltmosaic(images=pinfile,
outimages='',outpref=outpath+'m',geomfile=geomfile, fill=True,
interp=interp,cleanup=cleanup,clobber=clobber,logfile=logfile,
verbose=verbose)
profile=outpath+'mbxp'+infile
#remove intermediate steps
if cleanup:
if os.path.isfile(pinfile): os.remove(pinfile)
return
xkey=np.array(xdict.keys())
date=xkey[abs(xkey-obsdate).argmin()]
xcoeff=xdict[date]
except Exception,e :
msg='WARNING--Can not find xtalk coefficient for %s because %s' % (e, infile)
if log: log.warning(msg)
xcoeff=xdict[xdict.keys()[-1]]
else:
xcoeff=[]
struct = xtalk(struct, xcoeff, log=log, verbose=verbose)
#bias correct the files
if saltkey.fastmode(saltkey.get('DETMODE', struct[0])): order=1
struct=bias(struct,subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function,
order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter,
plotover=plotover, log=log, verbose=verbose)
#mosaic correct the files
return struct
def createmasterbiasname(infiles, biaskeys):
"""Create the name for the master bias file based on its parameters. The format for
hte name is
[S/P][YYYYMMDD]Bias[MODE][BINNING][GAINSET][ROSPEED].fits
where the following abbreviations are used:
def quickclean(filename,
interp='linear',
cleanup=True,
clobber=False,
logfile='saltclean.log',
verbose=True):
"""Start the process to reduce the data and produce a single mosaicked image"""
print filename
#create the input file name
status = 0
infile = os.path.basename(filename)
rawpath = os.path.dirname(filename)
outpath = './'
outfile = outpath + 'mbxp' + infile
print infile, rawpath, outpath
#check to see if it exists and return if clobber is no
if os.path.isfile(outfile) and not clobber: return
#set up the files needed
if infile[0] == 'P':
gaindb = iraf.osfn('pysalt$data/rss/RSSamps.dat')
xtalkfile = iraf.osfn('pysalt$data/rss/RSSxtalk.dat')
geomfile = iraf.osfn('pysalt$data/rss/RSSgeom.dat')
elif infile[0] == 'S':
gaindb = iraf.osfn('pysalt$data/scam/SALTICAMamps.dat')
xtalkfile = iraf.osfn('pysalt$data/scam/SALTICAMxtalk.dat')
geomfile = iraf.osfn('pysalt$data/scam/SALTICAMgeom.dat')
#verify the file
struct = saltio.openfits(rawpath + '/' + infile)
struct.verify('exception')
#check to see if detmode is there
if not saltkey.found('DETMODE', struct[0]):
return
#reduce the file
struct = prepare(struct, createvar=False, badpixelstruct=None)
#reset the names in the structures
for i in range(1, len(struct)):
struct[i].name = struct[i].header['EXTNAME']
#gain correct the files
usedb = True
dblist = saltio.readgaindb(gaindb)
log = open(logfile, 'a')
ampccd = struct[0].header['NAMPS'] / struct[0].header['NCCDS']
struct = gain(struct,
mult=True,
usedb=usedb,
dblist=dblist,
ampccd=ampccd,
log=None,
verbose=verbose)
struct = bias(struct,
subover=True,
trim=True,
subbias=False,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=10,
plotover=False,
log=None,
verbose=verbose)
if struct[0].header[
'CCDTYPE'] == 'OBJECT' and struct[0].header['EXPTIME'] > 90:
struct = multicrclean(struct, crtype='median', thresh=5, mbox=5, bbox=25, bthresh=5, flux_ratio=0.2, \
gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=5, log=None, verbose=verbose)
pinfile = outpath + 'bxp' + infile
saltio.writefits(struct, pinfile, clobber)
saltred.saltmosaic(images=pinfile,
outimages='',
outpref=outpath + 'm',
geomfile=geomfile,
fill=True,
interp=interp,
cleanup=cleanup,
clobber=clobber,
logfile=logfile,
verbose=verbose)
profile = outpath + 'mbxp' + infile
#remove intermediate steps
if cleanup:
if os.path.isfile(pinfile): os.remove(pinfile)
return
