def createobslogfits(headerDict):
"""Create the fits table for the observation log"""
# define generic columns of output table
col=[]
for k, f in zip(headerList, formatList):
print k,f, headerDict[k]
col.append(pyfits.Column(name=k, format=f, array=headerDict[k]))
for k, f in zip(scamheaderList, scamformatList):
print k,f, headerDict[k]
col.append(pyfits.Column(name=k, format=f, array=headerDict[k]))
for k, f in zip(rssheaderList, rssformatList):
print k,f, headerDict[k]
col.append(pyfits.Column(name=k, format=f, array=headerDict[k]))
# construct FITS table from columns
table = saltio.fitscolumns(col)
# write FITS table to output file
struct=saltio.newfitstable(table)
# name the table extension
saltkey.new('EXTNAME','OBSLOG','extension name', struct)
# housekeeping keywords
saltkey.put('SAL-TLM',time.asctime(time.localtime()), struct)
return struct
def addhousekeeping(struct, outhdu, outfile):
"""housekeeping keywords"""
saltsafekey.put('SAL-TLM',time.asctime(time.localtime()),struct,outfile)
saltsafekey.new('EXTNAME','SCI','Extension name',struct,outfile)
saltsafekey.new('EXTVER',outhdu,'Extension number',struct,outfile)
return struct
def addhousekeeping(struct, outhdu, outfile):
"""housekeeping keywords"""
saltsafekey.put('SAL-TLM', time.asctime(time.localtime()), struct, outfile)
saltsafekey.new('EXTNAME', 'SCI', 'Extension name', struct, outfile)
saltsafekey.new('EXTVER', outhdu, 'Extension number', struct, outfile)
return struct
def prepare(struct,createvar=False, badpixelstruct=None, namps=2):
#set up the file name
infile=saltkey.getimagename(struct[0])
#include information about which headers are science extensoisn
#and check to make sure each array is a data array
nextend=len(struct)-1
nsciext=nextend
for i in range(1,nextend+1):
try:
struct[i].header['XTENSION'] == 'IMAGE'
except:
msg='Extension %i in %s is not an image data'
raise SaltError(msg)
saltkey.new('EXTNAME','SCI','Extension name',struct[i])
saltkey.new('EXTVER',i,'Extension number',struct[i])
#check the number of amplifiers
#TODO: This is current hardwired at a value of 2
nccds = saltkey.get('NCCDS',struct[0])
if (nextend%(nccds*namps) != 0):
message = 'ERROR -- SALTPREPARE: Number of file extensions and '
message += 'number of amplifiers are not consistent in ' + infile
raise SaltError(message)
#Add the inv. variance frame if requested--assumes no variance frame
#and the science frames are in the first n frames
if createvar:
#create the inv. variance frames
for i in range(1, nsciext+1):
try:
hdu=CreateVariance(struct[i], i, nextend+i)
except Exception, e:
msg='Cannot create variance frame in extension %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header.update('VAREXT',nextend+i, comment='Extension for Variance Frame')
struct.append(hdu)
nextend+=nsciext
#create the badpixelframes
for i in range(1, nsciext+1):
try:
hdu=createbadpixel(struct, badpixelstruct, i, nextend+i)
except Exception, e:
msg='Could not create bad pixel extension in ext %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header.update('BPMEXT',nextend+i, comment='Extension for Bad Pixel Mask')
struct.append(hdu)
def prepare(struct,createvar=False, badpixelstruct=None, namps=2):
#set up the file name
infile=saltkey.getimagename(struct[0])
#include information about which headers are science extensoisn
#and check to make sure each array is a data array
nextend=len(struct)-1
nsciext=nextend
for i in range(1,nextend+1):
try:
struct[i].header['XTENSION'] == 'IMAGE'
except:
msg='Extension %i in %s is not an image data'
raise SaltError(msg)
saltkey.new('EXTNAME','SCI','Extension name',struct[i])
saltkey.new('EXTVER',i,'Extension number',struct[i])
#check the number of amplifiers
#TODO: This is current hardwired at a value of 2
nccds = saltkey.get('NCCDS',struct[0])
if (nextend%(nccds*namps) != 0):
message = 'ERROR -- SALTPREPARE: Number of file extensions and '
message += 'number of amplifiers are not consistent in ' + infile
raise SaltError(message)
#Add the inv. variance frame if requested--assumes no variance frame
#and the science frames are in the first n frames
if createvar:
#create the inv. variance frames
for i in range(1, nsciext+1):
try:
hdu=CreateVariance(struct[i], i, nextend+i)
except Exception, e:
msg='Cannot create variance frame in extension %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header.update('VAREXT',nextend+i, comment='Extension for Variance Frame')
struct.append(hdu)
nextend+=nsciext
#create the badpixelframes
for i in range(1, nsciext+1):
try:
hdu=createbadpixel(struct, badpixelstruct, i, nextend+i)
except Exception, e:
msg='Could not create bad pixel extension in ext %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header.update('BPMEXT',nextend+i, comment='Extension for Bad Pixel Mask')
struct.append(hdu)
def updateheaders(struct, ext, tdiff, real_expt, utc, infile):
# exit if tdiff wasn't updated
if tdiff == real_expt:
msg='No adequate correction found for frame %i in file %s' % (ext, infile)
raise SaltError(msg)
return struct
# calculate the new utc value
try:
ntime=salttime.sex2dec(utc)
ntime=ntime-tdiff/3600.0
newutc=salttime.dec2sex(ntime)
except Exception as e:
msg='Could not update UTC in %i header of image %s because %s' % (ext, infile, e)
raise SaltError(msg)
return struct
# update the headers
if utc==saltsafekey.get('UTC-OBS', struct):
expt_string='%5.4f' % real_expt
td_string='%5.4f' % tdiff
if not saltsafekey.found('DUTC', struct):
try:
saltsafekey.put('UTC-OBS', newutc, struct, infile)
saltsafekey.put('TIME-OBS', newutc, struct, infile)
saltsafekey.new('DWETIME', expt_string, 'Dwell Time', struct, infile)
saltsafekey.new('DUTC', td_string, 'Change in UTC time', struct, infile)
except Exception as e:
msg='Could not update %i header of image %s because %s' % (ext, infile, e)
raise SaltIOError(msg)
else:
try:
saltsafekey.put('UTC-OBS', newutc, struct, infile)
saltsafekey.put('TIME-OBS', newutc, struct, infile)
saltsafekey.put('DWETIME', real_expt, struct, infile)
saltsafekey.put('DUTC', tdiff, struct, infile)
except Exception as e:
msg='Could not update %i header of image %s because %s' % (ext, infile, e)
raise SaltError(msg)
else:
raise SaltIOError('Frame missing from list of times')
return struct
def readtimefix(hdu, dsteps=7, transtime=4e-3):
"""Update the hdu with the correct time for when the exposure started
and add the READTIME keyword
dsteps--the number of readouts to correct for
transtime--the transfer time between each frame
"""
#check for if the data has already been processed
if saltkey.found('READTIME', hdu):
raise SaltIOError(' has already been processed')
#determine the UTC time
utctime = saltkey.get('UTC-OBS', hdu)
timeobs = saltkey.get('TIME-OBS', hdu)
dateobs = saltkey.get('DATE-OBS', hdu)
exptime = float(saltkey.get('EXPTIME', hdu))
#add the readtime header
saltkey.new("READTIME", utctime, 'Time of the readout of the frame', hdu)
#correct the utctime--first switch to datetime to properly handle
#dates around changes in hours
y, m, d = dateobs.split('-')
H, M, S = utctime.split(':')
s = int(float(S))
ms = int(1e6 * (float(S) - s))
newtime = datetime.datetime(int(y), int(m), int(d), int(H), int(M), s, ms)
#correct the datetime
dtime = dsteps * (exptime + transtime)
s = int(dtime)
ms = int(1e6 * (dtime - s))
newtime = newtime - datetime.timedelta(0, s, ms)
#update the headkeywords
hdu.header["UTC-OBS"] = str(newtime.time())
saltkey.put("UTC-OBS", str(newtime.time()), hdu)
saltkey.put("TIME-OBS", str(newtime.time()), hdu)
saltkey.put("DATE-OBS", str(newtime.date()), hdu)
return hdu
def readtimefix(hdu, dsteps=7, transtime=4e-3):
"""Update the hdu with the correct time for when the exposure started
and add the READTIME keyword
dsteps--the number of readouts to correct for
transtime--the transfer time between each frame
"""
#check for if the data has already been processed
if saltkey.found('READTIME', hdu):
raise SaltIOError(' has already been processed')
#determine the UTC time
utctime=saltkey.get('UTC-OBS', hdu)
timeobs=saltkey.get('TIME-OBS', hdu)
dateobs=saltkey.get('DATE-OBS', hdu)
exptime=float(saltkey.get('EXPTIME', hdu))
#add the readtime header
saltkey.new("READTIME",utctime,'Time of the readout of the frame', hdu)
#correct the utctime--first switch to datetime to properly handle
#dates around changes in hours
y,m,d=dateobs.split('-')
H,M,S=utctime.split(':')
s=int(float(S))
ms=int(1e6*(float(S)-s))
newtime=datetime.datetime(int(y),int(m),int(d),int(H),int(M),s,ms)
#correct the datetime
dtime=dsteps*(exptime+transtime)
s=int(dtime)
ms=int(1e6*(dtime-s))
newtime=newtime-datetime.timedelta(0, s, ms)
#update the headkeywords
hdu.header.update("UTC-OBS", str(newtime.time()))
saltkey.put("UTC-OBS", str(newtime.time()), hdu)
saltkey.put("TIME-OBS", str(newtime.time()), hdu)
saltkey.put("DATE-OBS", str(newtime.date()), hdu)
return hdu
def createrecord(recfile, fitcol, keycol, oldcol, newcol, clobber):
"""Create the fits table record of all of the changes that were made"""
col1 = fits.Column(name='FILE',format='32A',array=fitcol)
col2 = fits.Column(name='KEYWD',format='8A',array=keycol)
col3 = fits.Column(name='OLD_VAL',format='24A',array=oldcol)
col4 = fits.Column(name='NEW_VAL',format='24A',array=newcol)
cols = fits.ColDefs([col1,col2,col3,col4])
rectable = fits.BinTableHDU.from_columns(cols)
#add some additional keywords
saltkey.new('EXTNAME','NEWKEYS','Name of this binary table extension', rectable)
saltkey.new('OBSERVAT','SALT','South African Large Telescope', rectable)
saltkey.new('SAL-TLM',time.asctime(time.localtime()), 'File last updated by PySALT tools',rectable)
saltkey.new('SAL-EDT',time.asctime(time.localtime()), 'Keywords updated by SALTEDTKY',rectable)
#write it out
saltio.writefits(rectable, recfile, clobber)
def hrsclean(images,
outpath,
obslogfile=None,
subover=True,
trim=True,
masbias=None,
subbias=True,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""Convert MEF HRS data into a single image. If variance frames and BPMs, then
convert them to the same format as well. Returns an MEF image but that is
combined into a single frame
"""
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
if saltio.checkfornone(obslogfile) is None:
raise SaltError('Obslog file is required')
# Delete the obslog file if it already exists
if (os.path.isfile(obslogfile)
and clobber) or not os.path.isfile(obslogfile):
if os.path.isfile(obslogfile): saltio.delete(obslogfile)
#read in the obsveration log or create it
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
else:
obsstruct = saltio.openfits(obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('OBJECT')
biaslist = filename[ccdtype == 'Bias']
masterbias_dict = {}
if log: log.message('Processing Bias Frames')
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = pyfits.open(img)
bimg = outpath + 'bgph' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message,
with_stdout=verbose,
with_header=False)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
subover=subover,
trim=trim,
subbias=False,
imstack=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'HPREPARE',
'Images have been prepared', hist)
saltkey.new('HGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=hrsbiasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(
blist, bkeys, x1=5, x2=13)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#apply full reductions to the science data
for img in infiles:
nimg = os.path.basename(img)
if not nimg in biaslist:
#open the image
struct = pyfits.open(img)
simg = outpath + 'mbgph' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#get master bias frame
masterbias = get_masterbias(struct,
masterbias_dict,
keylist=hrsbiasheader_list)
if masterbias:
subbias = True
bstruct = saltio.openfits(masterbias)
else:
subbias = False
bstruct = None
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
subover=subover,
trim=trim,
subbias=subbias,
imstack=True,
bstruct=bstruct,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'HPREPARE',
'Images have been prepared', hist)
saltkey.new('HGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
return
def wavemap(
hdu,
soldict,
caltype="line",
function="poly",
order=3,
blank=0,
nearest=False,
array_only=False,
clobber=True,
log=None,
verbose=True,
):
"""Read in an image and a set of wavlength solutions. Calculate the best
wavelength solution for a given dataset and then apply that data set to the
image
return
"""
# set up the time of the observation
dateobs = saltkey.get("DATE-OBS", hdu[0])
utctime = saltkey.get("TIME-OBS", hdu[0])
exptime = saltkey.get("EXPTIME", hdu[0])
instrume = saltkey.get("INSTRUME", hdu[0]).strip()
grating = saltkey.get("GRATING", hdu[0]).strip()
if caltype == "line":
grang = saltkey.get("GRTILT", hdu[0])
arang = saltkey.get("CAMANG", hdu[0])
else:
grang = saltkey.get("GR-ANGLE", hdu[0])
arang = saltkey.get("AR-ANGLE", hdu[0])
filtername = saltkey.get("FILTER", hdu[0]).strip()
slitname = saltkey.get("MASKID", hdu[0])
slit = st.getslitsize(slitname)
xbin, ybin = saltkey.ccdbin(hdu[0])
timeobs = sr.enterdatetime("%s %s" % (dateobs, utctime))
# check to see if there is more than one solution
if caltype == "line":
if len(soldict) == 1:
sol = soldict.keys()[0]
slitid = None
if not sr.matchobservations(soldict[sol], instrume, grating, grang, arang, filtername, slitid):
msg = "Observations do not match setup for transformation but using the solution anyway"
if log:
log.warning(msg)
for i in range(1, len(hdu)):
if hdu[i].name == "SCI":
if log:
log.message("Correcting extension %i" % i)
istart = int(0.5 * len(hdu[i].data))
# open up the data
# set up the xarr and initial wavlength solution
xarr = np.arange(len(hdu[i].data[istart]), dtype="int64")
# get the slitid
try:
slitid = saltkey.get("SLITNAME", hdu[i])
except:
slitid = None
# check to see if wavext is already there and if so, then check update
# that for the transformation from xshift to wavelength
if saltkey.found("WAVEXT", hdu[i]):
w_ext = saltkey.get("WAVEXT", hdu[i]) - 1
wavemap = hdu[w_ext].data
function, order, coef = sr.findlinesol(
soldict,
istart,
nearest,
timeobs,
exptime,
instrume,
grating,
grang,
arang,
filtername,
slitid,
xarr,
)
ws = WavelengthSolution.WavelengthSolution(xarr, xarr, function=function, order=order)
ws.set_coef(coef)
for j in range(len(hdu[i].data)):
wavemap[j, :] = ws.value(wavemap[j, :])
if array_only:
return wavemap
hdu[w_ext].data = wavemap
continue
# set up a wavelength solution -- still in here for testing MOS data
try:
w_arr = sr.findsol(
xarr,
soldict,
istart,
caltype,
nearest,
timeobs,
exptime,
instrume,
grating,
grang,
arang,
filtername,
slit,
xbin,
ybin,
slitid,
function,
order,
)
except SALTSpecError as e:
if slitid:
msg = "SLITID %s: %s" % (slitid, e)
if log:
log.warning(msg)
continue
else:
raise SALTSpecError(e)
if w_arr is None:
w_arr = sr.findsol(
xarr,
soldict,
istart,
"rss",
nearest,
timeobs,
exptime,
instrume,
grating,
grang,
arang,
filtername,
slit,
xbin,
ybin,
slitid,
function,
order,
)
# for each line in the data, determine the wavelength solution
# for a given line in the image
wavemap = np.zeros_like(hdu[i].data)
for j in range(len(hdu[i].data)):
# find the wavelength solution for the data
w_arr = sr.findsol(
xarr,
soldict,
j,
caltype,
nearest,
timeobs,
exptime,
instrume,
grating,
grang,
arang,
filtername,
slit,
xbin,
ybin,
slitid,
function,
order,
)
if w_arr is not None:
wavemap[j, :] = w_arr
if array_only:
return wavemap
# write out the oimg
hduwav = fits.ImageHDU(data=wavemap, header=hdu[i].header, name="WAV")
hdu.append(hduwav)
saltkey.new("WAVEXT", len(hdu) - 1, "Extension for Wavelength Map", hdu[i])
return hdu
def make_mosaic(struct,
gap,
xshift,
yshift,
rotation,
interp_type='linear',
boundary='constant',
constant=0,
geotran=True,
fill=False,
cleanup=True,
log=None,
verbose=False):
"""Given a SALT image struct, combine each of the individual amplifiers and
apply the geometric CCD transformations to the image
"""
# get the name of the file
infile = saltkey.getimagename(struct[0], base=True)
outpath = './'
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = \
saltkey.instrumid(struct)
# how many amplifiers?
nsciext = saltkey.get('NSCIEXT', struct[0])
nextend = saltkey.get('NEXTEND', struct[0])
nccds = saltkey.get('NCCDS', struct[0])
amplifiers = nccds * 2
if nextend > nsciext:
varframe = True
else:
varframe = False
# CCD geometry coefficients
if (instrume == 'RSS' or instrume == 'PFIS'):
xsh = [0., xshift[0], 0., xshift[1]]
ysh = [0., yshift[0], 0., yshift[1]]
rot = [0., rotation[0], 0., rotation[1]]
elif instrume == 'SALTICAM':
xsh = [0., xshift[0], 0.]
ysh = [0., yshift[0], 0.]
rot = [0., rotation[0], 0]
# how many extensions?
nextend = saltkey.get('NEXTEND', struct[0])
# CCD on-chip binning
xbin, ybin = saltkey.ccdbin(struct[0])
# create temporary primary extension
outstruct = []
outstruct.append(struct[0])
# define temporary FITS file store tiled CCDs
tilefile = saltio.tmpfile(outpath)
tilefile += 'tile.fits'
if varframe:
tilehdu = [None] * (3 * int(nsciext / 2) + 1)
else:
tilehdu = [None] * int(nsciext / 2 + 1)
tilehdu[0] = fits.PrimaryHDU()
#tilehdu[0].header = struct[0].header
if log:
log.message('', with_stdout=verbose)
# iterate over amplifiers, stich them to produce file of CCD images
for i in range(int(nsciext / 2)):
hdu = i * 2 + 1
# amplifier = hdu%amplifiers
# if (amplifier == 0): amplifier = amplifiers
# read DATASEC keywords
datasec1 = saltkey.get('DATASEC', struct[hdu])
datasec2 = saltkey.get('DATASEC', struct[hdu + 1])
xdsec1, ydsec1 = saltstring.secsplit(datasec1)
xdsec2, ydsec2 = saltstring.secsplit(datasec2)
# read images
imdata1 = saltio.readimage(struct, hdu)
imdata2 = saltio.readimage(struct, hdu + 1)
# tile 2n amplifiers to yield n CCD images
outdata = numpy.zeros(
(int(ydsec1[1] + abs(ysh[i + 1] / ybin)),
int(xdsec1[1] + xdsec2[1] + abs(xsh[i + 1] / xbin))),
numpy.float32)
# set up the variance frame
if varframe:
vardata = outdata.copy()
vdata1 = saltio.readimage(struct, struct[hdu].header['VAREXT'])
vdata2 = saltio.readimage(struct, struct[hdu + 1].header['VAREXT'])
bpmdata = outdata.copy()
bdata1 = saltio.readimage(struct, struct[hdu].header['BPMEXT'])
bdata2 = saltio.readimage(struct, struct[hdu + 1].header['BPMEXT'])
x1 = xdsec1[0] - 1
if x1 != 0:
msg = 'The data in %s have not been trimmed prior to mosaicking.' \
% infile
log.error(msg)
if xsh[i + 1] < 0:
x1 += int(abs(xsh[i + 1] / xbin))
x2 = x1 + xdsec1[1]
y1 = ydsec1[0] - 1
if ysh[i + 1] < 0:
y1 += int(abs(ysh[i + 1] / ybin))
y2 = y1 + ydsec1[1]
outdata[y1:y2, x1:x2] =\
imdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
x1 = x2
x2 = x1 + xdsec2[1]
y1 = ydsec2[0] - 1
if ysh[i + 1] < 0:
y1 += abs(ysh[i + 1] / ybin)
y2 = y1 + ydsec2[1]
outdata[y1:y2, x1:x2] =\
imdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
# size of new image
naxis1 = str(xdsec1[1] + xdsec2[1])
naxis2 = str(ydsec1[1])
# add image and keywords to HDU list
tilehdu[i + 1] = fits.ImageHDU(outdata)
tilehdu[i + 1].header = struct[hdu].header
#tilehdu[
# i + 1].header['DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
if varframe:
vext = i + 1 + int(nsciext / 2.)
tilehdu[vext] = fits.ImageHDU(vardata)
#tilehdu[vext].header = struct[struct[hdu].header['VAREXT']].header
#tilehdu[vext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
bext = i + 1 + 2 * int(nsciext / 2.)
tilehdu[bext] = fits.ImageHDU(bpmdata)
#tilehdu[bext].header = struct[struct[hdu].header['BPMEXT']].header
#tilehdu[bext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
# image tile log message #1
if log:
message = os.path.basename(infile) + '[' + str(hdu) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
message = os.path.basename(infile) + '[' + str(hdu + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[1] + 1) + ':' + \
str(xdsec1[1] + xdsec2[1]) + ','
message += str(ydsec2[0]) + ':' + str(ydsec2[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
# write temporary file of tiled CCDs
hdulist = fits.HDUList(tilehdu)
hdulist.writeto(tilefile)
# iterate over CCDs, transform and rotate images
yrot = [None] * 4
xrot = [None] * 4
tranfile = [' ']
tranhdu = [0]
if varframe:
tranfile = [''] * (3 * int(nsciext / 2) + 1)
tranhdu = [0] * (3 * int(nsciext / 2) + 1)
else:
tranfile = [''] * int(nsciext / 2 + 1)
tranhdu = [0] * int(nsciext / 2 + 1)
# this is hardwired for SALT where the second CCD is considered the
# fiducial
for hdu in range(1, int(nsciext / 2 + 1)):
tranfile[hdu] = saltio.tmpfile(outpath)
tranfile[hdu] += 'tran.fits'
if varframe:
tranfile[hdu + nccds] = saltio.tmpfile(outpath) + 'tran.fits'
tranfile[hdu + 2 * nccds] = saltio.tmpfile(outpath) + 'tran.fits'
ccd = hdu % nccds
if (ccd == 0):
ccd = nccds
# correct rotation for CCD binning
yrot[ccd] = rot[ccd] * ybin / xbin
xrot[ccd] = rot[ccd] * xbin / ybin
dxshift = xbin * int(float(int(gap) / xbin) + 0.5) - gap
# transformation using geotran IRAF task
# if (ccd == 1):
if (ccd != 2):
if geotran:
message = '\nSALTMOSAIC -- geotran ' + tilefile + \
'[' + str(ccd) + '] ' + tranfile[hdu]
message += ' \"\" \"\" xshift=' + \
str((xsh[ccd] + (2 - ccd) * dxshift) / xbin) + ' '
message += 'yshift=' + \
str(ysh[ccd] / ybin) + ' xrotation=' + str(xrot[ccd]) + ' '
message += 'yrotation=' + \
str(yrot[ccd]) + ' xmag=1 ymag=1 xmin=\'INDEF\''
message += 'xmax=\'INDEF\' ymin=\'INDEF\' ymax=\'INDEF\' '
message += 'ncols=\'INDEF\' '
message += 'nlines=\'INDEF\' verbose=\'no\' '
message += 'fluxconserve=\'yes\' nxblock=2048 '
message += 'nyblock=2048 interpolant=\'' + \
interp_type + '\' boundary=\'constant\' constant=0'
log.message(message, with_stdout=verbose)
yd, xd = tilehdu[ccd].data.shape
ncols = 'INDEF' # ncols=xd+abs(xsh[ccd]/xbin)
nlines = 'INDEF' # nlines=yd+abs(ysh[ccd]/ybin)
geo_xshift = xsh[ccd] + (2 - ccd) * dxshift / xbin
geo_yshift = ysh[ccd] / ybin
iraf.images.immatch.geotran(tilefile + "[" + str(ccd) + "]",
tranfile[hdu],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1,
ymag=1,
xmin='INDEF',
xmax='INDEF',
ymin='INDEF',
ymax='INDEF',
ncols=ncols,
nlines=nlines,
verbose='no',
fluxconserve='yes',
nxblock=2048,
nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
if varframe:
var_infile = tilefile + "[" + str(ccd + nccds) + "]"
iraf.images.immatch.geotran(var_infile,
tranfile[hdu + nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1,
ymag=1,
xmin='INDEF',
xmax='INDEF',
ymin='INDEF',
ymax='INDEF',
ncols=ncols,
nlines=nlines,
verbose='no',
fluxconserve='yes',
nxblock=2048,
nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
var2_infile = tilefile + "[" + str(ccd + 2 * nccds) + "]"
iraf.images.immatch.geotran(var2_infile,
tranfile[hdu + 2 * nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1,
ymag=1,
xmin='INDEF',
xmax='INDEF',
ymin='INDEF',
ymax='INDEF',
ncols=ncols,
nlines=nlines,
verbose='no',
fluxconserve='yes',
nxblock=2048,
nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
# open the file and copy the data to tranhdu
tstruct = fits.open(tranfile[hdu])
tranhdu[hdu] = tstruct[0].data
tstruct.close()
if varframe:
tranhdu[hdu + nccds] = fits.open(tranfile[hdu +
nccds])[0].data
tranhdu[hdu + 2 * nccds] = fits.open(
tranfile[hdu + 2 * nccds])[0].data
else:
log.message("Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd, xsh[ccd] / 2.0, ysh[ccd] / 2.0, xrot[ccd]),
with_stdout=verbose,
with_header=False)
tranhdu[hdu] = geometric_transform(
tilehdu[ccd].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1,
xrot[ccd], yrot[ccd]))
tstruct = fits.PrimaryHDU(tranhdu[hdu])
tstruct.writeto(tranfile[hdu])
if varframe:
tranhdu[hdu + nccds] = geometric_transform(
tilehdu[hdu + 3].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1,
xrot[ccd], yrot[ccd]))
tranhdu[hdu + 2 * nccds] = geometric_transform(
tilehdu[hdu + 6].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1,
xrot[ccd], yrot[ccd]))
else:
log.message("Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd, 0, 0, 0),
with_stdout=verbose,
with_header=False)
tranhdu[hdu] = tilehdu[ccd].data
if varframe:
tranhdu[hdu + nccds] = tilehdu[ccd + nccds].data
tranhdu[hdu + 2 * nccds] = tilehdu[ccd + 2 * nccds].data
# open outfile
if varframe:
outlist = 4 * [None]
else:
outlist = 2 * [None]
#outlist[0] = struct[0].copy()
outlist[0] = fits.PrimaryHDU()
outlist[0].header = struct[0].header
naxis1 = int(gap / xbin * (nccds - 1))
naxis2 = 0
for i in range(1, nccds + 1):
yw, xw = tranhdu[i].shape
naxis1 += xw + int(abs(xsh[ccd] / xbin)) + 1
naxis2 = max(naxis2, yw)
outdata = numpy.zeros((naxis2, naxis1), numpy.float32)
outdata.shape = naxis2, naxis1
if varframe:
vardata = outdata * 0
bpmdata = outdata * 0 + 1
# iterate over CCDs, stich them to produce a full image
hdu = 0
totxshift = 0
for hdu in range(1, nccds + 1):
# read DATASEC keywords
ydsec, xdsec = tranhdu[hdu].shape
# define size and shape of final image
# tile CCDs to yield mosaiced image
x1 = int((hdu - 1) * (xdsec + gap / xbin)) + int(totxshift)
x2 = xdsec + x1
y1 = int(0)
y2 = int(ydsec)
outdata[y1:y2, x1:x2] = tranhdu[hdu]
totxshift += int(abs(xsh[hdu] / xbin)) + 1
if varframe:
vardata[y1:y2, x1:x2] = tranhdu[hdu + nccds]
bpmdata[y1:y2, x1:x2] = tranhdu[hdu + 2 * nccds]
# make sure to cover up all the gaps include bad areas
if varframe:
baddata = (outdata == 0)
baddata = nd.maximum_filter(baddata, size=3)
bpmdata[baddata] = 1
# fill in the gaps if requested
if fill:
if varframe:
outdata = fill_gaps(outdata, 0)
else:
outdata = fill_gaps(outdata, 0)
# add to the file
outlist[1] = fits.ImageHDU(outdata)
if varframe:
outlist[2] = fits.ImageHDU(vardata, name='VAR')
outlist[3] = fits.ImageHDU(bpmdata, name='BPM')
# create the image structure
outstruct = fits.HDUList(outlist)
# update the head informaation
# housekeeping keywords
saltkey.put('NEXTEND', 2, outstruct[0])
saltkey.new('EXTNAME', 'SCI', 'Extension name', outstruct[1])
saltkey.new('EXTVER', 1, 'Extension number', outstruct[1])
if varframe:
saltkey.new('VAREXT', 2, 'Variance frame extension', outstruct[1])
saltkey.new('BPMEXT', 3, 'BPM Extension', outstruct[1])
try:
saltkey.copy(struct[1], outstruct[1], 'CCDSUM')
except:
pass
# Add keywords associated with geometry
saltkey.new('SGEOMGAP', gap, 'SALT Chip Gap', outstruct[0])
c1str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[0], yshift[0], rotation[0])
saltkey.new('SGEOM1', c1str, 'SALT Chip 1 Transform', outstruct[0])
c2str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[1], yshift[1], rotation[1])
saltkey.new('SGEOM2', c2str, 'SALT Chip 2 Transform', outstruct[0])
# WCS keywords
saltkey.new('CRPIX1', 0, 'WCS: X reference pixel', outstruct[1])
saltkey.new('CRPIX2', 0, 'WCS: Y reference pixel', outstruct[1])
saltkey.new('CRVAL1', float(xbin), 'WCS: X reference coordinate value',
outstruct[1])
saltkey.new('CRVAL2', float(ybin), 'WCS: Y reference coordinate value',
outstruct[1])
saltkey.new('CDELT1', float(xbin), 'WCS: X pixel size', outstruct[1])
saltkey.new('CDELT2', float(ybin), 'WCS: Y pixel size', outstruct[1])
saltkey.new('CTYPE1', 'pixel', 'X type', outstruct[1])
saltkey.new('CTYPE2', 'pixel', 'Y type', outstruct[1])
# cleanup temporary files
if cleanup:
for tfile in tranfile:
if os.path.isfile(tfile):
saltio.delete(tfile)
if os.path.isfile(tilefile):
status = saltio.delete(tilefile)
# return the file
return outstruct
def write_ext_header (hdu, file, bhdu, ut,date,bscale,bzero,exptime,gain,rdnoise,
datasec,detsec,ccdsec,ampsec,biassec, deadtime=None, framecnt=None):
"""Update the header values for the extension
returns
"""
#if (status==0): status = saltkey.new("BSCALE",bscale,"Val=BSCALE*pix+BZERO",hdu,file,logfile)
#if (status==0): status = saltkey.new("BZERO",bzero,"Val=BSCALE*pix+BZERO",hdu,file,logfile)
saltkey.new("UTC-OBS",ut,"UTC start of observation",hdu,file)
saltkey.new("TIME-OBS",ut,"UTC start of observation",hdu,file)
saltkey.new("DATE-OBS",date,"Date of the observation",hdu,file)
saltkey.new("EXPTIME",exptime,"Exposure time",hdu,file)
saltkey.new("GAIN",gain,"Nominal CCD gain (e/ADU)",hdu,file)
saltkey.new("RDNOISE",rdnoise,"Nominal readout noise in e",hdu,file)
saltkey.new("CCDSUM", bhdu.header['CCDSUM'], 'On chip summation', hdu, file)
saltkey.copy(hdu,bhdu,"DETSIZE")
saltkey.new("DATASEC",datasec,"Data Section",hdu,file)
saltkey.new("DETSEC",detsec,"Detector Section",hdu,file)
saltkey.new("CCDSEC ",ccdsec,"CCD Section",hdu,file)
saltkey.new("AMPSEC",ampsec,"Amplifier Section",hdu,file)
saltkey.new("BIASSEC",biassec,"Bias Section",hdu,file)
if deadtime:
saltkey.new("DEADTIME",deadtime,"Milliseconds waiting for readout", hdu, file)
if framecnt:
saltkey.new("FRAMECNT",framecnt,"Frame counter", hdu, file)
return hdu
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='',
clobber=False, cleanup=True, logfile='salt.log', verbose=True):
"""SALTADVANCE provides advanced data reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
infiles.sort()
# create list of output files
outpath=saltio.abspath(outpath)
#log into the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
obsmode=obsstruct.data.field('OBSMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
propcode=obsstruct.data.field('PROPID')
masktype=obsstruct.data.field('MASKTYP')
#set the bias list of objects
biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')]
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=fits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=fits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the arc data
arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')]
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if nimg in arclist:
#open the image
struct=fits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
obsdate=os.path.basename(img)[1:9]
#print the message
if log:
message='Processing ARC frame %s' % img
log.message(message, with_stdout=verbose)
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
log=log, verbose=verbose)
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the images
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#measure the arcdata
arcimage=outpath+'mbxgp'+nimg
dbfile=outpath+obsdate+'_specid.db'
lamp = obsstruct.data.field('LAMPID')[i]
lamp = lamp.replace(' ', '')
lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
print arcimage, lampfile, os.getcwd()
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod='Matchlines', function='legendre',
order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3,
startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile,
verbose=verbose)
try:
ximg = outpath+'xmbxgp'+os.path.basename(arcimage)
specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except:
pass
#process the science data
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if not (nimg in flatlist or nimg in biaslist or nimg in arclist):
#open the image
struct=fits.open(img)
if struct[0].header['PROPID'].count('CAL_GAIN'): continue
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#Check to see if it is RSS 2x2 and add bias subtraction
instrume=saltkey.get('INSTRUME', struct[0]).strip()
gainset = saltkey.get('GAINSET', struct[0])
rospeed = saltkey.get('ROSPEED', struct[0])
target = saltkey.get('OBJECT', struct[0]).strip()
exptime = saltkey.get('EXPTIME', struct[0])
obsmode = saltkey.get('OBSMODE', struct[0]).strip()
detmode = saltkey.get('DETMODE', struct[0]).strip()
masktype = saltkey.get('MASKTYP', struct[0]).strip()
xbin, ybin = saltkey.ccdbin( struct[0], img)
obsdate=os.path.basename(img)[1:9]
bstruct=None
crtype=None
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=5
subbias=False
if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW':
bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin)
if os.path.exists(bfile):
bstruct=fits.open(bfile)
subbias=True
if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5:
crtype='edge'
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=3
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \
bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise,
bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter,
log=log, verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#if the file is spectroscopic mode, apply the wavelength correction
if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT':
dbfile=outpath+obsdate+'_specid.db'
try:
ximg = outpath+'xmbxgp'+os.path.basename(img)
specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except Exception, e:
log.message('%s' % e)
#clean up the results
if cleanup:
#clean up the bias frames
for i in masterbias_dict.keys():
blist=masterbias_dict[i][1:]
for b in blist: saltio.delete(b)
#clean up the flat frames
for i in masterflat_dict.keys():
flist=masterflat_dict[i][1:]
for f in flist: saltio.delete(f)
def saltfpzeropoint(images,outimages, outpref, calfile, fpa, fpb, fpc, fpd, fpe, fpf, clobber=False,logfile='salt.log',verbose=True):
"""Adds zeropoint information to each image"""
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create the output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
#calculate the zeropoint coefficients
fpcoef=np.array([fpa,fpb,fpc,fpd,fpe,fpf])
zpcoef,tstart=calc_zpcoef(calfile, fpcoef)
# open each image and detect the ring
for img, oimg in zip(infiles, outfiles):
hdu=saltio.openfits(img)
#get the image time
t=(get_datetime(hdu)-tstart).seconds
#add the header values to the image
saltkey.new('FPA',fpa+zpcoef[1]+zpcoef[0]*t,'FPA Coef',hdu[0])
saltkey.new('FPB',fpb,'FPB Coef',hdu[0])
saltkey.new('FPC',fpc,'FPC Coef',hdu[0])
saltkey.new('FPD',fpd,'FPD Coef',hdu[0])
saltkey.new('FPE', 0,'FPE Coef',hdu[0])
saltkey.new('FPF',fpf,'FPF Coef',hdu[0])
#write the file out
saltio.writefits(hdu, oimg, clobber)
#log the action
msg='Updating the FP information in %s' % (img)
log.message(msg, with_stdout=verbose, with_header=False)
except Exception, e:
msg = 'Could not update UTC in %i header of image %s because %s' % (
ext, infile, e)
raise SaltError(msg)
return struct
# update the headers
if utc == saltsafekey.get('UTC-OBS', struct):
expt_string = '%5.4f' % real_expt
td_string = '%5.4f' % tdiff
if not saltsafekey.found('DUTC', struct):
try:
saltsafekey.put('UTC-OBS', newutc, struct, infile)
saltsafekey.put('TIME-OBS', newutc, struct, infile)
saltsafekey.new('DWETIME', expt_string, 'Dwell Time', struct,
infile)
saltsafekey.new('DUTC', td_string, 'Change in UTC time',
struct, infile)
except Exception, e:
msg = 'Could not update %i header of image %s because %s' % (
ext, infile, e)
raise SaltIOError(msg)
else:
try:
saltsafekey.put('UTC-OBS', newutc, struct, infile)
saltsafekey.put('TIME-OBS', newutc, struct, infile)
saltsafekey.put('DWETIME', real_expt, struct, infile)
saltsafekey.put('DUTC', tdiff, struct, infile)
except Exception, e:
msg = 'Could not update %i header of image %s because %s' % (
ext, infile, e)
def slot(struct, infile, dbspeed, dbrate, dbgain, dbnoise, dbbias, dbamp,
xcoeff, gaindb, xtalkfile, logfile, verbose):
import saltprint, saltkey, saltio, saltstat, time
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot, status = saltkey.instrumid(
struct, infile, logfile)
# number of image HDU
nextend = 0
while (status == 0):
try:
struct[nextend + 1].header['XTENSION']
nextend += 1
except:
break
nccds, status = saltkey.get('NCCDS', struct[0], infile, logfile)
amplifiers = nccds * 2
if (nextend % (amplifiers) != 0):
message = '\nERROR -- SALTSLOT: Number of image extensions and'
message += 'number of amplifiers are not consistent'
status = saltprint.err(saltlog, message)
status = saltkey.new('NSCIEXT', nextend, 'Number of science extensions',
struct[0], infile, logfile)
status = saltkey.new('NEXTEND', nextend, 'Number of data extensions',
struct[0], infile, logfile)
# check image file and gain database are compatible
if (status == 0):
ngains = len(dbgain)
if (int(max(dbamp)) != amplifiers):
message = '\nERROR -- SALTGSLOT: ' + infile + ' contains ' + str(
amplifiers) + ' amplifiers'
message += ', the gaindb file ' + gaindb + ' contains ' + str(
max(dbamp)) + ' amplifiers'
status = saltprint.err(logfile, message)
# check image file and cross talk database are compatible
if (status == 0):
if (len(xcoeff) - 1 != amplifiers):
message = '\nERROR -- SALTSLOT: ' + infile + ' contains ' + str(
amplifiers) + ' amplifiers'
message += ', the cross talk file ' + xtalkfile + ' contains ' + str(
len(xcoeff) - 1) + ' amplifiers'
status = saltprint.err(logfile, message)
# housekeeping keywords
if (status == 0):
status = saltkey.put('SAL-TLM', time.asctime(time.localtime()),
struct[0], infile, logfile)
status = saltkey.new(keyslot, time.asctime(time.localtime()),
'Data have been cleaned by SALTSLOT', struct[0],
infile, logfile)
# keywords for image extensions
for i in range(nextend):
hdu = i + 1
status = saltkey.new('EXTNAME', 'SCI', 'Extension name', struct[hdu],
infile, logfile)
status = saltkey.new('EXTVER', hdu, 'Extension number', struct[hdu],
infile, logfile)
# log coefficent table
if (status == 0):
message = '%30s %5s %4s %8s' % ('HDU', 'Gain', 'Bias', 'Xtalk')
saltprint.log(logfile,
'\n ---------------------------------------------',
verbose)
saltprint.log(logfile, message, verbose)
saltprint.log(logfile,
' ---------------------------------------------',
verbose)
# loop over image extensions
if (status == 0):
for i in range(nextend / 2):
hdu = i * 2 + 1
amplifier = hdu % amplifiers
if (amplifier == 0): amplifier = amplifiers
if (status == 0):
value, status = saltkey.get('NAXIS1', struct[hdu], infile,
logfile)
naxis1 = int(value)
if (status == 0):
value, status = saltkey.get('NAXIS1', struct[hdu + 1], infile,
logfile)
naxis2 = int(value)
if (status == 0 and hdu == 1):
biassec, status = saltkey.get('BIASSEC', struct[hdu], infile,
logfile)
if (status == 0 and hdu == 1):
ranges = biassec.lstrip('[').rstrip(']').split(',')
x1_1 = int(ranges[0].split(':')[0]) - 1
x2_1 = int(ranges[0].split(':')[1]) - 1
y1_1 = int(ranges[1].split(':')[0]) - 1
y2_1 = int(ranges[1].split(':')[1]) - 1
if (status == 0 and hdu == 1):
biassec, status = saltkey.get('BIASSEC', struct[hdu + 1],
infile, logfile)
if (status == 0 and hdu == 1):
ranges = biassec.lstrip('[').rstrip(']').split(',')
x1_2 = int(ranges[0].split(':')[0]) - 1
x2_2 = int(ranges[0].split(':')[1]) - 1
y1_2 = int(ranges[1].split(':')[0]) - 1
y2_2 = int(ranges[1].split(':')[1]) - 1
if (status == 0 and hdu == 1):
datasec, status = saltkey.get('DATASEC', struct[hdu], infile,
logfile)
if (status == 0 and hdu == 1):
ranges = datasec.lstrip('[').rstrip(']').split(',')
dx1_1 = int(ranges[0].split(':')[0]) - 1
dx2_1 = int(ranges[0].split(':')[1])
dy1_1 = int(ranges[1].split(':')[0]) - 1
dy2_1 = int(ranges[1].split(':')[1])
if (status == 0 and hdu == 1):
datasec, status = saltkey.get('DATASEC', struct[hdu + 1],
infile, logfile)
if (status == 0 and hdu == 1):
ranges = datasec.lstrip('[').rstrip(']').split(',')
dx1_2 = int(ranges[0].split(':')[0]) - 1
dx2_2 = int(ranges[0].split(':')[1])
dy1_2 = int(ranges[1].split(':')[0]) - 1
dy2_2 = int(ranges[1].split(':')[1])
if (status == 0 and dx2_1 - dx1_1 != dx2_2 - dx1_2):
message = 'ERROR -- SALTSLOT: HDUs ' + infile
message += '[' + str(hdu) + '] and ' + infile + '[' + str(
hdu + 1) + ']'
message += ' have different dimensions'
status = saltprint.err(logfile, message)
# read speed and gain of each exposure
if (status == 0 and hdu == 1):
gainset, status = saltkey.get('GAINSET', struct[0], infile,
logfile)
rospeed, status = saltkey.get('ROSPEED', struct[0], infile,
logfile)
if (rospeed == 'NONE'):
saltprint.log(logfile, " ", verbose)
message = "ERROR -- SALTSLOT: Readout speed is 'NONE' in "
message += "primary keywords of " + infile
status = saltprint.err(logfile, message)
# read raw images
if (status == 0):
imagedata1, status = saltio.readimage(struct, hdu, logfile)
imagedata2, status = saltio.readimage(struct, hdu + 1, logfile)
# gain correction
if (status == 0):
for j in range(len(dbgain)):
if (gainset == dbrate[j] and rospeed == dbspeed[j]
and amplifier == int(dbamp[j])):
try:
gain1 = float(dbgain[j])
imagedata1 *= gain1
except:
mesage = 'ERROR -- SALTSLOT: Cannot perform gain correction on image '
message += infile + '[' + str(hdu) + ']'
status = saltprint.err(logfile, message)
elif (gainset == dbrate[j] and rospeed == dbspeed[j]
and amplifier + 1 == int(dbamp[j])):
try:
gain2 = float(dbgain[j])
imagedata2 *= gain2
except:
mesage = 'ERROR -- SALTSLOT: Cannot perform gain correction on image '
message += infile + '[' + str(hdu + 1) + ']'
status = saltprint.err(logfile, message)
# crosstalk correction
if (status == 0):
revimage1 = imagedata1 * float(xcoeff[amplifier])
revimage2 = imagedata2 * float(xcoeff[amplifier + 1])
for j in range(dx2_1 - dx1_1 + 1):
imagedata1[:, j] -= revimage2[:, dx2_2 - j - 1]
imagedata2[:, j] -= revimage1[:, dx2_1 - j - 1]
# bias subtraction
if (status == 0):
overx_val_1 = []
overx_val_2 = []
for x in range(x1_1, x2_1 + 1):
list_1 = imagedata1[y1_1:y2_1, x] * 1.0
overx_val_1.append(saltstat.median(list_1, logfile))
overlevel_1 = saltstat.median(overx_val_1, logfile)
for x in range(x1_2, x2_2 + 1):
list_2 = imagedata2[y1_2:y2_2, x] * 1.0
overx_val_2.append(saltstat.median(list_2, logfile))
overlevel_2 = saltstat.median(overx_val_2, logfile)
imagedata1 -= overlevel_1
imagedata2 -= overlevel_2
# trim overscan
if (status == 0):
imagedata1 = imagedata1[dy1_1:dy2_1, dx1_1:dx2_1]
imagedata2 = imagedata2[dy1_2:dy2_2, dx1_2:dx2_2]
datasec = '[1:' + str(dx2_1 - dx1_1) + ',1:' + str(dy2_1 -
dy1_1) + ']'
status = saltkey.put('DATASEC', datasec, struct[hdu], infile,
logfile)
status = saltkey.rem('BIASSEC', struct[hdu], infile, logfile)
datasec = '[1:' + str(dx2_2 - dx1_2) + ',1:' + str(dy2_2 -
dy1_2) + ']'
status = saltkey.put('DATASEC', datasec, struct[hdu + 1],
infile, logfile)
status = saltkey.rem('BIASSEC', struct[hdu + 1], infile,
logfile)
# log coefficient table
if (status == 0):
infilename = infile.split('/')
infilename = infilename[len(infilename) - 1]
message = '%25s[%3d] %5.2f %4d %8.6f' % \
(infilename, hdu, gain1, overlevel_1, float(xcoeff[amplifier+1]))
saltprint.log(logfile, message, verbose)
message = '%25s[%3d] %5.2f %4d %8.6f' % \
(infilename, hdu+1, gain2, overlevel_2,float(xcoeff[amplifier]))
saltprint.log(logfile, message, verbose)
# update image in HDU structure
if (status == 0):
struct, status = saltio.writeimage(struct, hdu, imagedata1,
logfile)
struct, status = saltio.writeimage(struct, hdu + 1, imagedata2,
logfile)
return struct, status
def wavemap(hdu,
soldict,
caltype='line',
function='poly',
order=3,
blank=0,
nearest=False,
array_only=False,
clobber=True,
log=None,
verbose=True):
"""Read in an image and a set of wavlength solutions. Calculate the best
wavelength solution for a given dataset and then apply that data set to the
image
return
"""
# set up the time of the observation
dateobs = saltkey.get('DATE-OBS', hdu[0])
utctime = saltkey.get('TIME-OBS', hdu[0])
exptime = saltkey.get('EXPTIME', hdu[0])
instrume = saltkey.get('INSTRUME', hdu[0]).strip()
grating = saltkey.get('GRATING', hdu[0]).strip()
if caltype == 'line':
grang = saltkey.get('GRTILT', hdu[0])
arang = saltkey.get('CAMANG', hdu[0])
else:
grang = saltkey.get('GR-ANGLE', hdu[0])
arang = saltkey.get('AR-ANGLE', hdu[0])
filtername = saltkey.get('FILTER', hdu[0]).strip()
slitname = saltkey.get('MASKID', hdu[0])
slit = st.getslitsize(slitname)
xbin, ybin = saltkey.ccdbin(hdu[0])
timeobs = sr.enterdatetime('%s %s' % (dateobs, utctime))
# check to see if there is more than one solution
if caltype == 'line':
if len(soldict) == 1:
sol = soldict.keys()[0]
slitid = None
if not sr.matchobservations(soldict[sol], instrume, grating, grang,
arang, filtername, slitid):
msg = 'Observations do not match setup for transformation but using the solution anyway'
if log:
log.warning(msg)
for i in range(1, len(hdu)):
if hdu[i].name == 'SCI':
if log:
log.message('Correcting extension %i' % i)
istart = int(0.5 * len(hdu[i].data))
# open up the data
# set up the xarr and initial wavlength solution
xarr = np.arange(len(hdu[i].data[istart]), dtype='int64')
# get the slitid
try:
slitid = saltkey.get('SLITNAME', hdu[i])
except:
slitid = None
#check to see if wavext is already there and if so, then check update
#that for the transformation from xshift to wavelength
if saltkey.found('WAVEXT', hdu[i]):
w_ext = saltkey.get('WAVEXT', hdu[i]) - 1
wavemap = hdu[w_ext].data
function, order, coef = sr.findlinesol(
soldict, istart, nearest, timeobs, exptime, instrume,
grating, grang, arang, filtername, slitid, xarr)
ws = WavelengthSolution.WavelengthSolution(xarr,
xarr,
function=function,
order=order)
ws.set_coef(coef)
for j in range(len(hdu[i].data)):
wavemap[j, :] = ws.value(wavemap[j, :])
if array_only: return wavemap
hdu[w_ext].data = wavemap
continue
# set up a wavelength solution -- still in here for testing MOS data
try:
w_arr = sr.findsol(xarr, soldict, istart, caltype, nearest,
timeobs, exptime, instrume, grating, grang,
arang, filtername, slit, xbin, ybin, slitid,
function, order)
except SALTSpecError as e:
if slitid:
msg = 'SLITID %s: %s' % (slitid, e)
if log:
log.warning(msg)
continue
else:
raise SALTSpecError(e)
if w_arr is None:
w_arr = sr.findsol(xarr, soldict, istart, 'rss', nearest,
timeobs, exptime, instrume, grating, grang,
arang, filtername, slit, xbin, ybin, slitid,
function, order)
# for each line in the data, determine the wavelength solution
# for a given line in the image
wavemap = np.zeros_like(hdu[i].data)
for j in range(len(hdu[i].data)):
# find the wavelength solution for the data
w_arr = sr.findsol(xarr, soldict, j, caltype, nearest, timeobs,
exptime, instrume, grating, grang, arang,
filtername, slit, xbin, ybin, slitid,
function, order)
if w_arr is not None: wavemap[j, :] = w_arr
if array_only: return wavemap
# write out the oimg
hduwav = fits.ImageHDU(data=wavemap,
header=hdu[i].header,
name='WAV')
hdu.append(hduwav)
saltkey.new('WAVEXT',
len(hdu) - 1, 'Extension for Wavelength Map', hdu[i])
return hdu
def gain(struct,mult=True,usedb=False, dblist=None, ampccd=2, log=None, verbose=True):
"""gain processes a image hduList and gain corrects each amplifier. It can
either use gain settings in the header or those supplied in a config file
which would be suppleid in the dblist (see helpfile for structure of the
config file). If variance frames exist, it will update those for changes
in the header value as well.
In the end, it will update the gain with a value of one signfing the data
has been transformed into e- from ADU
The program will look for the non-linear gain settings which are given by:
e = GAIN*(1 + GAIN1*E-6*ADU)*ADU
mult--if true, multiple the gains
usedb--use the values in the dblist, if false use the header values
dblist--values for the gain and readnoise from the
ampccd--number of amplifiers per ccd
dblist should have the following lists: speed, rate, gain, noise, bias, amp
"""
#get the infile name
infile=saltkey.getimagename(struct[0])
#how many science extensions
nsciext = saltkey.get('NSCIEXT',struct[0])
#how many data extensions
nextend = saltkey.get('NSCIEXT',struct[0])
# how many amplifiers?
amplifiers = ampccd*saltkey.get('NCCDS',struct[0])
#read the gain and rospeed for the image
gainset = saltkey.get('GAINSET',struct[0])
rospeed = saltkey.get('ROSPEED',struct[0])
#loop through each amplifier and gain correct it
if log:
message = '%28s %6s %5s %3s %5s %5s' \
% ('HDU','GAIN','SPEED','AMP','GAIN','NOISE')
log.message('\n ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
for i in range(nsciext):
hdu = i + 1
amp = i%amplifiers+1
#get the gain and rdnoise values for the array
if usedb:
gain, rdnoise=get_values(dblist, gainset, rospeed, amp)
gain1=0
else:
gain = saltkey.get('GAIN',struct[hdu])
rdnoise = saltkey.get('RDNOISE',struct[hdu])
try:
gain1=saltkey.get('GAIN1',struct[hdu])
except:
gain1=0
if mult:
#correct the gain
gainmult=1
try:
data=struct[hdu].data
struct[hdu].data=gain*data+gain1*data**2
except Exception as e:
msg='Cannot gain correct %s[%i] because %s' % (infile, hdu, e)
raise SaltError(msg)
#correct the variance frame
if saltkey.found('VAREXT', struct[hdu]):
vhdu=saltkey.get('VAREXT', struct[hdu])
try:
vdata=struct[vhdu].data
struct[vhdu].data=vdata*gain*(1+2*gain1*1e-6*data)
except Exception as e:
msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e)
raise SaltError(msg)
else:
gainmult=gain
#update the headers
if usedb:
saltkey.put('GAIN',gain,struct[hdu])
saltkey.put('RDNOISE',rdnoise,struct[hdu])
#add a keyword indicating what action was taken
saltkey.new('GAINMULT',gainmult,'Gain multiplication', struct[hdu])
#if logging is true, then print out the following information
if log:
message = '%25s[%1d] %6s %5s %2s %6.2f %5.2f' \
% (infile,hdu,gainset,rospeed,amp, gain, rdnoise)
log.message(message, with_header=False, with_stdout=verbose)
#just to make it look pretty
if log:
log.message('', with_header=False, with_stdout=verbose)
return struct
def addWCS(struct, xbin, ybin, outfile):
"""WCS keywords"""
saltsafekey.new('CRPIX1',1,'WCS: X reference pixel',struct,outfile)
saltsafekey.new('CRPIX2',1,'WCS: Y reference pixel',struct,outfile)
saltsafekey.new('CRVAL1',float(xbin), 'WCS: X reference coordinate value',struct,outfile)
saltsafekey.new('CRVAL2',float(ybin), 'WCS: Y reference coordinate value',struct,outfile)
saltsafekey.new('CDELT1',float(xbin),'WCS: X pixel size',struct,outfile)
saltsafekey.new('CDELT2',float(ybin),'WCS: Y pixel size',struct,outfile)
saltsafekey.new('CTYPE1','pixel','X type',struct,outfile)
saltsafekey.new('CTYPE2','pixel','Y type',struct,outfile)
return struct
def imcombine(infiles, method='average', reject=None, mask=True, weight=True, \
blank=0, scale=None, statsec=None, lthresh=3, hthresh=3):
"""Fast combine is for special circumstances where the files and the combination
can be down one by one and not all of the data need to be read in all at once.
It computes the average of the files input in infile
Input Variables:
infiles: List of files to combine
method: Combination method to use, either average or median
reject: Reject outliers using different methods
-ccdclip--reject the pixels according to the CCD parameters
-sigclip--reject the pixels according to a sigma clipping
mask: Whether to correct the data using the BPM frame
weight: If true, weight by the inverse of the variant frame
blank: Value if zero pixels are combined
scale: Method to scale the images by
statsec: Region for determining statistics if images are to be scaled
lthresh: low rejection threshold
hthresh: high rejection threshold
Return Variables:
hdustruct: Output struct
"""
hdu_list=[]
#read in all of the data
for infile in infiles:
#hdu_list.append(saltio.openfits(infile))
hdu_list.append(pyfits.open(infile, memmap=True))
#Copy the first image and create the HDU for the other images
try:
outhdu=hdu_list[0]
except Exception as e:
message='Cannot create output hduList because %s' % e
raise SaltError(message)
#determine the size of a single extension. Assumes all arrays are the same size
hdu=hdu_list[0]
#For each science extension, construct the array of arrays to go into that
#Three arrays need to be constructed including one for the data array,
#one for the inv variance array, and one for the BPM
#This assumes all the data are the same shape and size
nimages=len(hdu_list)
mem_lim=0.2
if reject or weight or mask:
mem_lim=mem_lim/4.0
for i in range(len(outhdu)):
if outhdu[i].name=='SCI':
nb=outhdu[i].data.nbytes
outhdu[i].data *= 0.0
#let's set the limit at 0.2 GB of memory
if nb*nimages/1e9>mem_lim:
imod=int((nb*nimages/1e9)/mem_lim)+1
y2,x2=outhdu[i].data.shape
ystep=y2/imod
xstep=x2/imod
for j in range(imod):
for k in range(imod):
datasec=[j*ystep, min(y2,(j+1)*ystep), k*xstep, min(x2, (k+1)*xstep)]
outhdu=hducombine(hdu_list, outhdu, ext=i, method=method, datasec=datasec,
reject=reject, mask=mask,weight=weight, scale=scale, statsec=statsec, blank=blank,
lthresh=lthresh, hthresh=hthresh)
#this step is needed to remove data arrays that are read into memory
#--not sure what object they are linked to, but this works
try:
for j in range(nimages): del hdu_list[j][i].data
except:
pass
else:
outhdu=hducombine(hdu_list, outhdu, ext=i, method=method, datasec=None,
reject=reject, mask=mask,weight=weight, scale=scale, statsec=statsec,
lthresh=lthresh, hthresh=hthresh)
#Add any header frames
saltkey.new('NCOMBINE',nimages,'Number of images combines', outhdu[0])
return outhdu
try:
vdata=struct[vhdu].data
struct[vhdu].data=vdata*gain*(1+2*gain1*1e-6*data)
except Exception, e:
msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e)
raise SaltError(msg)
else:
gainmult=gain
#update the headers
if usedb:
saltkey.put('GAIN',gain,struct[hdu])
saltkey.put('RDNOISE',rdnoise,struct[hdu])
#add a keyword indicating what action was taken
saltkey.new('GAINMULT',gainmult,'Gain multiplication', struct[hdu])
#if logging is true, then print out the following information
if log:
message = '%25s[%1d] %6s %5s %2s %6.2f %5.2f' \
% (infile,hdu,gainset,rospeed,amp, gain, rdnoise)
log.message(message, with_header=False, with_stdout=verbose)
#just to make it look pretty
if log:
log.message('', with_header=False, with_stdout=verbose)
return struct
def get_values(dblist, gainset, rospeed, amp):
"""Get values for gain and rdnoise from the dblist. The input of the dblist should be:
nextend+=nsciext
#create the badpixelframes
for i in range(1, nsciext+1):
try:
hdu=createbadpixel(struct, badpixelstruct, i, nextend+i)
except Exception, e:
msg='Could not create bad pixel extension in ext %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header.update('BPMEXT',nextend+i, comment='Extension for Bad Pixel Mask')
struct.append(hdu)
nextend+=nsciext
#update the number of extensions
saltkey.new('NSCIEXT',nsciext,'Number of science extensions', struct[0])
saltkey.new('NEXTEND',nextend,'Number of data extensions', struct[0])
return struct
def CreateVariance(inhdu, sci_ext, var_ext):
"""Create a variance hdu from an input hdu"""
rdnoise=inhdu.header['RDNOISE']
gain=inhdu.header['GAIN']
#create the variance array
data=inhdu.data.copy()
if (data <=0).any():
j=np.where(data>0)
min_pos=data[j].min()
j=np.where(data<=0)
def salteditkey(images,outimages,outpref, keyfile, recfile=None,clobber=False,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
#is key file defined
saltio.argdefined('keyfile',keyfile)
keyfile = keyfile.strip()
saltio.fileexists(keyfile)
# if the data are the same, set up to use update instead of write
openmode='copyonwrite'
if (infiles!=outfiles): openmode='copyonwrite'
# determine the date of the observations
obsdate=saltstring.makeobsdatestr(infiles, 1,9)
if len(obsdate)!=8:
message = 'Either FITS files from multiple dates exist, '
message += 'or raw FITS files exist with non-standard names.'
log.warning(message)
# FITS file columns to record keyword changes
fitcol = []
keycol = []
oldcol = []
newcol = []
# Set up the rules to change the files
keyedits=readkeyfile(keyfile, log=log, verbose=verbose)
#now step through the images
for img, oimg in zip(infiles, outfiles):
#determine the appropriate keyword edits for the image
klist=[]
for frange in keyedits:
if checkfitsfile(img, frange, keyedits[frange]):
klist.append(keyedits[frange][3])
if klist:
#open up the new files
struct = saltio.openfits(img,mode=openmode)
struct.verify('fix')
for kdict in klist:
for keyword in kdict:
#record the changes
value=kdict[keyword]
fitcol.append(img)
keycol.append(keyword)
newcol.append(value)
try:
oldcol.append(struct[0].header[keyword].lstrip())
except:
oldcol.append('None')
#update the keyword
if saltkey.found(keyword, struct[0]):
try:
saltkey.put(keyword,value,struct[0])
message='\tUpdating %s in %s to %s' % (keyword, os.path.basename(img), value)
log.message(message, with_header=False, with_stdout=verbose)
except Exception, e:
message = 'Could not update %s in %s because %s' % (keyword, img, str(e))
raise SaltError(message)
else:
try:
saltkey.new(keyword.strip(),value,'Added Comment',struct[0])
message='\tAdding %s in %s to %s' % (keyword, os.path.basename(img), value)
log.message(message, with_header=False, with_stdout=verbose)
except Exception,e :
message = 'Could not update %s in %s because %s' % (keyword, img, str(e))
raise SaltError(message)
#updat the history keywords
#fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
#saltkey.housekeeping(struct[0],'SAL-EDT', 'Keywords updated by SALTEDITKEY', hist)
#write the file out
if openmode=='update':
saltio.updatefits(struct)
message = 'Updated file ' + os.path.basename(oimg)
else:
saltio.writefits(struct, oimg, clobber)
message = 'Created file ' + os.path.basename(oimg)
log.message(message, with_header=False, with_stdout=True)
struct.close()
def rectify(hdu, soldict, caltype='line', function='poly', order=3, inttype='interp',
w1=None, w2=None, dw=None, nw=None, blank=0, pixscale=0.0, time_interp=False,
conserve=False, nearest=False, clobber=True, log=None, verbose=True):
"""Read in an image and a set of wavlength solutions. Calculate the best
wavelength solution for a given dataset and then apply that data set to the
image
return
"""
# set the basic values
set_w1 = (w1 is None)
set_w2 = (w2 is None)
set_dw = (dw is None)
set_nw = (nw is None)
# set up the time of the observation
dateobs = saltkey.get('DATE-OBS', hdu[0])
utctime = saltkey.get('TIME-OBS', hdu[0])
exptime = saltkey.get('EXPTIME', hdu[0])
instrume = saltkey.get('INSTRUME', hdu[0]).strip()
grating = saltkey.get('GRATING', hdu[0]).strip()
if caltype == 'line':
grang = saltkey.get('GRTILT', hdu[0])
arang = saltkey.get('CAMANG', hdu[0])
else:
grang = saltkey.get('GR-ANGLE', hdu[0])
arang = saltkey.get('AR-ANGLE', hdu[0])
filtername = saltkey.get('FILTER', hdu[0]).strip()
slitname = saltkey.get('MASKID', hdu[0])
slit = st.getslitsize(slitname)
xbin, ybin = saltkey.ccdbin(hdu[0])
timeobs = enterdatetime('%s %s' % (dateobs, utctime))
# check to see if there is more than one solution
if caltype == 'line':
if len(soldict) == 1:
sol = soldict.keys()[0]
slitid = None
if not matchobservations(
soldict[sol], instrume, grating, grang, arang, filtername, slitid):
msg = 'Observations do not match setup for transformation but using the solution anyway'
if log:
log.warning(msg)
for i in range(1, len(hdu)):
if hdu[i].name == 'SCI':
if log:
log.message('Correcting extension %i' % i)
istart = int(0.5 * len(hdu[i].data))
# open up the data
# set up the xarr and initial wavlength solution
xarr = np.arange(len(hdu[i].data[istart]), dtype='int64')
# get the slitid
try:
slitid = saltkey.get('SLITNAME', hdu[i])
except:
slitid = None
# set up a wavelength solution
try:
w_arr = findsol(xarr, soldict, istart, caltype, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername,
slit, xbin, ybin, slitid, function, order)
except SALTSpecError as e:
if slitid:
msg = 'SLITID %s: %s' % (slitid, e)
if log:
log.warning(msg)
continue
else:
raise SALTSpecError(e)
if w_arr is None:
w_arr = findsol(xarr, soldict, istart, 'rss', nearest, timeobs, exptime, instrume, grating, grang, arang, filtername,
slit, xbin, ybin, slitid, function, order)
# set up the output x-axis
if set_w1:
w1 = w_arr.min()
if set_w2:
w2 = w_arr.max()
if set_nw:
nw = len(xarr)
if set_dw:
dw = float(w2 - w1) / nw
nw_arr = createoutputxaxis(w1, w2, nw)
# setup the VARIANCE and BPM frames
if saltkey.found('VAREXT', hdu[i]):
varext = saltkey.get('VAREXT', hdu[i])
else:
varext = None
# setup the BPM frames
if saltkey.found('BPMEXT', hdu[i]):
bpmext = saltkey.get('BPMEXT', hdu[i])
else:
bpmext = None
# for each line in the data, determine the wavelength solution
# for a given line in the image
for j in range(len(hdu[i].data)):
# find the wavelength solution for the data
w_arr = findsol(xarr, soldict, j, caltype, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername,
slit, xbin, ybin, slitid, function, order)
# apply that wavelength solution to the data
if w_arr is not None:
try:
hdu[i].data[
j,
:] = st.interpolate(
nw_arr,
w_arr,
hdu[i].data[
j,
:],
inttype,
left=blank,
right=blank)
except Exception as e:
hdu[i].data[j, :] = hdu[i].data[j, :] * 0.0 + blank
msg = 'In row %i, solution cannot be found due to %s' % (
i, e)
# correct the variance frame
if varext:
try:
hdu[varext].data[
j,
:] = st.interpolate(
nw_arr,
w_arr,
hdu[varext].data[
j,
:],
inttype,
left=blank,
right=blank)
except Exception as e:
msg = 'In row %i, solution cannot be found due to %s' % (
i, e)
# correct the BPM frame
if bpmext:
try:
hdu[bpmext].data[
j,
:] = st.interpolate(
nw_arr,
w_arr,
hdu[bpmext].data[
j,
:],
inttype,
left=blank,
right=blank)
except Exception as e:
msg = 'In row %i, solution cannot be found due to %s' % (
i, e)
else:
hdu[i].data[j, :] = hdu[i].data[j, :] * 0.0 + blank
if conserve:
hdu[i].data = hdu[i].data / dw
if varext:
hdu[varext].data = hdu[varext].data / dw
# Add WCS information
saltkey.new('CTYPE1', 'LAMBDA', 'Coordinate Type', hdu[i])
saltkey.new('CTYPE2', 'PIXEL', 'Coordinate Type', hdu[i])
saltkey.new(
'CD1_1',
dw,
'WCS: Wavelength Dispersion in angstrom/pixel',
hdu[i])
saltkey.new('CD2_1', 0.0, 'WCS: ', hdu[i])
saltkey.new('CD1_2', 0.0, 'WCS: ', hdu[i])
saltkey.new('CD2_2', ybin * pixscale, 'WCS: ', hdu[i])
saltkey.new('CRPIX1', 0.0, 'WCS: X Reference pixel', hdu[i])
saltkey.new('CRPIX2', 0.0, 'WCS: Y Reference pixel', hdu[i])
saltkey.new('CRVAL1', w1, 'WCS: X Reference pixel', hdu[i])
saltkey.new('CRVAL2', 0.0, 'WCS: Y Reference pixel', hdu[i])
saltkey.new('CDELT1', 1.0, 'WCS: X pixel size', hdu[i])
saltkey.new('CDELT2', 1.0, 'WCS: Y pixel size', hdu[i])
saltkey.new('DC-FLAG', 0, 'Dispesion Corrected image', hdu[i])
return hdu
def addWCS(struct, xbin, ybin, outfile):
"""WCS keywords"""
saltsafekey.new('CRPIX1', 1, 'WCS: X reference pixel', struct, outfile)
saltsafekey.new('CRPIX2', 1, 'WCS: Y reference pixel', struct, outfile)
saltsafekey.new('CRVAL1', float(xbin), 'WCS: X reference coordinate value',
struct, outfile)
saltsafekey.new('CRVAL2', float(ybin), 'WCS: Y reference coordinate value',
struct, outfile)
saltsafekey.new('CDELT1', float(xbin), 'WCS: X pixel size', struct,
outfile)
saltsafekey.new('CDELT2', float(ybin), 'WCS: Y pixel size', struct,
outfile)
saltsafekey.new('CTYPE1', 'pixel', 'X type', struct, outfile)
saltsafekey.new('CTYPE2', 'pixel', 'Y type', struct, outfile)
return struct
newutc=salttime.dec2sex(ntime)
except Exception,e:
msg='Could not update UTC in %i header of image %s because %s' % (ext, infile, e)
raise SaltError(msg)
return struct
# update the headers
if utc==saltsafekey.get('UTC-OBS', struct):
expt_string='%5.4f' % real_expt
td_string='%5.4f' % tdiff
if not saltsafekey.found('DUTC', struct):
try:
saltsafekey.put('UTC-OBS', newutc, struct, infile)
saltsafekey.put('TIME-OBS', newutc, struct, infile)
saltsafekey.new('DWETIME', expt_string, 'Dwell Time', struct, infile)
saltsafekey.new('DUTC', td_string, 'Change in UTC time', struct, infile)
except Exception, e:
msg='Could not update %i header of image %s because %s' % (ext, infile, e)
raise SaltIOError(msg)
else:
try:
saltsafekey.put('UTC-OBS', newutc, struct, infile)
saltsafekey.put('TIME-OBS', newutc, struct, infile)
saltsafekey.put('DWETIME', real_expt, struct, infile)
saltsafekey.put('DUTC', tdiff, struct, infile)
except Exception, e:
msg='Could not update %i header of image %s because %s' % (ext, infile, e)
raise SaltError(msg)
else:
raise SaltIOError('Frame missing from list of times')
def bias(struct,
subover=True,
trim=True,
subbias=False,
bstruct=None,
median=False,
function='polynomial',
order=3,
rej_lo=3,
rej_hi=3,
niter=10,
plotover=False,
log=None,
verbose=True):
"""Bias subtracts the bias levels from a frame. It will fit and subtract the overscan
region, trim the images, and subtract a master bias if required.
struct--image structure
subover--subtract the overscan region
trim--trim the image
subbias--subtract master bias
bstruct--master bias image structure
median--use the median instead of mean in image statistics
function--form to fit to the overscan region
order--order for the function
rej_lo--sigma of low points to reject in the fit
rej_hi--sigma of high points to reject in the fit
niter--number of iterations
log--saltio log for recording information
verbose--whether to print to stdout
"""
infile = saltkey.getimagename(struct[0])
# how many extensions?
nsciext = saltkey.get('NSCIEXT', struct[0])
nextend = saltkey.get('NEXTEND', struct[0])
nccd = saltkey.get('NCCDS', struct[0])
# how many amplifiers?--this is hard wired
amplifiers = 2 * nccd
#log the process
if subover and log:
message = '%28s %7s %5s %4s %6s' % \
('HDU','Overscan','Order','RMS','Niter')
log.message(
'\n --------------------------------------------------',
with_header=False,
with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' --------------------------------------------------',
with_header=False,
with_stdout=verbose)
if (plotover):
plt.figure(1)
plt.axes([0.1, 0.1, 0.8, 0.8])
plt.xlabel('CCD Column')
plt.ylabel('Pixel Counts (e-)')
plt.ion()
#loop through the extensions and subtract the bias
for i in range(1, nsciext + 1):
if struct[i].name == 'SCI':
#get the bias section
biassec = saltkey.get('BIASSEC', struct[i])
y1, y2, x1, x2 = saltio.getSection(biassec, iraf_format=True)
#get the data section
datasec = saltkey.get('DATASEC', struct[i])
dy1, dy2, dx1, dx2 = saltio.getSection(datasec, iraf_format=True)
#setup the overscan region
if subover:
yarr = np.arange(y1, y2, dtype=float)
data = struct[i].data
odata = struct[i].data[y1:y2, x1:x2]
if median:
odata = np.median((struct[i].data[y1:y2, x1:x2]), axis=1)
olevel = np.median((struct[i].data[y1:y2, x1:x2]))
saltkey.new('OVERSCAN', '%f' % (olevel),
'Overscan median value', struct[i])
else:
odata = np.mean((struct[i].data[y1:y2, x1:x2]), axis=1)
olevel = np.mean((struct[i].data[y1:y2, x1:x2]))
saltkey.new('OVERSCAN', '%f' % (olevel),
'Overscan mean value', struct[i])
#fit the overscan region
ifit=saltfit.interfit(yarr, odata, function=function, \
order=order, thresh=rej_hi, niter=niter)
try:
ifit.interfit()
coeffs = ifit.coef
ofit = ifit(yarr)
omean, omed, osigma = saltstat.iterstat((odata - ofit),
sig=3,
niter=5)
except ValueError:
#catch the error if it is a zero array
ofit = np.array(yarr) * 0.0
osigma = 0.0
except TypeError:
#catch the error if it is a zero array
ofit = np.array(yarr) * 0.0
osigma = 0.0
#if it hasn't been already, convert image to
#double format
struct[i].data = 1.0 * struct[i].data
try:
struct[i].header.remove('BZERO')
struct[i].header.remove('BSCALE')
except:
pass
#subtract the overscan region
for j in range(len(struct[i].data[0])):
struct[i].data[y1:y2, j] -= ofit
#report the information
if log:
message = '%25s[%1d] %8.2f %3d %7.2f %3d' % \
(infile, i, olevel, order, osigma, niter)
log.message(message,
with_stdout=verbose,
with_header=False)
#add the statistics to the image header
saltkey.new('OVERRMS', '%f' % (osigma), 'Overscan RMS value',
struct[i])
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu = saltkey.get('VAREXT', struct[i])
try:
vdata = struct[vhdu].data
#The bias level should not be included in the noise from the signal
for j in range(len(struct[i].data[0])):
vdata[y1:y2, j] -= ofit
#add a bit to make sure that the minimum error is the rednoise
rdnoise = saltkey.get('RDNOISE', struct[i])
vdata[vdata < rdnoise**2] = rdnoise**2
struct[vhdu].data = vdata + osigma**2
except Exception, e:
msg = 'Cannot update the variance frame in %s[%i] because %s' % (
infile, vhdu, e)
raise SaltError(msg)
#plot the overscan region
if plotover:
plt.plot(yarr, odata)
plt.plot(yarr, ofit)
#trim the data and update the headers
if trim:
struct[i].data = struct[i].data[dy1:dy2, dx1:dx2]
datasec = '[1:' + str(dx2 - dx1) + ',1:' + str(dy2 - dy1) + ']'
saltkey.put('DATASEC', datasec, struct[i])
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu = saltkey.get('VAREXT', struct[i])
struct[vhdu].data = struct[vhdu].data[dy1:dy2, dx1:dx2]
datasec = '[1:' + str(dx2 - dx1) + ',1:' + str(dy2 -
dy1) + ']'
saltkey.put('DATASEC', datasec, struct[vhdu])
#update the BPM frame
if saltkey.found('BPMEXT', struct[i]):
bhdu = saltkey.get('BPMEXT', struct[i])
struct[bhdu].data = struct[bhdu].data[dy1:dy2, dx1:dx2]
datasec = '[1:' + str(dx2 - dx1) + ',1:' + str(dy2 -
dy1) + ']'
saltkey.put('DATASEC', datasec, struct[bhdu])
#subtract the master bias if necessary
if subbias and bstruct:
struct[i].data -= bstruct[i].data
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu = saltkey.get('VAREXT', struct[i])
try:
vdata = struct[vhdu].data
struct[vhdu].data = vdata + bstruct[vhdu].data
except Exception, e:
msg = 'Cannot update the variance frame in %s[%i] because %s' % (
infile, vhdu, e)
raise SaltError(msg)
nextend+=nsciext
#create the badpixelframes
for i in range(1, nsciext+1):
try:
hdu=createbadpixel(struct, badpixelstruct, i, nextend+i)
except Exception, e:
msg='Could not create bad pixel extension in ext %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header.update('BPMEXT',nextend+i, comment='Extension for Bad Pixel Mask')
struct.append(hdu)
nextend+=nsciext
#update the number of extensions
saltkey.new('NSCIEXT',nsciext,'Number of science extensions', struct[0])
saltkey.new('NEXTEND',nextend,'Number of data extensions', struct[0])
return struct
def CreateVariance(inhdu, sci_ext, var_ext):
"""Create a variance hdu from an input hdu"""
rdnoise=inhdu.header['RDNOISE']
gain=inhdu.header['GAIN']
#create the variance array
data=inhdu.data.copy()
if (data <=0).any():
j=np.where(data>0)
min_pos=data[j].min()
j=np.where(data<=0)
def rectify(hdu, soldict, caltype='line', function='poly', order=3, inttype='interp',
w1=None, w2=None, dw=None, nw=None, blank=0, pixscale=0.0, time_interp=False,
conserve=False, nearest=False, clobber=True, log=None, verbose=True):
"""Read in an image and a set of wavlength solutions. Calculate the best
wavelength solution for a given dataset and then apply that data set to the
image
return
"""
# set the basic values
set_w1 = (w1 is None)
set_w2 = (w2 is None)
set_dw = (dw is None)
set_nw = (nw is None)
# set up the time of the observation
dateobs = saltkey.get('DATE-OBS', hdu[0])
utctime = saltkey.get('TIME-OBS', hdu[0])
exptime = saltkey.get('EXPTIME', hdu[0])
instrume = saltkey.get('INSTRUME', hdu[0]).strip()
grating = saltkey.get('GRATING', hdu[0]).strip()
if caltype == 'line':
grang = saltkey.get('GRTILT', hdu[0])
arang = saltkey.get('CAMANG', hdu[0])
else:
grang = saltkey.get('GR-ANGLE', hdu[0])
arang = saltkey.get('AR-ANGLE', hdu[0])
filtername = saltkey.get('FILTER', hdu[0]).strip()
slitname = saltkey.get('MASKID', hdu[0])
slit = st.getslitsize(slitname)
xbin, ybin = saltkey.ccdbin(hdu[0])
timeobs = enterdatetime('%s %s' % (dateobs, utctime))
# check to see if there is more than one solution
if caltype == 'line':
if len(soldict) == 1:
sol = soldict.keys()[0]
slitid = None
if not matchobservations(
soldict[sol], instrume, grating, grang, arang, filtername, slitid):
msg = 'Observations do not match setup for transformation but using the solution anyway'
if log:
log.warning(msg)
for i in range(1, len(hdu)):
if hdu[i].name == 'SCI':
if log:
log.message('Correcting extension %i' % i)
istart = int(0.5 * len(hdu[i].data))
# open up the data
# set up the xarr and initial wavlength solution
xarr = np.arange(len(hdu[i].data[istart]), dtype='int64')
# get the slitid
try:
slitid = saltkey.get('SLITNAME', hdu[i])
except:
slitid = None
# set up a wavelength solution
try:
w_arr = findsol(xarr, soldict, istart, caltype, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername,
slit, xbin, ybin, slitid, function, order)
except SALTSpecError as e:
if slitid:
msg = 'SLITID %s: %s' % (slitid, e)
if log:
log.warning(msg)
continue
else:
raise SALTSpecError(e)
if w_arr is None:
w_arr = findsol(xarr, soldict, istart, 'rss', nearest, timeobs, exptime, instrume, grating, grang, arang, filtername,
slit, xbin, ybin, slitid, function, order)
# set up the output x-axis
if set_w1:
w1 = w_arr.min()
if set_w2:
w2 = w_arr.max()
if set_nw:
nw = len(xarr)
if set_dw:
dw = float(w2 - w1) / nw
nw_arr = createoutputxaxis(w1, w2, nw)
# setup the VARIANCE and BPM frames
if saltkey.found('VAREXT', hdu[i]):
varext = saltkey.get('VAREXT', hdu[i])
else:
varext = None
# setup the BPM frames
if saltkey.found('BPMEXT', hdu[i]):
bpmext = saltkey.get('BPMEXT', hdu[i])
else:
bpmext = None
# for each line in the data, determine the wavelength solution
# for a given line in the image
for j in range(len(hdu[i].data)):
# find the wavelength solution for the data
w_arr = findsol(xarr, soldict, j, caltype, nearest, timeobs, exptime, instrume, grating, grang, arang, filtername,
slit, xbin, ybin, slitid, function, order)
# apply that wavelength solution to the data
if w_arr is not None:
try:
hdu[i].data[
j,
:] = st.interpolate(
nw_arr,
w_arr,
hdu[i].data[
j,
:],
inttype,
left=blank,
right=blank)
except Exception as e:
hdu[i].data[j, :] = hdu[i].data[j, :] * 0.0 + blank
msg = 'In row %i, solution cannot be found due to %s' % (
i, e)
# correct the variance frame
if varext:
try:
hdu[varext].data[
j,
:] = st.interpolate(
nw_arr,
w_arr,
hdu[varext].data[
j,
:],
inttype,
left=blank,
right=blank)
except Exception as e:
msg = 'In row %i, solution cannot be found due to %s' % (
i, e)
# correct the BPM frame
if bpmext:
try:
hdu[bpmext].data[
j,
:] = st.interpolate(
nw_arr,
w_arr,
hdu[bpmext].data[
j,
:],
inttype,
left=blank,
right=blank)
except Exception as e:
msg = 'In row %i, solution cannot be found due to %s' % (
i, e)
else:
hdu[i].data[j, :] = hdu[i].data[j, :] * 0.0 + blank
if conserve:
hdu[i].data = hdu[i].data / dw
if varext:
hdu[varext].data = hdu[varext].data / dw
# Add WCS information
saltkey.new('CTYPE1', 'LAMBDA', 'Coordinate Type', hdu[i])
saltkey.new('CTYPE2', 'PIXEL', 'Coordinate Type', hdu[i])
saltkey.new(
'CD1_1',
dw,
'WCS: Wavelength Dispersion in angstrom/pixel',
hdu[i])
saltkey.new('CD2_1', 0.0, 'WCS: ', hdu[i])
saltkey.new('CD1_2', 0.0, 'WCS: ', hdu[i])
saltkey.new('CD2_2', ybin * pixscale, 'WCS: ', hdu[i])
saltkey.new('CRPIX1', 0.0, 'WCS: X Reference pixel', hdu[i])
saltkey.new('CRPIX2', 0.0, 'WCS: Y Reference pixel', hdu[i])
saltkey.new('CRVAL1', w1, 'WCS: X Reference pixel', hdu[i])
saltkey.new('CRVAL2', 0.0, 'WCS: Y Reference pixel', hdu[i])
saltkey.new('CDELT1', 1.0, 'WCS: X pixel size', hdu[i])
saltkey.new('CDELT2', 1.0, 'WCS: Y pixel size', hdu[i])
saltkey.new('DC-FLAG', 0, 'Dispesion Corrected image', hdu[i])
return hdu
reject=reject, mask=mask,weight=weight, scale=scale, statsec=statsec, blank=blank,
lthresh=lthresh, hthresh=hthresh)
#this step is needed to remove data arrays that are read into memory
#--not sure what object they are linked to, but this works
try:
for j in range(nimages): del hdu_list[j][i].data
except:
pass
else:
outhdu=hducombine(hdu_list, outhdu, ext=i, method=method, datasec=None,
reject=reject, mask=mask,weight=weight, scale=scale, statsec=statsec,
lthresh=lthresh, hthresh=hthresh)
#Add any header frames
saltkey.new('NCOMBINE',nimages,'Number of images combines', outhdu[0])
return outhdu
def hducombine(hdu_list, outhdu, ext, method='average', datasec=None, reject=None, mask=False, weight=False, scale=None, statsec=None, blank=0, lthresh=3, hthresh=3):
"""Combine a set of images in imlist
"""
#set up i as the extionsion variable as shortcut
i=ext
nimages=len(hdu_list)
#set all the data arrays
data_list=[]
gain=np.zeros(nimages)
rdnoise=np.zeros(nimages)
hdu[varext].data[j,:]=st.interpolate(nw_arr, w_arr, hdu[varext].data[j,:], inttype, left=blank, right=blank)
except Exception, e:
msg='In row %i, solution cannot be found due to %s' % (i, e)
#correct the BPM frame
if bpmext:
try:
hdu[bpmext].data[j,:]=st.interpolate(nw_arr, w_arr, hdu[bpmext].data[j,:], inttype, left=blank, right=blank)
except Exception, e:
msg='In row %i, solution cannot be found due to %s' % (i, e)
else:
hdu[i].data[j,:]=hdu[i].data[j,:]*0.0+blank
#Add WCS information
saltkey.new('CTYPE1', 'LAMBDA', 'Coordinate Type', hdu[i])
saltkey.new('CTYPE2', 'PIXEL', 'Coordinate Type', hdu[i])
saltkey.new('CD1_1', dw, 'WCS: Wavelength Dispersion in angstrom/pixel', hdu[i])
saltkey.new('CD2_1', 0.0, 'WCS: ', hdu[i])
saltkey.new('CD1_2', 0.0, 'WCS: ', hdu[i])
saltkey.new('CD2_2', ybin*pixscale, 'WCS: ', hdu[i])
saltkey.new('CRPIX1', 0.0, 'WCS: X Reference pixel', hdu[i])
saltkey.new('CRPIX2', 0.0, 'WCS: Y Reference pixel', hdu[i])
saltkey.new('CRVAL1', w1, 'WCS: X Reference pixel', hdu[i])
saltkey.new('CRVAL2', 0.0, 'WCS: Y Reference pixel', hdu[i])
saltkey.new('CDELT1', 1.0, 'WCS: X pixel size', hdu[i])
saltkey.new('CDELT2', 1.0, 'WCS: Y pixel size', hdu[i])
return hdu
def makeinterpsolution(xarr, soldict, timeobs, exptime, instrume, grating, grang, arang, filtername, slitid):
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None,
subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3,
niter=5, interp='linear', clobber=False, logfile='salt.log',verbose=True):
"""Convert MEF HRS data into a single image. If variance frames and BPMs, then
convert them to the same format as well. Returns an MEF image but that is
combined into a single frame
"""
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
if saltio.checkfornone(obslogfile) is None:
raise SaltError('Obslog file is required')
# Delete the obslog file if it already exists
if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile):
if os.path.isfile(obslogfile): saltio.delete(obslogfile)
#read in the obsveration log or create it
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
else:
obsstruct=saltio.openfits(obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('OBJECT')
biaslist=filename[ccdtype=='Bias']
masterbias_dict={}
if log: log.message('Processing Bias Frames')
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bgph'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose, with_header=False)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
subover=subover, trim=trim, subbias=False, imstack=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist)
saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list)
#create the master bias frame
for i in list(masterbias_dict.keys()):
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys, x1=5, x2=13)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#apply full reductions to the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'mbgph'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#get master bias frame
masterbias=get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list)
if masterbias:
subbias=True
bstruct=saltio.openfits(masterbias)
else:
subbias=False
bstruct=None
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
subover=subover, trim=trim, subbias=subbias, imstack=True,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist)
saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
return
def make_mosaic(struct, gap, xshift, yshift, rotation, interp_type='linear',
boundary='constant', constant=0, geotran=True, fill=False,
cleanup=True, log=None, verbose=False):
"""Given a SALT image struct, combine each of the individual amplifiers and
apply the geometric CCD transformations to the image
"""
# get the name of the file
infile = saltkey.getimagename(struct[0], base=True)
outpath = './'
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = \
saltkey.instrumid(struct)
# how many amplifiers?
nsciext = saltkey.get('NSCIEXT', struct[0])
nextend = saltkey.get('NEXTEND', struct[0])
nccds = saltkey.get('NCCDS', struct[0])
amplifiers = nccds * 2
if nextend > nsciext:
varframe = True
else:
varframe = False
# CCD geometry coefficients
if (instrume == 'RSS' or instrume == 'PFIS'):
xsh = [0., xshift[0], 0., xshift[1]]
ysh = [0., yshift[0], 0., yshift[1]]
rot = [0., rotation[0], 0., rotation[1]]
elif instrume == 'SALTICAM':
xsh = [0., xshift[0], 0.]
ysh = [0., yshift[0], 0.]
rot = [0., rotation[0], 0]
# how many extensions?
nextend = saltkey.get('NEXTEND', struct[0])
# CCD on-chip binning
xbin, ybin = saltkey.ccdbin(struct[0])
# create temporary primary extension
outstruct = []
outstruct.append(struct[0])
# define temporary FITS file store tiled CCDs
tilefile = saltio.tmpfile(outpath)
tilefile += 'tile.fits'
if varframe:
tilehdu = [None] * (3 * int(nsciext / 2) + 1)
else:
tilehdu = [None] * int(nsciext / 2 + 1)
tilehdu[0] = fits.PrimaryHDU()
#tilehdu[0].header = struct[0].header
if log:
log.message('', with_stdout=verbose)
# iterate over amplifiers, stich them to produce file of CCD images
for i in range(int(nsciext / 2)):
hdu = i * 2 + 1
# amplifier = hdu%amplifiers
# if (amplifier == 0): amplifier = amplifiers
# read DATASEC keywords
datasec1 = saltkey.get('DATASEC', struct[hdu])
datasec2 = saltkey.get('DATASEC', struct[hdu + 1])
xdsec1, ydsec1 = saltstring.secsplit(datasec1)
xdsec2, ydsec2 = saltstring.secsplit(datasec2)
# read images
imdata1 = saltio.readimage(struct, hdu)
imdata2 = saltio.readimage(struct, hdu + 1)
# tile 2n amplifiers to yield n CCD images
outdata = numpy.zeros((ydsec1[1] +
abs(ysh[i +
1] /
ybin), xdsec1[1] +
xdsec2[1] +
abs(xsh[i +
1] /
xbin)), numpy.float32)
# set up the variance frame
if varframe:
vardata = outdata.copy()
vdata1 = saltio.readimage(struct, struct[hdu].header['VAREXT'])
vdata2 = saltio.readimage(struct, struct[hdu + 1].header['VAREXT'])
bpmdata = outdata.copy()
bdata1 = saltio.readimage(struct, struct[hdu].header['BPMEXT'])
bdata2 = saltio.readimage(struct, struct[hdu + 1].header['BPMEXT'])
x1 = xdsec1[0] - 1
if x1 != 0:
msg = 'The data in %s have not been trimmed prior to mosaicking.' \
% infile
log.error(msg)
if xsh[i + 1] < 0:
x1 += abs(xsh[i + 1] / xbin)
x2 = x1 + xdsec1[1]
y1 = ydsec1[0] - 1
if ysh[i + 1] < 0:
y1 += abs(ysh[i + 1] / ybin)
y2 = y1 + ydsec1[1]
outdata[y1:y2, x1:x2] =\
imdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
x1 = x2
x2 = x1 + xdsec2[1]
y1 = ydsec2[0] - 1
if ysh[i + 1] < 0:
y1 += abs(ysh[i + 1] / ybin)
y2 = y1 + ydsec2[1]
outdata[y1:y2, x1:x2] =\
imdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
# size of new image
naxis1 = str(xdsec1[1] + xdsec2[1])
naxis2 = str(ydsec1[1])
# add image and keywords to HDU list
tilehdu[i + 1] = fits.ImageHDU(outdata)
tilehdu[i + 1].header = struct[hdu].header
#tilehdu[
# i + 1].header['DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
if varframe:
vext = i + 1 + int(nsciext / 2.)
tilehdu[vext] = fits.ImageHDU(vardata)
#tilehdu[vext].header = struct[struct[hdu].header['VAREXT']].header
#tilehdu[vext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
bext = i + 1 + 2 * int(nsciext / 2.)
tilehdu[bext] = fits.ImageHDU(bpmdata)
#tilehdu[bext].header = struct[struct[hdu].header['BPMEXT']].header
#tilehdu[bext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
# image tile log message #1
if log:
message = os.path.basename(infile) + '[' + str(hdu) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
message = os.path.basename(infile) + '[' + str(hdu + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[1] + 1) + ':' + \
str(xdsec1[1] + xdsec2[1]) + ','
message += str(ydsec2[0]) + ':' + str(ydsec2[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
# write temporary file of tiled CCDs
hdulist = fits.HDUList(tilehdu)
hdulist.writeto(tilefile)
# iterate over CCDs, transform and rotate images
yrot = [None] * 4
xrot = [None] * 4
tranfile = [' ']
tranhdu = [0]
if varframe:
tranfile = [''] * (3 * int(nsciext / 2) + 1)
tranhdu = [0] * (3 * int(nsciext / 2) + 1)
else:
tranfile = [''] * int(nsciext / 2 + 1)
tranhdu = [0] * int(nsciext / 2 + 1)
# this is hardwired for SALT where the second CCD is considered the
# fiducial
for hdu in range(1, int(nsciext / 2 + 1)):
tranfile[hdu] = saltio.tmpfile(outpath)
tranfile[hdu] += 'tran.fits'
if varframe:
tranfile[hdu + nccds] = saltio.tmpfile(outpath) + 'tran.fits'
tranfile[hdu + 2 * nccds] = saltio.tmpfile(outpath) + 'tran.fits'
ccd = hdu % nccds
if (ccd == 0):
ccd = nccds
# correct rotation for CCD binning
yrot[ccd] = rot[ccd] * ybin / xbin
xrot[ccd] = rot[ccd] * xbin / ybin
dxshift = xbin * int(float(int(gap) / xbin) + 0.5) - gap
# transformation using geotran IRAF task
# if (ccd == 1):
if (ccd != 2):
if geotran:
message = '\nSALTMOSAIC -- geotran ' + tilefile + \
'[' + str(ccd) + '] ' + tranfile[hdu]
message += ' \"\" \"\" xshift=' + \
str((xsh[ccd] + (2 - ccd) * dxshift) / xbin) + ' '
message += 'yshift=' + \
str(ysh[ccd] / ybin) + ' xrotation=' + str(xrot[ccd]) + ' '
message += 'yrotation=' + \
str(yrot[ccd]) + ' xmag=1 ymag=1 xmin=\'INDEF\''
message += 'xmax=\'INDEF\' ymin=\'INDEF\' ymax=\'INDEF\' '
message += 'ncols=\'INDEF\' '
message += 'nlines=\'INDEF\' verbose=\'no\' '
message += 'fluxconserve=\'yes\' nxblock=2048 '
message += 'nyblock=2048 interpolant=\'' + \
interp_type + '\' boundary=\'constant\' constant=0'
log.message(message, with_stdout=verbose)
yd, xd = tilehdu[ccd].data.shape
ncols = 'INDEF' # ncols=xd+abs(xsh[ccd]/xbin)
nlines = 'INDEF' # nlines=yd+abs(ysh[ccd]/ybin)
geo_xshift = xsh[ccd] + (2 - ccd) * dxshift / xbin
geo_yshift = ysh[ccd] / ybin
iraf.images.immatch.geotran(tilefile + "[" + str(ccd) + "]",
tranfile[hdu],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1, ymag=1, xmin='INDEF',
xmax='INDEF', ymin='INDEF',
ymax='INDEF', ncols=ncols,
nlines=nlines, verbose='no',
fluxconserve='yes', nxblock=2048,
nyblock=2048, interpolant="linear",
boundary="constant", constant=0)
if varframe:
var_infile = tilefile + "[" + str(ccd + nccds) + "]"
iraf.images.immatch.geotran(var_infile,
tranfile[hdu + nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1, ymag=1, xmin='INDEF',
xmax='INDEF', ymin='INDEF',
ymax='INDEF', ncols=ncols,
nlines=nlines, verbose='no',
fluxconserve='yes',
nxblock=2048, nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
var2_infile = tilefile + "[" + str(ccd + 2 * nccds) + "]"
iraf.images.immatch.geotran(var2_infile,
tranfile[hdu + 2 * nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1, ymag=1, xmin='INDEF',
xmax='INDEF', ymin='INDEF',
ymax='INDEF', ncols=ncols,
nlines=nlines, verbose='no',
fluxconserve='yes',
nxblock=2048, nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
# open the file and copy the data to tranhdu
tstruct = fits.open(tranfile[hdu])
tranhdu[hdu] = tstruct[0].data
tstruct.close()
if varframe:
tranhdu[
hdu +
nccds] = fits.open(
tranfile[
hdu +
nccds])[0].data
tranhdu[
hdu +
2 *
nccds] = fits.open(
tranfile[
hdu +
2 *
nccds])[0].data
else:
log.message(
"Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd,
xsh[ccd] /
2.0,
ysh[ccd] /
2.0,
xrot[ccd]),
with_stdout=verbose,
with_header=False)
tranhdu[hdu] = geometric_transform(
tilehdu[ccd].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(
xsh[ccd] / 2,
ysh[ccd] / 2,
1,
1,
xrot[ccd],
yrot[ccd]))
tstruct = fits.PrimaryHDU(tranhdu[hdu])
tstruct.writeto(tranfile[hdu])
if varframe:
tranhdu[hdu + nccds] = geometric_transform(
tilehdu[hdu + 3].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(
xsh[ccd] / 2, ysh[ccd] / 2,
1, 1,
xrot[ccd], yrot[ccd]))
tranhdu[hdu + 2 * nccds] = geometric_transform(
tilehdu[hdu + 6].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(
xsh[ccd] / 2, ysh[ccd] / 2,
1, 1,
xrot[ccd], yrot[ccd]))
else:
log.message(
"Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd, 0, 0, 0), with_stdout=verbose, with_header=False)
tranhdu[hdu] = tilehdu[ccd].data
if varframe:
tranhdu[hdu + nccds] = tilehdu[ccd + nccds].data
tranhdu[hdu + 2 * nccds] = tilehdu[ccd + 2 * nccds].data
# open outfile
if varframe:
outlist = 4 * [None]
else:
outlist = 2 * [None]
#outlist[0] = struct[0].copy()
outlist[0] = fits.PrimaryHDU()
outlist[0].header = struct[0].header
naxis1 = int(gap / xbin * (nccds - 1))
naxis2 = 0
for i in range(1, nccds + 1):
yw, xw = tranhdu[i].shape
naxis1 += xw + int(abs(xsh[ccd] / xbin)) + 1
naxis2 = max(naxis2, yw)
outdata = numpy.zeros((naxis2, naxis1), numpy.float32)
outdata.shape = naxis2, naxis1
if varframe:
vardata = outdata * 0
bpmdata = outdata * 0 + 1
# iterate over CCDs, stich them to produce a full image
hdu = 0
totxshift = 0
for hdu in range(1, nccds + 1):
# read DATASEC keywords
ydsec, xdsec = tranhdu[hdu].shape
# define size and shape of final image
# tile CCDs to yield mosaiced image
x1 = int((hdu - 1) * (xdsec + gap / xbin)) + int(totxshift)
x2 = xdsec + x1
y1 = int(0)
y2 = int(ydsec)
outdata[y1:y2, x1:x2] = tranhdu[hdu]
totxshift += int(abs(xsh[hdu] / xbin)) + 1
if varframe:
vardata[y1:y2, x1:x2] = tranhdu[hdu + nccds]
bpmdata[y1:y2, x1:x2] = tranhdu[hdu + 2 * nccds]
# make sure to cover up all the gaps include bad areas
if varframe:
baddata = (outdata == 0)
baddata = nd.maximum_filter(baddata, size=3)
bpmdata[baddata] = 1
# fill in the gaps if requested
if fill:
if varframe:
outdata = fill_gaps(outdata, 0)
else:
outdata = fill_gaps(outdata, 0)
# add to the file
outlist[1] = fits.ImageHDU(outdata)
if varframe:
outlist[2] = fits.ImageHDU(vardata,name='VAR')
outlist[3] = fits.ImageHDU(bpmdata,name='BPM')
# create the image structure
outstruct = fits.HDUList(outlist)
# update the head informaation
# housekeeping keywords
saltkey.put('NEXTEND', 2, outstruct[0])
saltkey.new('EXTNAME', 'SCI', 'Extension name', outstruct[1])
saltkey.new('EXTVER', 1, 'Extension number', outstruct[1])
if varframe:
saltkey.new('VAREXT', 2, 'Variance frame extension', outstruct[1])
saltkey.new('BPMEXT', 3, 'BPM Extension', outstruct[1])
try:
saltkey.copy(struct[1], outstruct[1], 'CCDSUM')
except:
pass
# Add keywords associated with geometry
saltkey.new('SGEOMGAP', gap, 'SALT Chip Gap', outstruct[0])
c1str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[0],
yshift[0],
rotation[0])
saltkey.new('SGEOM1', c1str, 'SALT Chip 1 Transform', outstruct[0])
c2str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[1],
yshift[1],
rotation[1])
saltkey.new('SGEOM2', c2str, 'SALT Chip 2 Transform', outstruct[0])
# WCS keywords
saltkey.new('CRPIX1', 0, 'WCS: X reference pixel', outstruct[1])
saltkey.new('CRPIX2', 0, 'WCS: Y reference pixel', outstruct[1])
saltkey.new(
'CRVAL1',
float(xbin),
'WCS: X reference coordinate value',
outstruct[1])
saltkey.new(
'CRVAL2',
float(ybin),
'WCS: Y reference coordinate value',
outstruct[1])
saltkey.new('CDELT1', float(xbin), 'WCS: X pixel size', outstruct[1])
saltkey.new('CDELT2', float(ybin), 'WCS: Y pixel size', outstruct[1])
saltkey.new('CTYPE1', 'pixel', 'X type', outstruct[1])
saltkey.new('CTYPE2', 'pixel', 'Y type', outstruct[1])
# cleanup temporary files
if cleanup:
for tfile in tranfile:
if os.path.isfile(tfile):
saltio.delete(tfile)
if os.path.isfile(tilefile):
status = saltio.delete(tilefile)
# return the file
return outstruct
struct[vhdu].data = vdata * gain * (
1 + 2 * gain1 * 1e-6 * data)
except Exception, e:
msg = 'Cannot update the variance frame in %s[%i] because %s' % (
infile, vhdu, e)
raise SaltError(msg)
else:
gainmult = gain
#update the headers
if usedb:
saltkey.put('GAIN', gain, struct[hdu])
saltkey.put('RDNOISE', rdnoise, struct[hdu])
#add a keyword indicating what action was taken
saltkey.new('GAINMULT', gainmult, 'Gain multiplication', struct[hdu])
#if logging is true, then print out the following information
if log:
message = '%25s[%1d] %6s %5s %2s %6.2f %5.2f' \
% (infile,hdu,gainset,rospeed,amp, gain, rdnoise)
log.message(message, with_header=False, with_stdout=verbose)
#just to make it look pretty
if log:
log.message('', with_header=False, with_stdout=verbose)
return struct
def get_values(dblist, gainset, rospeed, amp):
def saltclean(images,
outpath,
obslogfile=None,
gaindb=None,
xtalkfile=None,
geomfile=None,
subover=True,
trim=True,
masbias=None,
subbias=False,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover = False
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist = saltio.readgaindb(gaindb)
else:
dblist = []
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict = None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg = 'The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist = filename[ccdtype == 'ZERO']
masterbias_dict = {}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = fits.open(img)
bimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(blist, bkeys)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist = filename[ccdtype == 'FLAT']
masterflat_dict = {}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct = fits.open(img)
fimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict = compareimages(struct,
fimg,
masterflat_dict,
keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys = masterflat_dict[i][0]
flist = masterflat_dict[i][1:]
mflatname = outpath + createmasterflatname(flist, fkeys)
ffiles = ','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg = os.path.basename(img)
#print nimg, nimg in flatlist, nimg in biaslist
if not (nimg in biaslist):
#open the image
struct = fits.open(img)
simg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg = outpath + 'mbxgp' + os.path.basename(img)
saltmosaic(images=simg,
outimages=mimg,
outpref='',
geomfile=geomfile,
interp=interp,
cleanup=True,
clobber=clobber,
logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
else:
outhdu = hducombine(hdu_list,
outhdu,
ext=i,
method=method,
datasec=None,
reject=reject,
mask=mask,
weight=weight,
scale=scale,
statsec=statsec,
lthresh=lthresh,
hthresh=hthresh)
#Add any header frames
saltkey.new('NCOMBINE', nimages, 'Number of images combines', outhdu[0])
return outhdu
def hducombine(hdu_list,
outhdu,
ext,
method='average',
datasec=None,
reject=None,
mask=False,
weight=False,
scale=None,
statsec=None,
blank=0,
def slot(struct,infile,dbspeed,dbrate,dbgain,dbnoise,dbbias,dbamp,xcoeff,gaindb,xtalkfile,
logfile,verbose):
import saltprint, saltkey, saltio, saltstat, time
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot,status = saltkey.instrumid(struct,infile,logfile)
# number of image HDU
nextend = 0
while (status == 0):
try:
struct[nextend+1].header['XTENSION']
nextend += 1
except:
break
nccds,status = saltkey.get('NCCDS',struct[0],infile,logfile)
amplifiers = nccds * 2
if (nextend%(amplifiers) != 0):
message = '\nERROR -- SALTSLOT: Number of image extensions and'
message += 'number of amplifiers are not consistent'
status = saltprint.err(saltlog,message)
status = saltkey.new('NSCIEXT',nextend,'Number of science extensions',struct[0],infile,logfile)
status = saltkey.new('NEXTEND',nextend,'Number of data extensions',struct[0],infile,logfile)
# check image file and gain database are compatible
if (status == 0):
ngains = len(dbgain)
if (int(max(dbamp)) != amplifiers):
message = '\nERROR -- SALTGSLOT: ' + infile + ' contains ' + str(amplifiers) + ' amplifiers'
message += ', the gaindb file ' + gaindb + ' contains ' + str(max(dbamp)) + ' amplifiers'
status = saltprint.err(logfile,message)
# check image file and cross talk database are compatible
if (status == 0):
if (len(xcoeff)-1 != amplifiers):
message = '\nERROR -- SALTSLOT: ' + infile + ' contains ' + str(amplifiers) + ' amplifiers'
message += ', the cross talk file ' + xtalkfile + ' contains ' + str(len(xcoeff)-1) + ' amplifiers'
status = saltprint.err(logfile,message)
# housekeeping keywords
if (status == 0):
status = saltkey.put('SAL-TLM',time.asctime(time.localtime()),struct[0],infile,logfile)
status = saltkey.new(keyslot,time.asctime(time.localtime()),
'Data have been cleaned by SALTSLOT',struct[0],infile,logfile)
# keywords for image extensions
for i in range(nextend):
hdu = i + 1
status = saltkey.new('EXTNAME','SCI','Extension name',struct[hdu],infile,logfile)
status = saltkey.new('EXTVER',hdu,'Extension number',struct[hdu],infile,logfile)
# log coefficent table
if (status == 0):
message = '%30s %5s %4s %8s' % ('HDU','Gain','Bias','Xtalk')
saltprint.log(logfile,'\n ---------------------------------------------',verbose)
saltprint.log(logfile,message,verbose)
saltprint.log(logfile,' ---------------------------------------------',verbose)
# loop over image extensions
if (status == 0):
for i in range(nextend/2):
hdu = i * 2 + 1
amplifier = hdu%amplifiers
if (amplifier == 0): amplifier = amplifiers
if (status == 0):
value,status = saltkey.get('NAXIS1',struct[hdu],infile,logfile)
naxis1 = int(value)
if (status == 0):
value,status = saltkey.get('NAXIS1',struct[hdu+1],infile,logfile)
naxis2 = int(value)
if (status == 0 and hdu == 1): biassec, status = saltkey.get('BIASSEC',struct[hdu],infile,logfile)
if (status == 0 and hdu == 1):
ranges = biassec.lstrip('[').rstrip(']').split(',')
x1_1 = int(ranges[0].split(':')[0]) - 1
x2_1 = int(ranges[0].split(':')[1]) - 1
y1_1 = int(ranges[1].split(':')[0]) - 1
y2_1 = int(ranges[1].split(':')[1]) - 1
if (status == 0 and hdu == 1): biassec, status = saltkey.get('BIASSEC',struct[hdu+1],infile,logfile)
if (status == 0 and hdu == 1):
ranges = biassec.lstrip('[').rstrip(']').split(',')
x1_2 = int(ranges[0].split(':')[0]) - 1
x2_2 = int(ranges[0].split(':')[1]) - 1
y1_2 = int(ranges[1].split(':')[0]) - 1
y2_2 = int(ranges[1].split(':')[1]) - 1
if (status == 0 and hdu == 1): datasec,status = saltkey.get('DATASEC',struct[hdu],infile,logfile)
if (status == 0 and hdu == 1):
ranges = datasec.lstrip('[').rstrip(']').split(',')
dx1_1 = int(ranges[0].split(':')[0]) - 1
dx2_1 = int(ranges[0].split(':')[1])
dy1_1 = int(ranges[1].split(':')[0]) - 1
dy2_1 = int(ranges[1].split(':')[1])
if (status == 0 and hdu == 1): datasec,status = saltkey.get('DATASEC',struct[hdu+1],infile,logfile)
if (status == 0 and hdu == 1):
ranges = datasec.lstrip('[').rstrip(']').split(',')
dx1_2 = int(ranges[0].split(':')[0]) - 1
dx2_2 = int(ranges[0].split(':')[1])
dy1_2 = int(ranges[1].split(':')[0]) - 1
dy2_2 = int(ranges[1].split(':')[1])
if (status == 0 and dx2_1 - dx1_1 != dx2_2 - dx1_2):
message = 'ERROR -- SALTSLOT: HDUs '+infile
message += '['+str(hdu)+'] and '+infile+'['+str(hdu+1)+']'
message += ' have different dimensions'
status = saltprint.err(logfile,message)
# read speed and gain of each exposure
if (status == 0 and hdu == 1):
gainset,status = saltkey.get('GAINSET',struct[0],infile,logfile)
rospeed,status = saltkey.get('ROSPEED',struct[0],infile,logfile)
if (rospeed == 'NONE'):
saltprint.log(logfile," ",verbose)
message = "ERROR -- SALTSLOT: Readout speed is 'NONE' in "
message += "primary keywords of " + infile
status = saltprint.err(logfile,message)
# read raw images
if (status == 0):
imagedata1,status = saltio.readimage(struct,hdu,logfile)
imagedata2,status = saltio.readimage(struct,hdu+1,logfile)
# gain correction
if (status == 0):
for j in range(len(dbgain)):
if (gainset == dbrate[j] and rospeed == dbspeed[j] and amplifier == int(dbamp[j])):
try:
gain1 = float(dbgain[j])
imagedata1 *= gain1
except:
mesage = 'ERROR -- SALTSLOT: Cannot perform gain correction on image '
message += infile+'['+str(hdu)+']'
status = saltprint.err(logfile,message)
elif (gainset == dbrate[j] and rospeed == dbspeed[j] and amplifier + 1 == int(dbamp[j])):
try:
gain2 = float(dbgain[j])
imagedata2 *= gain2
except:
mesage = 'ERROR -- SALTSLOT: Cannot perform gain correction on image '
message += infile+'['+str(hdu+1)+']'
status = saltprint.err(logfile,message)
# crosstalk correction
if (status == 0):
revimage1 = imagedata1 * float(xcoeff[amplifier])
revimage2 = imagedata2 * float(xcoeff[amplifier+1])
for j in range(dx2_1-dx1_1+1):
imagedata1[:,j] -= revimage2[:,dx2_2-j-1]
imagedata2[:,j] -= revimage1[:,dx2_1-j-1]
# bias subtraction
if (status == 0):
overx_val_1 = []
overx_val_2 = []
for x in range(x1_1,x2_1+1):
list_1 = imagedata1[y1_1:y2_1,x] * 1.0
overx_val_1.append(saltstat.median(list_1,logfile))
overlevel_1 = saltstat.median(overx_val_1,logfile)
for x in range(x1_2,x2_2+1):
list_2 = imagedata2[y1_2:y2_2,x] * 1.0
overx_val_2.append(saltstat.median(list_2,logfile))
overlevel_2 = saltstat.median(overx_val_2,logfile)
imagedata1 -= overlevel_1
imagedata2 -= overlevel_2
# trim overscan
if (status == 0):
imagedata1 = imagedata1[dy1_1:dy2_1,dx1_1:dx2_1]
imagedata2 = imagedata2[dy1_2:dy2_2,dx1_2:dx2_2]
datasec = '[1:'+str(dx2_1-dx1_1)+',1:'+str(dy2_1-dy1_1)+']'
status = saltkey.put('DATASEC',datasec,struct[hdu],infile,logfile)
status = saltkey.rem('BIASSEC',struct[hdu],infile,logfile)
datasec = '[1:'+str(dx2_2-dx1_2)+',1:'+str(dy2_2-dy1_2)+']'
status = saltkey.put('DATASEC',datasec,struct[hdu+1],infile,logfile)
status = saltkey.rem('BIASSEC',struct[hdu+1],infile,logfile)
# log coefficient table
if (status == 0):
infilename = infile.split('/')
infilename = infilename[len(infilename)-1]
message = '%25s[%3d] %5.2f %4d %8.6f' % \
(infilename, hdu, gain1, overlevel_1, float(xcoeff[amplifier+1]))
saltprint.log(logfile,message,verbose)
message = '%25s[%3d] %5.2f %4d %8.6f' % \
(infilename, hdu+1, gain2, overlevel_2,float(xcoeff[amplifier]))
saltprint.log(logfile,message,verbose)
# update image in HDU structure
if (status == 0):
struct,status = saltio.writeimage(struct,hdu,imagedata1,logfile)
struct,status = saltio.writeimage(struct,hdu+1,imagedata2,logfile)
return struct, status
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist=filename[ccdtype=='ZERO']
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=pyfits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in flatlist or not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def bias(struct,subover=True,trim=True, subbias=False, bstruct=None,
median=False, function='polynomial',order=3,rej_lo=3,rej_hi=3,niter=10,
plotover=False, log=None, verbose=True):
"""Bias subtracts the bias levels from a frame. It will fit and subtract the overscan
region, trim the images, and subtract a master bias if required.
struct--image structure
subover--subtract the overscan region
trim--trim the image
subbias--subtract master bias
bstruct--master bias image structure
median--use the median instead of mean in image statistics
function--form to fit to the overscan region
order--order for the function
rej_lo--sigma of low points to reject in the fit
rej_hi--sigma of high points to reject in the fit
niter--number of iterations
log--saltio log for recording information
verbose--whether to print to stdout
"""
infile=saltkey.getimagename(struct[0])
# how many extensions?
nsciext = saltkey.get('NSCIEXT',struct[0])
nextend = saltkey.get('NEXTEND',struct[0])
nccd = saltkey.get('NCCDS',struct[0])
# how many amplifiers?--this is hard wired
amplifiers = 2 * nccd
#log the process
if subover and log:
message = '%28s %7s %5s %4s %6s' % \
('HDU','Overscan','Order','RMS','Niter')
log.message('\n --------------------------------------------------',
with_header=False, with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' --------------------------------------------------',
with_header=False, with_stdout=verbose)
if (plotover):
plt.figure(1)
plt.axes([0.1,0.1,0.8,0.8])
plt.xlabel('CCD Column')
plt.ylabel('Pixel Counts (e-)')
plt.ion()
#loop through the extensions and subtract the bias
for i in range(1,nsciext+1):
if struct[i].name=='SCI':
#get the bias section
biassec = saltkey.get('BIASSEC',struct[i])
y1,y2,x1,x2 = saltio.getSection(biassec, iraf_format=True)
#get the data section
datasec = saltkey.get('DATASEC',struct[i])
dy1,dy2, dx1, dx2 = saltio.getSection(datasec, iraf_format=True)
#setup the overscan region
if subover:
yarr=np.arange(y1,y2, dtype=float)
data=struct[i].data
odata=struct[i].data[y1:y2,x1:x2]
if median:
odata=np.median((struct[i].data[y1:y2,x1:x2]),axis=1)
olevel=np.median((struct[i].data[y1:y2,x1:x2]))
saltkey.new('OVERSCAN','%f' % (olevel),'Overscan median value', struct[i])
else:
odata=np.mean((struct[i].data[y1:y2,x1:x2]),axis=1)
olevel=np.mean((struct[i].data[y1:y2,x1:x2]))
saltkey.new('OVERSCAN','%f' % (olevel),'Overscan mean value', struct[i])
#fit the overscan region
ifit=saltfit.interfit(yarr, odata, function=function, \
order=order, thresh=rej_hi, niter=niter)
try:
ifit.interfit()
coeffs=ifit.coef
ofit=ifit(yarr)
omean, omed, osigma=saltstat.iterstat((odata-ofit), sig=3, niter=5)
except ValueError:
#catch the error if it is a zero array
ofit=np.array(yarr)*0.0
osigma=0.0
except TypeError:
#catch the error if it is a zero array
ofit=np.array(yarr)*0.0
osigma=0.0
#if it hasn't been already, convert image to
#double format
struct[i].data = 1.0 * struct[i].data
try:
struct[i].header.remove('BZERO')
struct[i].header.remove('BSCALE')
except:
pass
#subtract the overscan region
for j in range(len(struct[i].data[0])):
struct[i].data[y1:y2,j] -= ofit
#report the information
if log:
message = '%25s[%1d] %8.2f %3d %7.2f %3d' % \
(infile, i, olevel, order, osigma, niter)
log.message(message, with_stdout=verbose, with_header=False)
#add the statistics to the image header
saltkey.new('OVERRMS','%f' % (osigma),'Overscan RMS value', struct[i])
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu=saltkey.get('VAREXT', struct[i])
try:
vdata=struct[vhdu].data
#The bias level should not be included in the noise from the signal
for j in range(len(struct[i].data[0])):
vdata[y1:y2,j] -= ofit
#add a bit to make sure that the minimum error is the rednoise
rdnoise= saltkey.get('RDNOISE',struct[i])
vdata[vdata<rdnoise**2]=rdnoise**2
struct[vhdu].data=vdata+osigma**2
except Exception, e:
msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e)
raise SaltError(msg)
#plot the overscan region
if plotover:
plt.plot(yarr, odata)
plt.plot(yarr, ofit)
#trim the data and update the headers
if trim:
struct[i].data=struct[i].data[dy1:dy2,dx1:dx2]
datasec = '[1:'+str(dx2-dx1)+',1:'+str(dy2-dy1)+']'
saltkey.put('DATASEC',datasec,struct[i])
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu=saltkey.get('VAREXT', struct[i])
struct[vhdu].data=struct[vhdu].data[dy1:dy2,dx1:dx2]
datasec = '[1:'+str(dx2-dx1)+',1:'+str(dy2-dy1)+']'
saltkey.put('DATASEC',datasec,struct[vhdu])
#update the BPM frame
if saltkey.found('BPMEXT', struct[i]):
bhdu=saltkey.get('BPMEXT', struct[i])
struct[bhdu].data=struct[bhdu].data[dy1:dy2,dx1:dx2]
datasec = '[1:'+str(dx2-dx1)+',1:'+str(dy2-dy1)+']'
saltkey.put('DATASEC',datasec,struct[bhdu])
#subtract the master bias if necessary
if subbias and bstruct:
struct[i].data -= bstruct[i].data
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu=saltkey.get('VAREXT', struct[i])
try:
vdata=struct[vhdu].data
struct[vhdu].data=vdata+bstruct[vhdu].data
except Exception, e:
msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e)
raise SaltError(msg)
def prepare(hdu):
"""Prepare HRS data to be similar to other SALT file formats
This includes splitting each amplifier into multi-extension
formats
"""
#set the detector
detname=saltkey.get('DETNAM', hdu[0])
if detname=='08443-03-01' or detname=='HRDET':
detector='hrdet'
elif detname=='04434-23-02' or detname=='HBDET':
detector='hbdet'
else:
raise SaltError('%s is not an HRS detector' % detnam)
#get key parameters
try:
nccd=saltkey.get('CCDNAMPS', hdu[0])
except:
nccd=saltkey.get('CCDAMPS', hdu[0])
xbin, ybin=saltkey.ccdbin(hdu[0])
gain=saltkey.get('GAIN', hdu[0])
gain=gain.split()
rospeed=saltkey.get('ROSPEED', hdu[0])
ccdshape=hdu[0].data.shape
#create a multiexention fits file
phdu=pyfits.PrimaryHDU()
phdu.header=hdu[0].header
nhdu = pyfits.HDUList(phdu)
#these keywords need to be added
saltkey.new('DETMODE', value='Normal', comment='Detector mode', hdu=nhdu[0])
saltkey.new('NCCDS', value=1, comment='Detector mode', hdu=nhdu[0])
saltkey.new('GAINSET', value='SLOW', comment='Detector mode', hdu=nhdu[0])
value='OBJECT'
if hdu[0].header['OBJECT']=='Bias': value='ZERO'
if hdu[0].header['OBJECT']=='Arc': value='ARC'
if hdu[0].header['OBJECT']=='Flat': value='FLAT'
saltkey.new('CCDTYPE', value=value, comment='CCD Type', hdu=nhdu[0])
if nccd==1:
nhdu.append(pyfits.ImageHDU(hdu[0].data))
j=1
saltkey.new('GAIN', value=float(gain[0]), comment='Nominal CCD gain (e/ADU)', hdu=nhdu[j])
saltkey.new('RDNOISE', value=0, comment='Nominal readout noise in e', hdu=nhdu[j])
saltkey.new('SATURATE', value=1, comment='Pixel saturation level in ADU', hdu=nhdu[j])
saltkey.new('XTALK', value=0.0, comment='Cross talk coefficient', hdu=nhdu[j])
detsize=getdetsize(ccdshape, xbin, ybin)
ampshape=nhdu[j].data.shape
ampsec=getdetsize(ccdshape, 1, 1)
saltkey.new('DETSIZE', value=detsize, comment='Detector size', hdu=nhdu[j])
saltkey.new('BIASSEC', value=getbiassec(j, ampshape, xbin, ybin), comment='Bias section', hdu=nhdu[j])
saltkey.new('DATASEC', value=getdatasec(j, ampshape, xbin, ybin), comment='Data section', hdu=nhdu[j])
saltkey.new('AMPSEC', value=ampsec, comment='Amplifier section', hdu=nhdu[j])
saltkey.new('CCDSEC', value=detsize, comment='CCD section', hdu=nhdu[j])
saltkey.new('DETSEC', value=detsize, comment='Detector section', hdu=nhdu[j])
return nhdu
for i in range(nccd):
j=i+1
x1,x2,y1,y2=definesection(j, detector, ccdshape)
nhdu.append(pyfits.ImageHDU(hdu[0].data[y1:y2,x1:x2]))
#keywords that need to be added include
#gain, rdnoise, xtalk, saturate,
saltkey.new('GAIN', value=float(gain[i]), comment='Nominal CCD gain (e/ADU)', hdu=nhdu[j])
saltkey.new('RDNOISE', value=0, comment='Nominal readout noise in e', hdu=nhdu[j])
saltkey.new('SATURATE', value=1, comment='Pixel saturation level in ADU', hdu=nhdu[j])
saltkey.new('XTALK', value=0.0, comment='Cross talk coefficient', hdu=nhdu[j])
#DETSIZE, BIASSEC, DATASEC,
#AMPSEC, CCDSEC, DETSEC
ampsec='[%i:%i,%i:%i]' % (x1+1,x2,y1+1,y2)
ampshape=nhdu[j].data.shape
saltkey.new('DETSIZE', value=getdetsize(ccdshape, xbin, ybin), comment='Detector size', hdu=nhdu[j])
saltkey.new('BIASSEC', value=getbiassec(j, ampshape, xbin, ybin), comment='Bias section', hdu=nhdu[j])
saltkey.new('DATASEC', value=getdatasec(j, ampshape, xbin, ybin), comment='Data section', hdu=nhdu[j])
saltkey.new('AMPSEC', value=ampsec, comment='Amplifier section', hdu=nhdu[j])
saltkey.new('CCDSEC', value=getdetsize(ccdshape, xbin, ybin), comment='CCD section', hdu=nhdu[j])
saltkey.new('DETSEC', value=getdetsize(ccdshape, xbin, ybin), comment='Detector section', hdu=nhdu[j])
#BSCALE, BZERO
#nhdu[j].header.set('BSCALE', value=1.0, comment='Val=BSCALE*pix+BZERO')
#nhdu[j].header.set('BZERO', value=32768., comment='Val=BSCALE*pix+BZERO')
#ATM1_1, ATM2_2,
#not implimented
return nhdu
