def saltprepare(images,outimages,outpref,createvar=False, badpixelimage=None, clobber=True,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# open the badpixel image
if saltio.checkfornone(badpixelimage) is None:
badpixelstruct=None
else:
try:
badpixelstruct = saltio.openfits(badpixelimage)
except saltio.SaltIOError,e:
msg='badpixel image must be specificied\n %s' % e
raise SaltError(msg)
# open each raw image file
for img, oimg, in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
#if createvar, throw a warning if it is using slotmode
if saltkey.fastmode(saltkey.get('DETMODE', struct[0])) and createvar:
msg='Creating variance frames for slotmode data in %s' % img
log.warning(msg)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)
# has file been prepared already?
try:
key = struct[0].header[keyprep]
message = 'ERROR -- SALTPREPARE: File ' + infile
message += ' has already been prepared'
raise SaltError(message)
except:
pass
# prepare file
struct=prepare(struct,createvar=createvar, badpixelstruct=badpixelstruct)
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],keyprep, 'File prepared for IRAF', hist)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
message = 'SALTPREPARE -- %s => %s' % (img, oimg)
log.message(message, with_header=False)
def saltillum(images, outimages, outpref, mbox=11, clobber=False, logfile="salt.log", verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack("Input", images)
# create list of output files
outfiles = saltio.listparse("Outfile", outimages, outpref, infiles, "")
# verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, "Input", "output")
# open each raw image file
for infile, outfile in zip(infiles, outfiles):
struct = saltio.openfits(infile)
struct = illum_cor(struct, mbox)
# add any header keywords like history
fname, hist = history(level=1, wrap=False)
saltkey.housekeeping(struct[0], "SILLUM", "File Illumination corrected", hist)
# write it out and close it
saltio.writefits(struct, outfile, clobber=clobber)
saltio.closefits(struct)
# output the information
log.message("Illumination corrected image %s " % (infile), with_header=False, with_stdout=verbose)
def saltsurface(images,outimages,outpref, mask=True, order=3, minlevel=0,
clobber=False, logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# open each raw image file
for infile, outfile in zip(infiles,outfiles):
struct = saltio.openfits(infile)
struct = surface_fit(struct, order=order, mask=mask, minlevel=minlevel)
#add any header keywords like history
fname, hist=history(level=1, wrap=False)
saltkey.housekeeping(struct[0],'SURFIT', 'File fit by a surface', hist)
#write it out and close it
saltio.writefits(struct,outfile,clobber=clobber)
saltio.closefits(struct)
#output the information
log.message('Surface fitted image %s ' % (infile), with_header=False, with_stdout=verbose)
def saltillum(images,outimages,outpref, mbox=11, clobber=False, \
logfile='salt.log',verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
#verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# open each raw image file
for infile, outfile in zip(infiles, outfiles):
struct = saltio.openfits(infile)
struct = illum_cor(struct, mbox)
#add any header keywords like history
fname, hist = history(level=1, wrap=False)
saltkey.housekeeping(struct[0], 'SILLUM',
'File Illumination corrected', hist)
#write it out and close it
saltio.writefits(struct, outfile, clobber=clobber)
saltio.closefits(struct)
#output the information
log.message('Illumination corrected image %s ' % (infile),
with_header=False,
with_stdout=verbose)
def saltprepare(images,outimages,outpref,createvar=False, badpixelimage=None, clobber=True,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# open the badpixel image
if saltio.checkfornone(badpixelimage) is None:
badpixelstruct=None
else:
try:
badpixelstruct = saltio.openfits(badpixelimage)
except saltio.SaltIOError,e:
msg='badpixel image must be specificied\n %s' % e
raise SaltError(msg)
# open each raw image file
for img, oimg, in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
#if createvar, throw a warning if it is using slotmode
if saltkey.fastmode(saltkey.get('DETMODE', struct[0])) and createvar:
msg='Creating variance frames for slotmode data in %s' % img
log.warning(msg)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)
# has file been prepared already?
try:
key = struct[0].header[keyprep]
message = 'ERROR -- SALTPREPARE: File ' + infile
message += ' has already been prepared'
raise SaltError(message)
except:
pass
# prepare file
struct=prepare(struct,createvar=createvar, badpixelstruct=badpixelstruct)
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],keyprep, 'File prepared for IRAF', hist)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
message = 'SALTPREPARE -- %s => %s' % (img, oimg)
log.message(message, with_header=False)
def saltgain(images,
outimages,
outpref,
gaindb=None,
usedb=False,
mult=True,
clobber=True,
logfile='salt.log',
verbose=True):
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
#verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# read in the database file if usedb is true
if usedb:
gaindb = gaindb.strip()
dblist = saltio.readgaindb(gaindb)
else:
dblist = []
for img, oimg in zip(infiles, outfiles):
#open the fits file
struct = saltio.openfits(img)
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid(
struct)
# has file been prepared already?
if saltkey.found(keygain, struct[0]):
message = 'SALTGAIN: %s has already been gain-corrected' % img
raise SaltError(message)
# gain correct the data
struct = gain(struct,
mult=mult,
usedb=usedb,
dblist=dblist,
log=log,
verbose=verbose)
# housekeeping keywords
fname, hist = history(level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], keygain,
'Images have been gain corrected', hist)
# write FITS file
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
def obslog(infiles, log=None):
"""For a set of input files, create a dictionary contain all the header
information from the files. Will print things to a saltlog if log is
not None
returns Dictionary
"""
#create the header dictionary
headerDict = {}
for k in headerList:
headerDict[k] = []
for k in scamheaderList:
headerDict[k] = []
for k in rssheaderList:
headerDict[k] = []
# interate over and open image files
infiles.sort()
for infile in infiles:
#open the file
struct = saltio.openfits(infile)
# instrument
scam = False
rss = False
instrume = saltkey.get('INSTRUME', struct[0])
if (instrume == 'RSS'): rss = True
if (instrume == 'SALTICAM'): scam = True
#add in the image name
headerDict['FILENAME'].append(os.path.basename(infile))
# ingest primary keywords from files in the image list
for k, f in zip(headerList[1:], formatList[1:]):
default = finddefault(f)
headerDict[k].append(
getkey(struct[0], k, default=default, log=log, warn=True))
# ingest scam specific primary keywords from files in the image list
for k, f in zip(scamheaderList[1:], scamformatList[1:]):
default = finddefault(f)
headerDict[k].append(
getkey(struct[0], k, default=default, log=log, warn=scam))
# ingest rss specific primary keywords from files in the image list
for k, f in zip(rssheaderList[1:], rssformatList[1:]):
default = finddefault(f)
headerDict[k].append(
getkey(struct[0], k, default=default, log=log, warn=rss))
# close image files
saltio.closefits(struct)
if log:
log.message('SALTOBSLOG -- read %s' % infile, with_header=False)
return headerDict
def saltfpskyring(
images,
outimages,
outpref,
axc,
ayc,
arad,
rxc,
ryc,
pmin,
pmax,
swindow=5,
clobber=False,
logfile="salt.log",
verbose=True,
):
"""Sky subtracts Fabry-Perot images"""
# start log now that all parameter are set up
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack("Input", images)
# create list of output files
outfiles = saltio.listparse("Outfile", outimages, outpref, infiles, "")
# verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, "Input", "output")
for img, oimg in zip(infiles, outfiles):
# open up the file
hdu = saltio.openfits(img)
# determine the azimuthally averaged profile
rpro, radial_profile = median_radial_profile(
hdu[0].data, xc=axc, yc=ayc, rmax=arad, nbins=100, pmin=pmin, pmax=pmax
)
if swindow > 1:
radial_profile = np.convolve(radial_profile, np.ones(swindow), mode="same") / swindow
# calculate the indices from the image
y, x = np.indices(hdu[0].data.shape)
radius = np.hypot(x - axc, y - ayc)
# subtract off the sky data
mdata = hdu[0].data - np.interp(radius, rpro, radial_profile)
hdu[0].data = mdata
# write FITS file
saltio.writefits(hdu, oimg, clobber=clobber)
saltio.closefits(hdu)
message = "SALTFPSKYRING -- Subtracted sky from %s" % (img)
log.message(message, with_header=False)
def saltcrclean(images,outimages,outpref,crtype='fast',thresh=5,mbox=3, \
bthresh=3, flux_ratio=0.2, bbox=11, gain=1, rdnoise=5, fthresh=5,\
bfactor=2, gbox=3, maxiter=5, multithread=False, update=True,
clobber=True, logfile='salt.log', verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
#check to see if multithreading is available
if mp:
pass
else:
multithread=False
log.warning('multiprocessing module is not available. Setting multiththread=False')
# Begin processes each file
for infile,outfile in zip(infiles,outfiles):
#open the infile
struct=saltio.openfits(infile)
#clean the cosmic rays
if multithread and len(struct)>1:
struct=multicrclean(struct, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \
gain, rdnoise, bfactor, fthresh, gbox, maxiter, log, verbose=verbose)
else:
struct=crclean(struct, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \
gain, rdnoise, bfactor, fthresh, gbox, maxiter, update, log, verbose=verbose)
#log the call
#log.message('Cleaned %i cosmic rays from %s using %s method' % (totcr, infile, crtype), with_header=False)
log.message('', with_header=False, with_stdout=verbose)
#add house keeping keywords
saltkey.put('SAL-TLM',time.asctime(time.localtime()), struct[0])
#add the history keyword
fname, hist=history(level=1, wrap=False)
saltkey.history(struct[0],hist)
#write out the file
saltio.writefits(struct, outfile, clobber=clobber)
#close the image
saltio.closefits(struct)
def saltxtalk(images,outimages,outpref,xtalkfile=None, usedb=False,
clobber=True, logfile='salt.log',verbose=True):
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
# are input and output lists the same length?
saltio.comparelists(infiles,outfiles,'Input','output')
# does crosstalk coefficient data exist
if usedb:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
for img, oimg in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
#find the best xcoeff for the image if using the db
if usedb:
obsdate=saltkey.get('DATE-OBS', struct[0])
obsdate=int('%s%s%s' % (obsdate[0:4],obsdate[5:7], obsdate[8:]))
xkey=np.array(xdict.keys())
date=xkey[abs(xkey-obsdate).argmin()]
xcoeff=xdict[date]
else:
xcoeff=[]
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)
# has file been prepared already?
if saltkey.found(keyxtalk, struct[0]):
message='%s has already been xtalk corrected' % img
raise SaltError(message)
#apply the cross-talk correction
struct = xtalk(struct, xcoeff, log=log, verbose=verbose)
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SXTALK', 'Images have been xtalk corrected', hist)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
def saltcrclean(images,outimages,outpref,crtype='fast',thresh=5,mbox=3, \
bthresh=3, flux_ratio=0.2, bbox=11, gain=1, rdnoise=5, fthresh=5,\
bfactor=2, gbox=3, maxiter=5, multithread=False, update=True,
clobber=True, logfile='salt.log', verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
#verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, 'Input', 'output')
#check to see if multithreading is available
if mp:
pass
else:
multithread = False
log.warning(
'multiprocessing module is not available. Setting multiththread=False'
)
# Begin processes each file
for infile, outfile in zip(infiles, outfiles):
#open the infile
struct = saltio.openfits(infile)
#clean the cosmic rays
if multithread and len(struct) > 1:
struct=multicrclean(struct, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \
gain, rdnoise, bfactor, fthresh, gbox, maxiter, log, verbose=verbose)
else:
struct=crclean(struct, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \
gain, rdnoise, bfactor, fthresh, gbox, maxiter, update, log, verbose=verbose)
#log the call
#log.message('Cleaned %i cosmic rays from %s using %s method' % (totcr, infile, crtype), with_header=False)
log.message('', with_header=False, with_stdout=verbose)
#add house keeping keywords
saltkey.put('SAL-TLM', time.asctime(time.localtime()), struct[0])
#add the history keyword
fname, hist = history(level=1, wrap=False)
saltkey.history(struct[0], hist)
#write out the file
saltio.writefits(struct, outfile, clobber=clobber)
#close the image
saltio.closefits(struct)
def saltslot(images,
outimages,
outpref,
gaindb='',
xtalkfile='',
usedb=False,
clobber=False,
logfile='salt.log',
verbose=True):
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
# are input and output lists the same length?
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# does crosstalk coefficient data exist
if usedb:
dblist = saltio.readgaindb(gaindb)
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
dblist = []
xdict = None
for img, oimg in zip(infiles, outfiles):
#open the fits file
struct = saltio.openfits(img)
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid(
struct)
# has file been prepared already?
if saltkey.found(keygain, struct[0]):
message = '%s has already been reduced' % img
raise SaltError(message)
# housekeeping keywords
fname, hist = history(level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], keyslot,
'Images have been slotmode reduced', hist)
# write FITS file
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
def saltarith(operand1, op, operand2, result, outpref, divzero=0, clobber=False, \
logfile='salt.log',verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', operand1)
# create list of output files
outfiles = saltio.listparse('Outfile', result, outpref, infiles, '')
#verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, 'Input', 'output')
#let's keep track of whether operand2 is an image or not
is_image = False
#load in operand2, or, if it's not an image, assume it's a number
try:
operand2struct = float(operand2)
except ValueError:
operand2struct = saltio.openfits(operand2)
is_image = True
#open the input image files
for infile, outfile in zip(infiles, outfiles):
struct = saltio.openfits(infile)
#do some math!
outstruct = arith(struct, op, operand2struct, is_image, divzero)
try:
pass
except Exception, e:
msg = 'Unable to do math %s because %s' % (infile, e)
raise SaltError(msg)
#update header stuff
fname, hist = history(level=1, wrap=False)
saltkey.housekeeping(struct[0], 'SARITH',
'Some arithmatic was performed', hist)
#write it. close it.
saltio.writefits(outstruct, outfile, clobber=clobber)
saltio.closefits(struct)
#output the information
log.message('imarith: %s %s %s %s' %
(infile, op, operand2, outfile),
with_header=False,
with_stdout=verbose)
#close the operand2 image
if is_image: saltio.closefits(operand2struct)
def obslog(infiles, log=None):
"""For a set of input files, create a dictionary contain all the header
information from the files. Will print things to a saltlog if log is
not None
returns Dictionary
"""
#create the header dictionary
headerDict={}
for k in headerList: headerDict[k]=[]
for k in scamheaderList: headerDict[k]=[]
for k in rssheaderList: headerDict[k]=[]
# interate over and open image files
infiles.sort()
for infile in infiles:
#open the file
struct = saltio.openfits(infile)
# instrument
scam = False
rss = False
instrume = saltkey.get('INSTRUME', struct[0])
if (instrume=='RSS'): rss = True
if (instrume=='SALTICAM'): scam=True
#add in the image name
headerDict['FILENAME'].append(os.path.basename(infile))
# ingest primary keywords from files in the image list
for k,f in zip(headerList[1:], formatList[1:]):
default=finddefault(f)
headerDict[k].append(getkey(struct[0], k, default=default, log=log, warn=True))
# ingest scam specific primary keywords from files in the image list
for k,f in zip(scamheaderList[1:], scamformatList[1:]):
default=finddefault(f)
headerDict[k].append(getkey(struct[0], k, default=default, log=log, warn=scam))
# ingest rss specific primary keywords from files in the image list
for k,f in zip(rssheaderList[1:], rssformatList[1:]):
default=finddefault(f)
headerDict[k].append(getkey(struct[0], k, default=default, log=log, warn=rss))
# close image files
saltio.closefits(struct)
if log: log.message('SALTOBSLOG -- read %s' % infile, with_header=False)
return headerDict
def saltarith(operand1, op, operand2, result, outpref, divzero=0, clobber=False, \
logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',operand1)
# create list of output files
outfiles=saltio.listparse('Outfile', result, outpref, infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
#let's keep track of whether operand2 is an image or not
is_image = False
#load in operand2, or, if it's not an image, assume it's a number
try: operand2struct = float(operand2)
except ValueError:
operand2struct = saltio.openfits(operand2)
is_image = True
#open the input image files
for infile, outfile in zip(infiles,outfiles):
struct = saltio.openfits(infile)
#do some math!
outstruct = arith(struct, op, operand2struct, is_image, divzero)
try:
pass
except Exception as e:
msg='Unable to do math %s because %s' % (infile, e)
raise SaltError(msg)
#update header stuff
fname, hist = history(level=1, wrap=False)
saltkey.housekeeping(struct[0],'SARITH', 'Some arithmatic was performed',hist)
#write it. close it.
saltio.writefits(outstruct,outfile,clobber=clobber)
saltio.closefits(struct)
#output the information
log.message('imarith: %s %s %s %s' % (infile, op, operand2, outfile), with_header=False, with_stdout=verbose)
#close the operand2 image
if is_image: saltio.closefits(operand2struct)
def saltmosaic(images,outimages,outpref,geomfile,interp='linear',geotran=True, cleanup=True,clobber=False,logfile=None,verbose=True):
#Start the logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# does CCD geometry definition file exist
geomfilefile = geomfile.strip()
saltio.fileexists(geomfile)
gap = 0
xshift = [0, 0]
yshift = [0, 0]
rotation = [0, 0]
gap, xshift, yshift, rotation=saltio.readccdgeom(geomfile)
# open each raw image file and apply the transformation to it
for img, oimg in zip(infiles, outfiles):
#open the structure
struct = saltio.openfits(img)
#create the mosaic
ostruct=make_mosaic(struct, gap, xshift, yshift, rotation, interp_type=interp, geotran=geotran, cleanup=cleanup, log=log, verbose=verbose)
#update the header information
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(ostruct[0], 'SMOSAIC', 'Images have been mosaicked ', hist)
#write the image out
saltio.writefits(ostruct,oimg, clobber=clobber)
#close the files
saltio.closefits(struct)
saltio.closefits(ostruct)
def saltgain(images,outimages, outpref, gaindb=None,usedb=False, mult=True,
clobber=True, logfile='salt.log',verbose=True):
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# read in the database file if usedb is true
if usedb:
gaindb = gaindb.strip()
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
for img, oimg in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)
# has file been prepared already?
if saltkey.found(keygain, struct[0]):
message='SALTGAIN: %s has already been gain-corrected' % img
raise SaltError(message)
# gain correct the data
struct = gain(struct,mult=mult, usedb=usedb, dblist=dblist, log=log, verbose=verbose)
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],keygain, 'Images have been gain corrected', hist)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
def saltslot(images,outimages,outpref,gaindb='',xtalkfile='',usedb=False, clobber=False,logfile='salt.log',verbose=True):
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
# are input and output lists the same length?
saltio.comparelists(infiles,outfiles,'Input','output')
# does crosstalk coefficient data exist
if usedb:
dblist= saltio.readgaindb(gaindb)
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
dblist=[]
xdict=None
for img, oimg in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)
# has file been prepared already?
if saltkey.found(keygain, struct[0]):
message='%s has already been reduced' % img
raise SaltError(message)
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],keyslot, 'Images have been slotmode reduced', hist)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
def saltsurface(images,
outimages,
outpref,
mask=True,
order=3,
minlevel=0,
clobber=False,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
#verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# open each raw image file
for infile, outfile in zip(infiles, outfiles):
struct = saltio.openfits(infile)
struct = surface_fit(struct,
order=order,
mask=mask,
minlevel=minlevel)
#add any header keywords like history
fname, hist = history(level=1, wrap=False)
saltkey.housekeeping(struct[0], 'SURFIT', 'File fit by a surface',
hist)
#write it out and close it
saltio.writefits(struct, outfile, clobber=clobber)
saltio.closefits(struct)
#output the information
log.message('Surface fitted image %s ' % (infile),
with_header=False,
with_stdout=verbose)
def sdbloadfits(infile, sdb, logfile, verbose):
"""Add a fits file to the science database
"""
#determine the base file name
FileName = findrawfilename(infile)
#determine the reduced file name
if FileName != infile:
PipelineFileName = infile
else:
PipelineFileName = ''
#open the image and extract the header
infits = saltio.openfits(infile)
#Extract the primary header
try:
ext = 0
ImageHeader = infits[ext].header
except:
message = 'SALTSDBLOADFITS -- ERROR: Can not access header for %s' % infile
raise SaltError(message)
#get the FileData_Id if it already exists
FileData_Id = checksdbforfits(FileName, sdb, logfile, verbose)
#create a new entry or update
if FileData_Id == -1:
#add to database and get its new id number
FileData_Id = saltmysql.createnewFileData(sdb, ImageHeader, FileName,
PipelineFileName)
else:
#update the FileData information
saltmysql.updateFileData(sdb, ImageHeader, FileData_Id, FileName,
PipelineFileName)
#Update all of the fits header tables
saltmysql.updateFitsHeaders(sdb, infits, FileData_Id)
saltio.closefits(infits)
return
def sdbloadfits(infile, sdb, logfile, verbose):
"""Add a fits file to the science database
"""
#determine the base file name
FileName = findrawfilename(infile)
#determine the reduced file name
if FileName!=infile:
PipelineFileName=infile
else:
PipelineFileName=''
#open the image and extract the header
infits=saltio.openfits(infile)
#Extract the primary header
try:
ext=0
ImageHeader = infits[ext].header
except:
message='SALTSDBLOADFITS -- ERROR: Can not access header for %s' % infile
raise SaltError(message)
#get the FileData_Id if it already exists
FileData_Id = checksdbforfits(FileName, sdb, logfile, verbose)
#create a new entry or update
if FileData_Id == -1:
#add to database and get its new id number
FileData_Id=saltmysql.createnewFileData(sdb,ImageHeader,FileName, PipelineFileName)
else:
#update the FileData information
saltmysql.updateFileData(sdb, ImageHeader, FileData_Id, FileName, PipelineFileName)
#Update all of the fits header tables
saltmysql.updateFitsHeaders(sdb, infits, FileData_Id)
saltio.closefits(infits)
return
def specprepare(images,outimages,outpref,badpixelimage='', \
clobber=True,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# open the badpixel image
if saltio.checkfornone(badpixelimage) is None:
badpixelstruct=None
else:
try:
badpixelstruct = saltio.openfits(badpixelimage)
except saltio.SaltIOError,e:
msg='badpixel image must be specificied\n %s' % e
raise SALTSpecError(msg)
# open each raw image file
for img, oimg, in zip(infiles, outfiles):
#open the fits file
struct=saltio.openfits(img)
# prepare file
struct=prepare(struct,badpixelstruct)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
message = 'SPECPREPARE -- %s => %s' % (img, oimg)
log.message(message)
def specprepare(images, outimages, outpref, badpixelimage='',
clobber=True, logfile='salt.log', verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
# verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# open the badpixel image
if saltio.checkfornone(badpixelimage) is None:
badpixelstruct = None
else:
try:
badpixelstruct = saltio.openfits(badpixelimage)
except saltio.SaltIOError as e:
msg = 'badpixel image must be specificied\n %s' % e
raise SALTSpecError(msg)
# open each raw image file
for img, oimg, in zip(infiles, outfiles):
# open the fits file
struct = saltio.openfits(img)
# prepare file
struct = prepare(struct, badpixelstruct)
# write FITS file
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
message = 'SPECPREPARE -- %s => %s' % (img, oimg)
log.message(message)
def hrsclean(images,
outpath,
obslogfile=None,
subover=True,
trim=True,
masbias=None,
subbias=True,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""Convert MEF HRS data into a single image. If variance frames and BPMs, then
convert them to the same format as well. Returns an MEF image but that is
combined into a single frame
"""
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
if saltio.checkfornone(obslogfile) is None:
raise SaltError('Obslog file is required')
# Delete the obslog file if it already exists
if (os.path.isfile(obslogfile)
and clobber) or not os.path.isfile(obslogfile):
if os.path.isfile(obslogfile): saltio.delete(obslogfile)
#read in the obsveration log or create it
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
else:
obsstruct = saltio.openfits(obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('OBJECT')
biaslist = filename[ccdtype == 'Bias']
masterbias_dict = {}
if log: log.message('Processing Bias Frames')
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = pyfits.open(img)
bimg = outpath + 'bgph' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message,
with_stdout=verbose,
with_header=False)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
subover=subover,
trim=trim,
subbias=False,
imstack=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'HPREPARE',
'Images have been prepared', hist)
saltkey.new('HGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=hrsbiasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(
blist, bkeys, x1=5, x2=13)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#apply full reductions to the science data
for img in infiles:
nimg = os.path.basename(img)
if not nimg in biaslist:
#open the image
struct = pyfits.open(img)
simg = outpath + 'mbgph' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#get master bias frame
masterbias = get_masterbias(struct,
masterbias_dict,
keylist=hrsbiasheader_list)
if masterbias:
subbias = True
bstruct = saltio.openfits(masterbias)
else:
subbias = False
bstruct = None
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
subover=subover,
trim=trim,
subbias=subbias,
imstack=True,
bstruct=bstruct,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'HPREPARE',
'Images have been prepared', hist)
saltkey.new('HGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
return
(infile, i, utc_list[j], utc_new,
utc_new_sec, t_diff[j], nd[j]))
j += 1
# Housekeeping key words
if update:
history = 'SALTUTCFIX -- '
history += 'images=' + infile + ' '
saltsafekey.housekeeping(struct[0], 'SLOTUTC',
'UTC has been corrected', history,
infile)
# update fits file
if update:
saltsafeio.updatefits(struct)
saltsafeio.closefits(struct)
# close outfile
if outfile:
fout.close()
# Keep plot window open
if plotdata:
plt.show()
def updateheaders(struct, ext, tdiff, real_expt, utc, infile):
# exit if tdiff wasn't updated
if tdiff == real_expt:
msg = 'No adequate correction found for frame %i in file %s' % (ext,
infile)
def saltfpskyring(images,
outimages,
outpref,
axc,
ayc,
arad,
rxc,
ryc,
pmin,
pmax,
swindow=5,
clobber=False,
logfile='salt.log',
verbose=True):
"""Sky subtracts Fabry-Perot images"""
# start log now that all parameter are set up
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
#verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, 'Input', 'output')
for img, oimg in zip(infiles, outfiles):
#open up the file
hdu = saltio.openfits(img)
#determine the azimuthally averaged profile
rpro, radial_profile = median_radial_profile(hdu[0].data,
xc=axc,
yc=ayc,
rmax=arad,
nbins=100,
pmin=pmin,
pmax=pmax)
if swindow > 1:
radial_profile = np.convolve(
radial_profile, np.ones(swindow), mode='same') / swindow
# calculate the indices from the image
y, x = np.indices(hdu[0].data.shape)
radius = np.hypot(x - axc, y - ayc)
#subtract off the sky data
mdata = hdu[0].data - np.interp(radius, rpro, radial_profile)
hdu[0].data = mdata
# write FITS file
saltio.writefits(hdu, oimg, clobber=clobber)
saltio.closefits(hdu)
message = 'SALTFPSKYRING -- Subtracted sky from %s' % (img)
log.message(message, with_header=False)
def slotmerge(images, outimages, outpref, geomfile, clobber, logfile, verbose):
with logging(logfile, debug) as log:
# are the arguments defined
saltsafeio.argdefined('images', images)
saltsafeio.argdefined('geomfile', geomfile)
saltsafeio.argdefined('logfile', logfile)
# if the input file is a list, does it exist?
if images[0] == '@':
saltsafeio.listexists('Input', images)
# parse list of input files
infiles = saltsafeio.listparse('Raw image', images, '', '', '')
# check input files exist
saltsafeio.filesexist(infiles, '', 'r')
# load output name list: @list, * and comma separated
outimages = outimages.strip()
outpref = outpref.strip()
if len(outpref) == 0 and len(outimages) == 0:
raise SaltIOError('Output file(s) not specified')
# test output @filelist exists
if len(outimages) > 0 and outimages[0] == '@':
saltsafeio.listexists('Output', outimages)
# parse list of output files
outfiles = saltsafeio.listparse('Output image', outimages, outpref,
infiles, '')
# are input and output lists the same length?
saltsafeio.comparelists(infiles, outfiles, 'Input', 'output')
# do the output files already exist?
if not clobber:
saltsafeio.filesexist(outfiles, '', 'w')
# does CCD geometry definition file exist
geomfilefile = geomfile.strip()
saltsafeio.fileexists(geomfile)
# read geometry definition file
gap = 0
xshift = [0, 0]
yshift = [0, 0]
rotation = [0, 0]
gap, xshift, yshift, rotation = saltsafeio.readccdgeom(geomfile)
for ro in rotation:
if ro != 0:
log.warning('SLOTMERGE currently ignores CCD rotation')
# Begin processes each file
for infile, outfile in zip(infiles, outfiles):
# determine the name for the output file
outpath = outfile.rstrip(os.path.basename(outfile))
if (len(outpath) == 0):
outpath = '.'
# open each raw image
struct = saltsafeio.openfits(infile)
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltsafekey.instrumid(
struct, infile)
# how many amplifiers?
nccds = saltsafekey.get('NCCDS', struct[0], infile)
amplifiers = nccds * 2
#if (nccds != 2):
# raise SaltError('Can not currently handle more than two CCDs')
# CCD geometry coefficients
if instrume == 'RSS' or instrume == 'PFIS':
xsh = [xshift[0], 0., xshift[1]]
ysh = [yshift[0], 0., yshift[1]]
rot = [rotation[0], 0., rotation[1]]
refid = 1
if instrume == 'SALTICAM':
xsh = [xshift[0], 0.]
ysh = [yshift[0], 0.]
rot = [rotation[0], 0]
refid = 1
# how many extensions?
nextend = saltsafekey.get('NEXTEND', struct[0], infile)
# how many exposures
exposures = nextend / amplifiers
# CCD on-chip binning
xbin, ybin = saltsafekey.ccdbin(struct[0], infile)
gp = int(gap / xbin)
# create output hdu structure
outstruct = [None] * int(exposures + 1)
outstruct[0] = struct[0]
# iterate over exposures, stitch them to produce file of CCD images
for i in range(exposures):
# Determine the total size of the image
xsize = 0
ysize = 0
for j in range(amplifiers):
hdu = i * amplifiers + j + 1
try:
xsize += len(struct[hdu].data[0])
if ysize < len(struct[hdu].data):
ysize = len(struct[hdu].data)
except:
msg = 'Unable to access extension %i ' % hdu
raise SaltIOError(msg)
xsize += gp * (nccds - 1)
maxxsh, minxsh = determineshifts(xsh)
maxysh, minysh = determineshifts(ysh)
xsize += (maxxsh - minxsh)
ysize += (maxysh - minysh)
# Determine the x and y origins for each frame
xdist = 0
ydist = 0
shid = 0
x0 = np.zeros(amplifiers)
y0 = np.zeros(amplifiers)
for j in range(amplifiers):
x0[j] = xdist + xsh[shid] - minxsh
y0[j] = ysh[shid] - minysh
hdu = i * amplifiers + j + 1
darr = struct[hdu].data
xdist += len(darr[0])
if j % 2 == 1:
xdist += gp
shid += 1
# make the out image
outarr = np.zeros((ysize, xsize), np.float64)
# Embed each frame into the output array
for j in range(amplifiers):
hdu = i * amplifiers + j + 1
darr = struct[hdu].data
outarr = salttran.embed(darr, x0[j], y0[j], outarr)
# Add the outimage to the output structure
hdu = i * amplifiers + 1
outhdu = i + 1
outstruct[outhdu] = pyfits.ImageHDU(outarr)
outstruct[outhdu].header = struct[hdu].header
# Fix the headers in each extension
datasec = '[1:%4i,1:%4i]' % (xsize, ysize)
saltsafekey.put('DATASEC', datasec, outstruct[outhdu], outfile)
saltsafekey.rem('DETSIZE', outstruct[outhdu], outfile)
saltsafekey.rem('DETSEC', outstruct[outhdu], outfile)
saltsafekey.rem('CCDSEC', outstruct[outhdu], outfile)
saltsafekey.rem('AMPSEC', outstruct[outhdu], outfile)
# add housekeeping key words
outstruct[outhdu] = addhousekeeping(outstruct[outhdu], outhdu,
outfile)
# close input FITS file
saltsafeio.closefits(struct)
# housekeeping keywords
keymosaic = 'SLOTMERG'
fname, hist = history(level=1, wrap=False)
saltsafekey.housekeeping(struct[0], keymosaic,
'Amplifiers have been mosaiced', hist)
#saltsafekey.history(outstruct[0],hist)
# this is added for later use by
saltsafekey.put('NCCDS', 0.5, outstruct[0])
saltsafekey.put('NSCIEXT', exposures, outstruct[0])
saltsafekey.put('NEXTEND', exposures, outstruct[0])
# write FITS file of mosaiced image
outstruct = pyfits.HDUList(outstruct)
saltsafeio.writefits(outstruct, outfile, clobber=clobber)
def saltbias(images,
outimages,
outpref,
subover=True,
trim=True,
subbias=False,
masterbias='bias.fits',
median=False,
function='polynomial',
order=3,
rej_lo=3,
rej_hi=3,
niter=10,
plotover=False,
turbo=False,
clobber=False,
logfile='salt.log',
verbose=True):
status = 0
ifil = 0
ii = 0
mbiasdata = []
bstruct = ''
biasgn = ''
biassp = ''
biasbn = ''
biasin = ''
filetime = {}
biastime = {}
for i in range(1, 7):
filetime[i] = []
biastime[i] = []
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
# are input and output lists the same length?
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# Does master bias frame exist?
# gain, speed, binning and instrument of master bias frame
if subbias:
if os.path.isfile(masterbias):
bstruct = saltio.openfits(masterbias)
else:
message = 'Master bias frame %s does not exist' % masterbias
raise SaltError(message)
else:
bstruct = None
# open each raw image file
for img, oimg in zip(infiles, outfiles):
#open the file
struct = saltio.openfits(img)
#check to see if it has already been bias subtracted
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid(
struct)
# has file been biaseded already?
try:
key = struct[0].header[keybias]
message = 'File %s has already been de-biased ' % infile
raise SaltError(message)
except:
pass
#compare with the master bias to make sure they are the same
if subbias:
pass
#subtract the bias
struct = bias(struct,
subover=subover,
trim=trim,
subbias=subbias,
bstruct=bstruct,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], keybias,
'Images have been de-biased', hist)
# write FITS file
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
def slotback(images,
outfits,
extension,
imgtype='image',
subbacktype='median',
sigback=3,
mbin=7,
sorder=3,
niter=5,
ampperccd=2,
ignorexp=6,
clobber=False,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# set up the variables
order = sorder
plotfreq = 0.01
ftime = plotfreq
# is the input file specified?
infiles = saltio.argunpack('Input', images)
# is the output file specified?
saltio.filedefined('Output', outfits)
#open the first file and check its charactistics
struct = saltio.openfits(infiles[0])
# how many extensions?
nextend = saltkey.get('NEXTEND', struct[0])
if nextend < extension:
msg = 'Insufficient number of extensions in %s' % (infile)
raise SaltIOError(msg)
# how many amplifiers?
amplifiers = saltkey.get('NCCDS', struct[0])
amplifiers = int(ampperccd * float(amplifiers))
if ampperccd > 0:
nframes = nextend / amplifiers
nstep = amplifiers
else:
nframes = nextend
nstep = 1
ntotal = nframes * len(infiles)
# image size
naxis1 = saltkey.get('NAXIS1', struct[extension])
naxis2 = saltkey.get('NAXIS2', struct[extension])
# CCD binning
ccdsum = saltkey.get('CCDSUM', struct[0])
binx = int(ccdsum.split(' ')[0])
biny = int(ccdsum.split(' ')[1])
# If a total file is to written out, create it and update it
hdu = 1
# delete the file if clobber is on and the file exists
if os.path.isfile(outfits):
if clobber:
saltio.delete(outfits)
else:
raise SaltIOError(
'Output fits file ' + writenewfits +
'already exists. Use Clobber=yes to overwrite file')
# create the output file
try:
hdu = pyfits.PrimaryHDU()
hdu.header = struct[0].header
hdu.header['NCCDS'] = 1
hdu.header['NSCIEXT'] = ntotal - ignorexp
hdu.header['NEXTEND'] = ntotal - ignorexp
hduList = pyfits.HDUList(hdu)
hduList.verify()
hduList.writeto(outfits)
except:
raise SaltIOError('Could not create new fits file named ' +
writenewfits)
# read it back in for updating
hduList = saltio.openfits(outfits, mode='update')
# set up the completeness tracker
if verbose:
j = 0
x2 = float(j) / float(ntotal)
ctext = 'Percentage Complete: %3.2f' % x2
sys.stdout.write(ctext)
sys.stdout.flush()
for infile in infiles:
struct = saltio.openfits(infile)
# Skip through the frames and process each frame individually
for i in range(nframes):
if not (infile == infiles[0] and i < ignorexp):
hdu = extension + i * nstep
try:
header = struct[hdu].header
array = struct[hdu].data
array = array * 1.0
except Exception, e:
msg = 'Unable to open extension %i in image %s because %s' % (
hdu, infile, e)
raise SaltIOError(msg)
# start the analysis of each frame
# gain and readout noise
try:
gain = float(header['GAIN'])
except:
gain = 1
log.warning('Gain not specified in image header')
try:
rdnoise = float(header['RDNOISE'])
except:
rdnoise = 0
log.warning('RDNOISE not specified in image header')
# background subtraction
if not subbacktype == 'none':
try:
narray = subbackground(array, sigback, mbin, order,
niter, subbacktype)
except:
log.warning('Image ' + infile + ' extention ' +
str(i) + ' is blank, skipping')
continue
# create output array
try:
if imgtype == 'background':
oarray = narray - array
else:
oarray = narray
except:
oarray = array
# print progress
if verbose:
x2 = float(j) / float(ntotal)
if x2 > ftime:
ctext = '\b\b\b\b\b %3.2f' % x2
sys.stdout.write(ctext)
sys.stdout.flush()
ftime += plotfreq
# update the header values with the image name and extension number
try:
hdue = pyfits.ImageHDU(oarray)
hdue.header = header
hdue.header.update('ONAME', infile,
'Original image name')
hdue.header.update('OEXT', hdu,
'Original extension number')
except Exception, e:
msg='SALTPHOT--WARNING: Could not update image in newfits file for %s ext %i because %s' \
% (infile, hdu, e)
raise SaltIOError(msg)
hduList.append(hdue)
j += 1
# close FITS file
saltio.closefits(struct)
def slotutcfix(images, update, outfile, ampperccd, ignorexp, droplimit, inter,
plotdata, logfile, verbose, debug):
with logging(logfile, debug) as log:
# set up the variables
utc_list = []
# is the input file specified?
saltsafeio.filedefined('Input', images)
# if the input file is a list, does it exist?
if images[0] == '@':
saltsafeio.listexists('Input', images)
# parse list of input files and place them in order
infiles = saltsafeio.listparse('Raw image', images, '', '', '')
infiles.sort()
# check input files exist
saltsafeio.filesexist(infiles, '', 'r')
# check to see if the output file exists and if so, clobber it
if os.path.isfile(outfile):
try:
os.remove(outfile)
except:
raise SaltIOError('File ' + outfile + ' can not be removed')
# open the outfile
if outfile:
try:
fout = open(outfile, 'w')
except:
raise SaltIOError('File ' + outfile + ' can not be opened')
# get time of first exposure and basic information about the observations
infile = infiles[0]
struct = saltsafeio.openfits(infile)
# check to make sure slotmode data
detmode = saltsafekey.get('DETMODE', struct[0], infile)
if detmode != 'Slot Mode':
raise SaltIOError('Data are not Slot Mode Observations')
# Check to see if SLOTUTCFIX has already been run
# and print a warning if they have
if saltsafekey.found('SLOTUTC', struct[0]):
message = 'Data have already been processed by SLOTUTCFIX'
log.warning(message)
# check to make sure that it is the right version of the software
scamver = saltsafekey.get('DETSWV', struct[0], infile)
try:
scamver = float(scamver.split('-')[-1])
if 4.42 <= scamver <= 5.00:
pass
else:
raise SaltError(
'cannot currently correct this version of the SCAM software.'
)
except:
raise SaltError('Not able to read software version')
# requested exposure time
req_texp = saltsafekey.get('EXPTIME', struct[0], infile)
# how many extensions?
nextend = saltsafekey.get('NEXTEND', struct[0], infile)
# how many amplifiers
amplifiers = saltsafekey.get('NCCDS', struct[0], infile)
amplifiers = int(ampperccd * float(amplifiers))
if ampperccd > 0:
nframes = nextend / amplifiers
nstep = amplifiers
else:
nframes = nextend
nstep = 1
# how many total frame and unique times
ntotal = nextend * len(infiles)
nunique = len(infiles) * nframes - ignorexp + 1
# Create arrays necessary for analysis
id_arr = np.arange(nunique)
utc_arr = np.zeros(nunique, dtype=float)
# Read in each file and make a list of the UTC values
if verbose:
log.message('Reading in files to create list of UTC values.')
j = 0
for n, infile in enumerate(infiles):
# Show progress
if verbose:
percent = 100. * float(n) / float(len(infiles))
ctext = 'Percentage Complete: %.2f\r' % percent
sys.stdout.write(ctext)
sys.stdout.flush()
struct = saltsafeio.openfits(infile)
if not len(struct) - 1 == nextend:
raise SaltIOError(
infile,
' has a different number of extensions from the first file'
)
# Skip through the frames and read in the utc
istart = 1
if infile == infiles[0]:
istart = ignorexp * amplifiers + 1
for i in range(istart, len(struct), amplifiers):
try:
utc_list.append(
saltsafekey.get('UTC-OBS', struct[i], infile))
utc_arr[j] = slottool.getobstime(struct[i], infile)
j += 1
except Exception, e:
raise SaltIOError(
'Unable to create array of UTC times. Please check the number of extensions in the files'
)
# close FITS file
saltsafeio.closefits(struct)
# set up the other important arrays
try:
diff_arr = utc_arr.copy()
diff_arr[1:] = diff_arr[1:] - utc_arr[:-1]
diff_arr[0] = -1
dsec_arr = utc_arr - utc_arr.astype(int)
except:
raise SaltIOError('Unable to create timing arrays')
# calculate the real exposure time
if verbose:
log.message('Calculating real exposure time.')
real_expt, med_expt, t_start, t_arr, ysum_arr = calculate_realexptime(
id_arr, utc_arr, dsec_arr, diff_arr, req_texp, utc_list)
# plot the results
if plotdata:
if verbose:
log.message('Plotting data.')
plt.ion()
plt.plot(t_arr,
ysum_arr,
linewidth=0.5,
linestyle='-',
marker='',
color='b')
plt.xlabel('Time (s)')
plt.ylabel('Fit')
# Calculate the corrrect values
if verbose:
log.message('Calculating correct values')
i_start = abs(utc_arr - t_start).argmin()
t_diff = utc_arr * 0.0 + real_expt
nd = utc_arr * 0.0
ndrop = 0
for i in range(len(utc_arr)):
if utc_arr[i] >= t_start:
t_new = t_start + real_expt * (i - i_start + ndrop)
t_diff[i] = utc_arr[i] - t_new
while (t_diff[i] > real_expt and nd[i] < droplimit):
nd[i] += 1
t_new = t_start + real_expt * (i - i_start + ndrop + nd[i])
t_diff[i] = utc_arr[i] - t_new
if (nd[i] < droplimit):
ndrop += nd[i]
else:
t_new = t_start + real_expt * (i - i_start)
t_diff[i] = utc_arr[i] - t_new
while (t_diff[i] > real_expt and nd[i] < droplimit):
nd[i] += 1
t_new = t_start + real_expt * (i - i_start - nd[i])
t_diff[i] = utc_arr[i] - t_new
# calculate the corrected timestamp by counting 6 record files forward and
# 8 recored + unrecorded files back--or just 8*t_exp forward.
# if the object is near the end of the run, then just replace it with
# the correct value assuming no dropped exposures.
# first make the array of new times
new_arr = utc_arr - t_diff
# Next loop through them to find the corrected time
corr_arr = utc_arr * 0.0
for i in range(len(new_arr)):
if i + 6 < len(new_arr) - 1:
corr_arr[i] = new_arr[i + 6] - 8 * real_expt
else:
corr_arr[i] = new_arr[i] - 2 * real_expt
t_diff = utc_arr - corr_arr
# write out the first results
msg = "Dwell Time=%5.3f Requested Exposure Time=%5.3f Nobs = %i Dropped = %i" % (
real_expt, req_texp, nunique, ndrop)
if verbose:
log.message(msg)
if outfile:
fout.write('#' + msg + '\n')
fout.write('#%23s %2s %12s %12s %10s %8s %4s \n' %
('File', 'N', 'UTC_old', 'UTC_new', 'UTC_new(s)',
'Diff', 'drop'))
# Give the user a chance to update the value
if inter:
message = 'Update headers with a dwell time of %5.3f s [y/n]? ' % real_expt
update = saltsafeio.yn_ask(message)
if not update:
message = 'Set Dwell Time manually [y/n]? '
update = saltsafeio.yn_ask(message)
if update:
message = 'New Dwell Time: '
real_expt = saltsafeio.ask(message)
try:
real_expt = float(real_expt)
except Exception, e:
msg = 'Could not set user dwell time because %s' % e
raise SaltError(msg)
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None,
subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3,
niter=5, interp='linear', clobber=False, logfile='salt.log',verbose=True):
"""Convert MEF HRS data into a single image. If variance frames and BPMs, then
convert them to the same format as well. Returns an MEF image but that is
combined into a single frame
"""
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
if saltio.checkfornone(obslogfile) is None:
raise SaltError('Obslog file is required')
# Delete the obslog file if it already exists
if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile):
if os.path.isfile(obslogfile): saltio.delete(obslogfile)
#read in the obsveration log or create it
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
else:
obsstruct=saltio.openfits(obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('OBJECT')
biaslist=filename[ccdtype=='Bias']
masterbias_dict={}
if log: log.message('Processing Bias Frames')
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bgph'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose, with_header=False)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
subover=subover, trim=trim, subbias=False, imstack=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist)
saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list)
#create the master bias frame
for i in list(masterbias_dict.keys()):
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys, x1=5, x2=13)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#apply full reductions to the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'mbgph'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#get master bias frame
masterbias=get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list)
if masterbias:
subbias=True
bstruct=saltio.openfits(masterbias)
else:
subbias=False
bstruct=None
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
subover=subover, trim=trim, subbias=subbias, imstack=True,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist)
saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
return
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist=filename[ccdtype=='ZERO']
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=pyfits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in flatlist or not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def specreduce(
images,
badpixelimage=None,
caltype="rss",
function="polynomial",
order=3,
skysub=True,
skysection=None,
findobj=False,
objsection=None,
thresh=3.0,
clobber=True,
logfile="salt.log",
verbose=True,
):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack("Input", images)
# open the badpixelstruct
if saltio.checkfornone(badpixelimage):
badpixelstruct = saltio.openfits(badpixelimage)
else:
badpixelstruct = None
# set up the section for sky estimate
if skysection is not None:
skysection = makesection(skysection)
else:
skysub = False
# set up the section for sky estimate
section = saltio.checkfornone(objsection)
if section is not None:
sections = saltio.getSection(section, iraf_format=False)
objsection = []
for i in range(0, len(sections), 2):
objsection.append((sections[i], sections[i + 1]))
# determine the wavelength solutions
if caltype == "line":
calc_wavesol(infiles)
# correct the images
for img in infiles:
# open the fits file
struct = saltio.openfits(img)
# prepare filep
log.message("Preparing %s" % img)
struct = prepare(struct, badpixelstruct)
# rectify the spectrum
log.message("Rectifying %s using %s" % (img, caltype))
struct = rectify(struct, None, caltype=caltype, function=function, order=order)
# sky subtract the spectrum
# assumes the data is long slit and in the middle of the field
if skysub:
log.message("Subtracting the sky from %s" % img)
struct = skysubtract(struct, method="normal", section=skysection)
# extract the spectrum
log.message("Extracting the spectrum from %s" % img)
print objsection
aplist = extract(struct, method="normal", section=objsection, thresh=thresh)
oimg = os.path.dirname(os.path.abspath(img)) + "/s" + os.path.basename(img.strip())
ofile = oimg[:-5] + ".txt"
write_extract(ofile, aplist, clobber=clobber)
# write FITS file
log.message("Writing 2-D corrected image as %s" % oimg)
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
def saltxtalk(images,
outimages,
outpref,
xtalkfile=None,
usedb=False,
clobber=True,
logfile='salt.log',
verbose=True):
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
# are input and output lists the same length?
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# does crosstalk coefficient data exist
if usedb:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict = None
for img, oimg in zip(infiles, outfiles):
#open the fits file
struct = saltio.openfits(img)
#find the best xcoeff for the image if using the db
if usedb:
obsdate = saltkey.get('DATE-OBS', struct[0])
obsdate = int('%s%s%s' %
(obsdate[0:4], obsdate[5:7], obsdate[8:]))
xkey = np.array(xdict.keys())
date = xkey[abs(xkey - obsdate).argmin()]
xcoeff = xdict[date]
else:
xcoeff = []
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltkey.instrumid(
struct)
# has file been prepared already?
if saltkey.found(keyxtalk, struct[0]):
message = '%s has already been xtalk corrected' % img
raise SaltError(message)
#apply the cross-talk correction
struct = xtalk(struct, xcoeff, log=log, verbose=verbose)
# housekeeping keywords
fname, hist = history(level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SXTALK',
'Images have been xtalk corrected', hist)
# write FITS file
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
def slotview(newfits,indata , fileout, srcfile, fps=10.0, phottype='square', sigdet=5, contpix=10, \
driftlimit=10, clobber=True,logfile='slotview.log',verbose=True):
#set up the variables
status = 0
entries = []
vig_lo = {}
vig_hi = {}
hour = 0
min = 0
sec = 0.
time0 = 0.
nframes = 0
sleep = 0
with logging(logfile, debug) as log:
#enter in the input data
saltio.fileexists(newfits)
#set the sleep parameter
if fps > 0: sleep = 1.0 / (fps)
# read in the data file
id, time, ratio, rerr, tx, ty, tflux, terr, cx, cy, cflux, cerr = st.readlcfile(
indata)
# read extraction region defintion file
amp, x, y, x_o, y_o, r, br1, br2 = st.readsrcfile(srcfile)
#determine the size of the data arrays
struct = saltio.openfits(newfits)
naxis1 = saltkey.get('NAXIS1', struct[1])
naxis2 = saltkey.get('NAXIS2', struct[1])
# Plot all of the data and the first image
# Create GUI
App = QtGui.QApplication([])
aw=SlotViewWindow(struct, id, tflux, cflux, ratio, time, phottype, sleep, \
tx, ty, cx, cy, r, br1, br2, naxis1, naxis2, sigdet, contpix, driftlimit)
aw.show()
# Start application event loop
app_exit = App.exec_()
# Check if GUI was executed succesfully
if app_exit != 0:
raise SALTError('InterIdentify GUI has unexpected exit status ' +
str(exit))
ratio, tflux, cflux, gframe, newphot = aw.ratio, aw.tflux, aw.cflux, aw.goodframes, aw.newphot
#close the input file
saltio.closefits(struct)
# Update the indata file if necessary
lc = saltio.openascii(fileout, 'w')
for i in range(len(ratio)):
x['target'] = tx[i]
x['comparison'] = cx[i]
y['target'] = ty[i]
y['comparison'] = cy[i]
reltime = False
if gframe[i]:
st.writedataout(lc, id[i], time[i], x, y, tflux[i], terr[i], \
cflux[i], cerr[i], ratio[i], rerr[i], time[0], reltime)
saltio.closeascii(lc)
def specreduce(images, badpixelimage=None, caltype='rss',
function='polynomial', order=3,
skysub=True, skysection=None, findobj=False,
objsection=None, thresh=3.0,
clobber=True, logfile='salt.log', verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# open the badpixelstruct
if saltio.checkfornone(badpixelimage):
badpixelstruct = saltio.openfits(badpixelimage)
else:
badpixelstruct = None
# set up the section for sky estimate
if skysection is not None:
skysection = makesection(skysection)
else:
skysub = False
# set up the section for sky estimate
section = saltio.checkfornone(objsection)
if section is not None:
sections = saltio.getSection(section, iraf_format=False)
objsection = []
for i in range(0, len(sections), 2):
objsection.append((sections[i], sections[i + 1]))
# determine the wavelength solutions
if caltype == 'line':
calc_wavesol(infiles)
# correct the images
for img in infiles:
# open the fits file
struct = saltio.openfits(img)
# prepare filep
log.message('Preparing %s' % img)
struct = prepare(struct, badpixelstruct)
# rectify the spectrum
log.message('Rectifying %s using %s' % (img, caltype))
struct = rectify(
struct,
None,
caltype=caltype,
function=function,
order=order)
# sky subtract the spectrum
# assumes the data is long slit and in the middle of the field
if skysub:
log.message('Subtracting the sky from %s' % img)
struct = skysubtract(
struct,
method='normal',
section=skysection)
# extract the spectrum
log.message('Extracting the spectrum from %s' % img)
print objsection
aplist = extract(
struct,
method='normal',
section=objsection,
thresh=thresh)
oimg = os.path.dirname(
os.path.abspath(img)) + '/s' + os.path.basename(img.strip())
ofile = oimg[:-5] + '.txt'
write_extract(ofile, aplist, clobber=clobber)
# write FITS file
log.message('Writing 2-D corrected image as %s' % oimg)
saltio.writefits(struct, oimg, clobber=clobber)
saltio.closefits(struct)
def slotback(images,outfits,extension,imgtype='image',subbacktype='median',
sigback=3,mbin=7,sorder=3,niter=5,ampperccd=2,ignorexp=6,
clobber=False,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# set up the variables
order=sorder
plotfreq=0.01
ftime=plotfreq
# is the input file specified?
infiles = saltio.argunpack ('Input',images)
# is the output file specified?
saltio.filedefined('Output',outfits)
#open the first file and check its charactistics
struct=saltio.openfits(infiles[0])
# how many extensions?
nextend=saltkey.get('NEXTEND',struct[0])
if nextend < extension:
msg='Insufficient number of extensions in %s' % (infile)
raise SaltIOError(msg)
# how many amplifiers?
amplifiers=saltkey.get('NCCDS',struct[0])
amplifiers = int(ampperccd*float(amplifiers))
if ampperccd>0:
nframes = nextend/amplifiers
nstep=amplifiers
else:
nframes = nextend
nstep=1
ntotal=nframes*len(infiles)
# image size
naxis1=saltkey.get('NAXIS1',struct[extension])
naxis2=saltkey.get('NAXIS2',struct[extension])
# CCD binning
ccdsum=saltkey.get('CCDSUM',struct[0])
binx=int(ccdsum.split(' ')[0])
biny=int(ccdsum.split(' ')[1])
# If a total file is to written out, create it and update it
hdu = 1
# delete the file if clobber is on and the file exists
if os.path.isfile(outfits):
if clobber:
saltio.delete(outfits)
else:
raise SaltIOError('Output fits file '+writenewfits+'already exists. Use Clobber=yes to overwrite file')
# create the output file
try:
hdu = pyfits.PrimaryHDU()
hdu.header=struct[0].header
hdu.header['NCCDS']=1
hdu.header['NSCIEXT']=ntotal-ignorexp
hdu.header['NEXTEND']=ntotal-ignorexp
hduList = pyfits.HDUList(hdu)
hduList.verify()
hduList.writeto(outfits)
except:
raise SaltIOError('Could not create new fits file named '+writenewfits)
# read it back in for updating
hduList = saltio.openfits(outfits,mode='update')
# set up the completeness tracker
if verbose:
j=0
x2=float(j)/float(ntotal)
ctext='Percentage Complete: %3.2f' % x2
sys.stdout.write(ctext)
sys.stdout.flush()
for infile in infiles:
struct=saltio.openfits(infile)
# Skip through the frames and process each frame individually
for i in range(nframes):
if not (infile==infiles[0] and i < ignorexp):
hdu=extension+i*nstep
try:
header=struct[hdu].header
array=struct[hdu].data
array=array*1.0
except Exception as e:
msg='Unable to open extension %i in image %s because %s' % (hdu, infile, e)
raise SaltIOError(msg)
# start the analysis of each frame
# gain and readout noise
try:
gain=float(header['GAIN'])
except:
gain=1
log.warning('Gain not specified in image header')
try:
rdnoise=float(header['RDNOISE'])
except:
rdnoise=0
log.warning('RDNOISE not specified in image header')
# background subtraction
if not subbacktype=='none':
try:
narray=subbackground(array, sigback, mbin, order, niter, subbacktype)
except:
log.warning('Image '+infile+' extention '+str(i)+' is blank, skipping')
continue
# create output array
try:
if imgtype=='background':
oarray=narray-array
else:
oarray=narray
except:
oarray=array
# print progress
if verbose:
x2=float(j)/float(ntotal)
if x2 > ftime:
ctext='\b\b\b\b\b %3.2f' % x2
sys.stdout.write(ctext)
sys.stdout.flush()
ftime += plotfreq
# update the header values with the image name and extension number
try:
hdue = pyfits.ImageHDU(oarray)
hdue.header=header
hdue.header.update('ONAME',infile,'Original image name')
hdue.header.update('OEXT',hdu,'Original extension number')
except Exception as e:
msg='SALTPHOT--WARNING: Could not update image in newfits file for %s ext %i because %s' \
% (infile, hdu, e)
raise SaltIOError(msg)
hduList.append(hdue)
j +=1
# close FITS file
saltio.closefits(struct)
# close the output fits file:
try:
# write out the file
hduList.flush()
hduList.close()
except Exception as e:
raise SaltIOError('Failed to write %s because %s' % (outfits, e))
def saltobsid(propcode,obslog,rawpath,prodpath,outpath,prefix='mbxgp', fprefix='bxgp',clobber=False,logfile='salt.log',verbose=True):
"""Split data into their different data directories
"""
with logging(logfile,debug) as log:
# are the arguments defined
pids = saltio.argunpack('propcode',propcode)
# check observation log file exists
obslog = obslog.strip()
saltio.fileexists(obslog)
#open the observing log
obstruct = saltio.openfits(obslog)
obstab = saltio.readtab(obstruct[1],obslog)
saltio.closefits(obstruct)
#read in the file information
filenames = saltstring.listfunc(obstab.field('filename'),'lstrip')
instrumes = saltstring.listfunc(obstab.field('instrume'),'lstrip')
proposers = saltstring.listfunc(obstab.field('proposer'),'clean')
propids = saltstring.listfunc(obstab.field('propid'),'clean')
ccdtypes = saltstring.listfunc(obstab.field('ccdtype'),'clean')
ccdsums = saltstring.listfunc(obstab.field('ccdsum'),'clean')
gainsets = saltstring.listfunc(obstab.field('gainset'),'clean')
rospeeds = saltstring.listfunc(obstab.field('rospeed'),'clean')
detmodes = saltstring.listfunc(obstab.field('detmode'),'clean')
filters = saltstring.listfunc(obstab.field('filter'),'clean')
gratings = saltstring.listfunc(obstab.field('grating'),'clean')
gr_angles = obstab.field('gr-angle')
ar_angles = obstab.field('ar-angle')
# Create the list of proposals
try:
pids=saltio.cleanpropcode(pids, propids)
except SaltIOError:
#throw a warning adn exit if not data needs to be filterd
log.warning('No data to filter\n', with_stdout=verbose)
return
# check paths exist, end with a "/" and convert them to absolute paths
rawpath = saltio.abspath(rawpath)
prodpath = saltio.abspath(prodpath)
outpath = saltio.abspath(outpath)
#create the symlink raw path
rawsplit=rawpath.strip().split('/')
symrawpath='../../%s/%s/' % (rawsplit[-3], rawsplit[-2])
prodsplit=prodpath.strip().split('/')
symprodpath='../../%s/%s/' % (prodsplit[-3], prodsplit[-2])
# create PI directories
for pid in pids:
saltio.createdir(outpath+pid)
saltio.createdir(outpath+pid+'/raw')
saltio.createdir(outpath+pid+'/product')
#copy the data that belongs to a pid into that directory
log.message('SALTOBSID -- filtering images to proposal directories\n', with_stdout=verbose)
#copy data for a given proposal to the raw and produce directories
for i in range(len(obstab)):
if os.path.exists(outpath+obstab[i]['propid']):
if obstab[i]['object'].upper() not in ['ZERO', 'BIAS']:
fname=obstab[i]['filename']
pdir=obstab[i]['propid']
detmode=obstab[i]['detmode']
linkfiles(fname, pdir,detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#look through the bias/flat/arc/standard data to see if there is any relavent data
log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose)
caldata=['ZERO', 'FLAT', 'ARC']
biasheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED']
flatheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
archeader_list=['OBSMODE', 'DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
calproplist=['CAL_SPST']
#Include bias frames
log.message('SALTOBSID -- filtering bias files to proposal directories\n', with_stdout=verbose)
for i in range(len(obstab)):
fname=obstab[i]['filename']
prop_list=[]
#if it is a zero, check to see what other data have the same settings
if obstab[i]['CCDTYPE'].strip().upper()=='ZERO' or obstab[i]['OBJECT'].strip().upper() in ['BIAS', 'ZERO']:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], biasheader_list):
prop_list.append(obstab[i]['PROPID'])
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
detmode=obstab[i]['detmode']
linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, fprefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#Include calibration frames
log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose)
for i in range(len(obstab)):
fname=obstab[i]['filename']
prop_list=[]
#if it is a flat, check to see what other data have the same settings
#this is turned off
if obstab[i]['CCDTYPE'].strip().upper()=='FLAT' and False:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], flatheader_list):
prop_list.append(obstab[j]['PROPID'])
#if it is a arc, check to see what other data have the same settings
#this is turned off
if obstab[i]['CCDTYPE'].strip().upper()=='ARC' and False:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], archeader_list):
prop_list.append(obstab[j]['PROPID'])
#if it is a calibration standard, see what other data have the same settings
if obstab[i]['PROPID'].strip().upper() in calproplist:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], flatheader_list):
prop_list.append(obstab[j]['PROPID'])
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
if pdir!=obstab[i]['propid']:
detmode=obstab[i]['detmode']
linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#Include master (bias or flat) frames
log.message('SALTOBSID -- filtering master calibration files to proposal directories\n', with_stdout=verbose)
masterlist=glob.glob(prodpath+'*Bias*')+glob.glob(prodpath+'*Flat*')
for bimg in masterlist:
struct=pyfits.open(bimg)
bdict={}
prop_list=[]
for k in biasheader_list:
bdict[k]=saltkey.get(k, struct[0])
for i in range(len(obstab)):
if comparefiles(obstab[i], bdict, biasheader_list):
prop_list.append(obstab[i]['PROPID'])
struct.close()
#copy the files over to the directory
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
fname=os.path.basename(bimg)
infile = symprodpath+fname
link = outpath+pdir+'/product/'+fname
saltio.symlink(infile,link,clobber)
message='Copying %s to %s' % (fname ,pdir)
log.message(message, with_header=False, with_stdout=verbose)
def slotmerge(images,outimages,outpref,geomfile,clobber,logfile,verbose):
with logging(logfile,debug) as log:
# are the arguments defined
saltsafeio.argdefined('images',images)
saltsafeio.argdefined('geomfile',geomfile)
saltsafeio.argdefined('logfile',logfile)
# if the input file is a list, does it exist?
if images[0] == '@':
saltsafeio.listexists('Input',images)
# parse list of input files
infiles=saltsafeio.listparse('Raw image',images,'','','')
# check input files exist
saltsafeio.filesexist(infiles,'','r')
# load output name list: @list, * and comma separated
outimages = outimages.strip()
outpref = outpref.strip()
if len(outpref) == 0 and len(outimages) == 0:
raise SaltIOError('Output file(s) not specified')
# test output @filelist exists
if len(outimages) > 0 and outimages[0] == '@':
saltsafeio.listexists('Output',outimages)
# parse list of output files
outfiles=saltsafeio.listparse('Output image',outimages,outpref,infiles,'')
# are input and output lists the same length?
saltsafeio.comparelists(infiles,outfiles,'Input','output')
# do the output files already exist?
if not clobber:
saltsafeio.filesexist(outfiles,'','w')
# does CCD geometry definition file exist
geomfilefile = geomfile.strip()
saltsafeio.fileexists(geomfile)
# read geometry definition file
gap = 0
xshift = [0, 0]
yshift = [0, 0]
rotation = [0, 0]
gap, xshift, yshift, rotation=saltsafeio.readccdgeom(geomfile)
for ro in rotation:
if ro!=0:
log.warning('SLOTMERGE currently ignores CCD rotation')
# Begin processes each file
for infile, outfile in zip(infiles, outfiles):
# determine the name for the output file
outpath = outfile.rstrip(os.path.basename(outfile))
if (len(outpath) == 0):
outpath = '.'
# open each raw image
struct=saltsafeio.openfits(infile)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot=saltsafekey.instrumid(struct,infile)
# how many amplifiers?
nccds=saltsafekey.get('NCCDS',struct[0],infile)
amplifiers = nccds * 2
#if (nccds != 2):
# raise SaltError('Can not currently handle more than two CCDs')
# CCD geometry coefficients
if instrume == 'RSS' or instrume == 'PFIS':
xsh = [xshift[0], 0., xshift[1]]
ysh = [yshift[0], 0., yshift[1]]
rot = [rotation[0], 0., rotation[1]]
refid = 1
if instrume == 'SALTICAM':
xsh = [xshift[0], 0.]
ysh = [yshift[0], 0.]
rot = [rotation[0], 0]
refid = 1
# how many extensions?
nextend=saltsafekey.get('NEXTEND',struct[0],infile)
# how many exposures
exposures = nextend/amplifiers
# CCD on-chip binning
xbin, ybin=saltsafekey.ccdbin(struct[0],infile)
gp = int(gap / xbin)
# create output hdu structure
outstruct = [None] * int(exposures+1)
outstruct[0]=struct[0]
# iterate over exposures, stitch them to produce file of CCD images
for i in range(exposures):
# Determine the total size of the image
xsize=0
ysize=0
for j in range(amplifiers):
hdu=i*amplifiers+j+1
try:
xsize += len(struct[hdu].data[0])
if ysize < len(struct[hdu].data):
ysize=len(struct[hdu].data)
except:
msg='Unable to access extension %i ' % hdu
raise SaltIOError(msg)
xsize += gp* (nccds-1)
maxxsh, minxsh = determineshifts(xsh)
maxysh, minysh = determineshifts(ysh)
xsize += (maxxsh-minxsh)
ysize += (maxysh-minysh)
# Determine the x and y origins for each frame
xdist=0
ydist=0
shid=0
x0=np.zeros(amplifiers)
y0=np.zeros(amplifiers)
for j in range(amplifiers):
x0[j]=xdist+xsh[shid]-minxsh
y0[j]=ysh[shid]-minysh
hdu=i*amplifiers+j+1
darr=struct[hdu].data
xdist += len(darr[0])
if j%2==1:
xdist += gp
shid += 1
# make the out image
outarr=np.zeros((ysize, xsize), np.float64)
# Embed each frame into the output array
for j in range(amplifiers):
hdu=i*amplifiers+j+1
darr=struct[hdu].data
outarr=salttran.embed(darr, x0[j], y0[j], outarr)
# Add the outimage to the output structure
hdu=i*amplifiers+1
outhdu=i+1
outstruct[outhdu] = pyfits.ImageHDU(outarr)
outstruct[outhdu].header=struct[hdu].header
# Fix the headers in each extension
datasec='[1:%4i,1:%4i]' % (xsize, ysize)
saltsafekey.put('DATASEC',datasec, outstruct[outhdu], outfile)
saltsafekey.rem('DETSIZE',outstruct[outhdu],outfile)
saltsafekey.rem('DETSEC',outstruct[outhdu],outfile)
saltsafekey.rem('CCDSEC',outstruct[outhdu],outfile)
saltsafekey.rem('AMPSEC',outstruct[outhdu],outfile)
# add housekeeping key words
outstruct[outhdu]=addhousekeeping(outstruct[outhdu], outhdu, outfile)
# close input FITS file
saltsafeio.closefits(struct)
# housekeeping keywords
keymosaic='SLOTMERG'
fname, hist=history(level=1, wrap=False)
saltsafekey.housekeeping(struct[0],keymosaic,'Amplifiers have been mosaiced', hist)
#saltsafekey.history(outstruct[0],hist)
# this is added for later use by
saltsafekey.put('NCCDS', 0.5, outstruct[0])
saltsafekey.put('NSCIEXT', exposures, outstruct[0])
saltsafekey.put('NEXTEND', exposures, outstruct[0])
# write FITS file of mosaiced image
outstruct=pyfits.HDUList(outstruct)
saltsafeio.writefits(outstruct, outfile, clobber=clobber)
def saltflat(images,outimages,outpref, flatimage,minflat=1, allext=False, clobber=False, \
logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# check flatfield image exists
flaimage= flatimage.strip()
saltio.fileexists(flatimage)
flatstruct= saltio.openfits(flatimage)
# Normalize the flat field image
# This requires to go through each science extension and divide it by
# mean of the image. Things that have to be checked:
# that data exist, that it is a science extension
#determine the global mean
fmean=0
fcount=0
#replace bad pixels
for i in range(len(flatstruct)):
if flatstruct[i].data is not None and (flatstruct[i].name=='SCI' or flatstruct[i].name=='PRIMARY'):
data = flatstruct[i].data
mask = (data > minflat)
if (numpy.nan==flatstruct[i].data).sum() or (numpy.inf==flatstruct[i].data).sum():
message = '\nWARNING -- SALTFLAT: %s contains invalid values' % flatimage
log.warning(message,with_stdout=verbose)
flatstruct[i].data[mask==0] = minflat
flatstruct[i].data[flatstruct[i].data==numpy.inf] = minflat
#determine the mean
mask = (data > minflat)
fmean += data[mask].sum()
fcount += data[mask].size
if fcount>0: fmean=fmean/fcount
for i in range(len(flatstruct)):
if flatstruct[i].name=='PRIMARY':
#is it a flat--if not throw a warning
try:
key_ccdtype=saltkey.get('CCDTYPE', flatstruct[i])
except:
key_ccdtype=None
if key_ccdtype!='FLAT':
message = '\nWARNING -- SALTFLAT: %s does not have CCDTYPE=FLAT' % flatimage
log.warning(message,with_stdout=verbose)
#if there are data, normalize it
if flatstruct[i].data is not None:
flatstruct[i].data=flatnormalize(flatstruct[i].data, minflat)
#Noramlize the science extensions
if flatstruct[i].name=='SCI':
if flatstruct[i].data is not None:
if allext is False: fmean=flatstruct[i].data.mean()
flatstruct[i].data=flatnormalize(flatstruct[i].data, minflat, fmean)
#Apply to the variance frames
if saltkey.found('VAREXT', flatstruct[i]):
varext=saltkey.get('VAREXT',flatstruct[i])
flatstruct[varext].data=flatstruct[varext].data/fmean**2
# open each raw image file
for infile, outfile in zip(infiles,outfiles):
struct = saltio.openfits(infile)
# flat field correct the image
outstruct = flat(struct, flatstruct)
try:
pass
except Exception,e:
msg='Unable to flatten %s because %s' % (infile, e)
raise SaltError(msg)
#add any header keywords like history
fname, hist=history(level=1, wrap=False)
saltkey.housekeeping(struct[0],'SFLAT', 'File flatfield corrected', hist)
#write it out and close it
saltio.writefits(outstruct,outfile,clobber=clobber)
saltio.closefits(struct)
#output the information
log.message('Flatfields image %s using %s' % (infile, flatimage), with_header=False, with_stdout=verbose)
#clost the flatfield image
saltio.closefits(flatstruct)
def slotutcfix(images,update,outfile,ampperccd,ignorexp,droplimit,inter,plotdata,logfile,verbose,debug):
with logging(logfile,debug) as log:
# set up the variables
utc_list = []
# is the input file specified?
saltsafeio.filedefined('Input',images)
# if the input file is a list, does it exist?
if images[0] == '@':
saltsafeio.listexists('Input',images)
# parse list of input files and place them in order
infiles=saltsafeio.listparse('Raw image',images,'','','')
infiles.sort()
# check input files exist
saltsafeio.filesexist(infiles,'','r')
# check to see if the output file exists and if so, clobber it
if os.path.isfile(outfile):
try:
os.remove(outfile)
except:
raise SaltIOError('File ' + outfile + ' can not be removed')
# open the outfile
if outfile:
try:
fout=open(outfile,'w')
except:
raise SaltIOError('File ' + outfile + ' can not be opened')
# get time of first exposure and basic information about the observations
infile=infiles[0]
struct=saltsafeio.openfits(infile)
# check to make sure slotmode data
detmode=saltsafekey.get('DETMODE',struct[0], infile)
if detmode != 'Slot Mode':
raise SaltIOError('Data are not Slot Mode Observations')
# Check to see if SLOTUTCFIX has already been run
# and print a warning if they have
if saltsafekey.found('SLOTUTC', struct[0]):
message='Data have already been processed by SLOTUTCFIX'
log.warning(message)
# check to make sure that it is the right version of the software
scamver=saltsafekey.get('DETSWV', struct[0], infile)
try:
scamver=float(scamver.split('-')[-1])
if 4.42 <= scamver <= 5.00:
pass
else:
raise SaltError('cannot currently correct this version of the SCAM software.')
except:
raise SaltError('Not able to read software version')
# requested exposure time
req_texp=saltsafekey.get('EXPTIME',struct[0],infile)
# how many extensions?
nextend=saltsafekey.get('NEXTEND',struct[0],infile)
# how many amplifiers
amplifiers=saltsafekey.get('NCCDS',struct[0],infile)
amplifiers = int(ampperccd*float(amplifiers))
if ampperccd>0:
nframes = nextend/amplifiers
nstep=amplifiers
else:
nframes = nextend
nstep=1
# how many total frame and unique times
ntotal=nextend*len(infiles)
nunique=len(infiles)*nframes-ignorexp+1
# Create arrays necessary for analysis
id_arr=np.arange(nunique)
utc_arr=np.zeros(nunique,dtype=float)
# Read in each file and make a list of the UTC values
if verbose:
log.message('Reading in files to create list of UTC values.')
j=0
for n,infile in enumerate(infiles):
# Show progress
if verbose:
percent=100.*float(n)/float(len(infiles))
ctext='Percentage Complete: %.2f\r' % percent
sys.stdout.write(ctext)
sys.stdout.flush()
struct=saltsafeio.openfits(infile)
if not len(struct)-1==nextend:
raise SaltIOError(infile,' has a different number of extensions from the first file')
# Skip through the frames and read in the utc
istart=1
if infile==infiles[0]:
istart=ignorexp*amplifiers+1
for i in range(istart,len(struct), amplifiers):
try:
utc_list.append(saltsafekey.get('UTC-OBS', struct[i], infile))
utc_arr[j]=slottool.getobstime(struct[i], infile)
j += 1
except Exception, e:
raise SaltIOError('Unable to create array of UTC times. Please check the number of extensions in the files')
# close FITS file
saltsafeio.closefits(struct)
# set up the other important arrays
try:
diff_arr=utc_arr.copy()
diff_arr[1:]=diff_arr[1:]-utc_arr[:-1]
diff_arr[0]=-1
dsec_arr=utc_arr-utc_arr.astype(int)
except:
raise SaltIOError('Unable to create timing arrays')
# calculate the real exposure time
if verbose:
log.message('Calculating real exposure time.')
real_expt, med_expt, t_start, t_arr, ysum_arr=calculate_realexptime(id_arr, utc_arr, dsec_arr, diff_arr, req_texp, utc_list)
# plot the results
if plotdata:
if verbose:
log.message('Plotting data.')
plt.ion()
plt.plot(t_arr,ysum_arr,linewidth=0.5,linestyle='-',marker='',color='b')
plt.xlabel('Time (s)')
plt.ylabel('Fit')
# Calculate the corrrect values
if verbose:
log.message('Calculating correct values')
i_start = abs(utc_arr-t_start).argmin()
t_diff=utc_arr*0.0+real_expt
nd=utc_arr*0.0
ndrop=0
for i in range(len(utc_arr)):
if utc_arr[i] >= t_start:
t_new=t_start+real_expt*(i-i_start+ndrop)
t_diff[i]=utc_arr[i]-t_new
while (t_diff[i]>real_expt and nd[i] < droplimit):
nd[i]+= 1
t_new=t_start+real_expt*(i-i_start+ndrop+nd[i])
t_diff[i]=utc_arr[i]-t_new
if (nd[i]<droplimit):
ndrop += nd[i]
else:
t_new=t_start+real_expt*(i-i_start)
t_diff[i]=utc_arr[i]-t_new
while (t_diff[i]>real_expt and nd[i] < droplimit):
nd[i]+= 1
t_new=t_start+real_expt*(i-i_start-nd[i])
t_diff[i]=utc_arr[i]-t_new
# calculate the corrected timestamp by counting 6 record files forward and
# 8 recored + unrecorded files back--or just 8*t_exp forward.
# if the object is near the end of the run, then just replace it with
# the correct value assuming no dropped exposures.
# first make the array of new times
new_arr=utc_arr-t_diff
# Next loop through them to find the corrected time
corr_arr=utc_arr*0.0
for i in range(len(new_arr)):
if i+6 < len(new_arr)-1:
corr_arr[i]=new_arr[i+6]-8*real_expt
else:
corr_arr[i]=new_arr[i]-2*real_expt
t_diff=utc_arr-corr_arr
# write out the first results
msg="Dwell Time=%5.3f Requested Exposure Time=%5.3f Nobs = %i Dropped = %i" % (real_expt, req_texp, nunique, ndrop)
if verbose:
log.message(msg)
if outfile:
fout.write('#'+msg+'\n')
fout.write('#%23s %2s %12s %12s %10s %8s %4s \n' % ('File', 'N', 'UTC_old', 'UTC_new', 'UTC_new(s)', 'Diff', 'drop' ))
# Give the user a chance to update the value
if inter:
message='Update headers with a dwell time of %5.3f s [y/n]? ' % real_expt
update=saltsafeio.yn_ask(message)
if not update:
message='Set Dwell Time manually [y/n]? '
update=saltsafeio.yn_ask(message)
if update:
message='New Dwell Time: '
real_expt=saltsafeio.ask(message)
try:
real_expt=float(real_expt)
except Exception, e:
msg='Could not set user dwell time because %s' % e
raise SaltError(msg)
if outfile:
utc_new=saltsafekey.get('UTC-OBS', struct[i+k], infile)
utc_new_sec=slottool.getobstime(struct[i], infile)
fout.write('%25s %2i %12s %12s %7.3f %5.4f %4i \n' % (infile, i, utc_list[j], utc_new, utc_new_sec, t_diff[j], nd[j] ))
j += 1
# Housekeeping key words
if update:
history = 'SALTUTCFIX -- '
history += 'images='+infile+' '
saltsafekey.housekeeping(struct[0],'SLOTUTC','UTC has been corrected',history,infile)
# update fits file
if update:
saltsafeio.updatefits(struct)
saltsafeio.closefits(struct)
# close outfile
if outfile:
fout.close()
# Keep plot window open
if plotdata:
plt.show()
def updateheaders(struct, ext, tdiff, real_expt, utc, infile):
# exit if tdiff wasn't updated
if tdiff == real_expt:
msg='No adequate correction found for frame %i in file %s' % (ext, infile)
raise SaltError(msg)
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='',
clobber=False, cleanup=True, logfile='salt.log', verbose=True):
"""SALTADVANCE provides advanced data reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
infiles.sort()
# create list of output files
outpath=saltio.abspath(outpath)
#log into the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
obsmode=obsstruct.data.field('OBSMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
propcode=obsstruct.data.field('PROPID')
masktype=obsstruct.data.field('MASKTYP')
#set the bias list of objects
biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')]
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=fits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=fits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the arc data
arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')]
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if nimg in arclist:
#open the image
struct=fits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
obsdate=os.path.basename(img)[1:9]
#print the message
if log:
message='Processing ARC frame %s' % img
log.message(message, with_stdout=verbose)
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
log=log, verbose=verbose)
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the images
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#measure the arcdata
arcimage=outpath+'mbxgp'+nimg
dbfile=outpath+obsdate+'_specid.db'
lamp = obsstruct.data.field('LAMPID')[i]
lamp = lamp.replace(' ', '')
lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
print arcimage, lampfile, os.getcwd()
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod='Matchlines', function='legendre',
order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3,
startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile,
verbose=verbose)
try:
ximg = outpath+'xmbxgp'+os.path.basename(arcimage)
specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except:
pass
#process the science data
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if not (nimg in flatlist or nimg in biaslist or nimg in arclist):
#open the image
struct=fits.open(img)
if struct[0].header['PROPID'].count('CAL_GAIN'): continue
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#Check to see if it is RSS 2x2 and add bias subtraction
instrume=saltkey.get('INSTRUME', struct[0]).strip()
gainset = saltkey.get('GAINSET', struct[0])
rospeed = saltkey.get('ROSPEED', struct[0])
target = saltkey.get('OBJECT', struct[0]).strip()
exptime = saltkey.get('EXPTIME', struct[0])
obsmode = saltkey.get('OBSMODE', struct[0]).strip()
detmode = saltkey.get('DETMODE', struct[0]).strip()
masktype = saltkey.get('MASKTYP', struct[0]).strip()
xbin, ybin = saltkey.ccdbin( struct[0], img)
obsdate=os.path.basename(img)[1:9]
bstruct=None
crtype=None
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=5
subbias=False
if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW':
bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin)
if os.path.exists(bfile):
bstruct=fits.open(bfile)
subbias=True
if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5:
crtype='edge'
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=3
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \
bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise,
bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter,
log=log, verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#if the file is spectroscopic mode, apply the wavelength correction
if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT':
dbfile=outpath+obsdate+'_specid.db'
try:
ximg = outpath+'xmbxgp'+os.path.basename(img)
specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except Exception, e:
log.message('%s' % e)
#clean up the results
if cleanup:
#clean up the bias frames
for i in masterbias_dict.keys():
blist=masterbias_dict[i][1:]
for b in blist: saltio.delete(b)
#clean up the flat frames
for i in masterflat_dict.keys():
flist=masterflat_dict[i][1:]
for f in flist: saltio.delete(f)
def saltclean(images,
outpath,
obslogfile=None,
gaindb=None,
xtalkfile=None,
geomfile=None,
subover=True,
trim=True,
masbias=None,
subbias=False,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover = False
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist = saltio.readgaindb(gaindb)
else:
dblist = []
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict = None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg = 'The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist = filename[ccdtype == 'ZERO']
masterbias_dict = {}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = fits.open(img)
bimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(blist, bkeys)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist = filename[ccdtype == 'FLAT']
masterflat_dict = {}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct = fits.open(img)
fimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict = compareimages(struct,
fimg,
masterflat_dict,
keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys = masterflat_dict[i][0]
flist = masterflat_dict[i][1:]
mflatname = outpath + createmasterflatname(flist, fkeys)
ffiles = ','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg = os.path.basename(img)
#print nimg, nimg in flatlist, nimg in biaslist
if not (nimg in biaslist):
#open the image
struct = fits.open(img)
simg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg = outpath + 'mbxgp' + os.path.basename(img)
saltmosaic(images=simg,
outimages=mimg,
outpref='',
geomfile=geomfile,
interp=interp,
cleanup=True,
clobber=clobber,
logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def slotview(newfits,indata , fileout, srcfile, fps=10.0, phottype='square', sigdet=5, contpix=10, \
driftlimit=10, clobber=True,logfile='slotview.log',verbose=True):
#set up the variables
status = 0
entries = []
vig_lo = {}
vig_hi = {}
hour = 0
min = 0
sec = 0.
time0 = 0.
nframes = 0
sleep=0
with logging(logfile,debug) as log:
#enter in the input data
saltio.fileexists(newfits)
#set the sleep parameter
if fps>0: sleep=1.0/(fps)
# read in the data file
id, time, ratio, rerr, tx, ty, tflux, terr, cx, cy, cflux, cerr=st.readlcfile(indata)
# read extraction region defintion file
amp, x, y, x_o, y_o, r, br1, br2=st.readsrcfile(srcfile)
#determine the size of the data arrays
struct = saltio.openfits(newfits)
naxis1 = saltkey.get('NAXIS1',struct[1])
naxis2 = saltkey.get('NAXIS2',struct[1])
# Plot all of the data and the first image
# Create GUI
App = QtGui.QApplication([])
aw=SlotViewWindow(struct, id, tflux, cflux, ratio, time, phottype, sleep, \
tx, ty, cx, cy, r, br1, br2, naxis1, naxis2, sigdet, contpix, driftlimit)
aw.show()
# Start application event loop
app_exit=App.exec_()
# Check if GUI was executed succesfully
if app_exit!=0:
raise SALTError('InterIdentify GUI has unexpected exit status '+str(exit))
ratio, tflux, cflux, gframe, newphot=aw.ratio, aw.tflux, aw.cflux, aw.goodframes, aw.newphot
#close the input file
saltio.closefits(struct)
# Update the indata file if necessary
lc=saltio.openascii(fileout,'w')
for i in range(len(ratio)):
x['target']=tx[i]
x['comparison']=cx[i]
y['target']=ty[i]
y['comparison']=cy[i]
reltime=False
if gframe[i]:
st.writedataout(lc, id[i], time[i], x, y, tflux[i], terr[i], \
cflux[i], cerr[i], ratio[i], rerr[i], time[0], reltime)
saltio.closeascii(lc)
def saltbias(images,outimages,outpref,subover=True,trim=True,subbias=False,
masterbias='bias.fits', median=False, function='polynomial',
order=3, rej_lo=3, rej_hi=3, niter=10, plotover=False,
turbo=False, clobber=False, logfile='salt.log', verbose=True):
status = 0
ifil = 0
ii = 0
mbiasdata = []
bstruct = ''
biasgn = ''
biassp = ''
biasbn = ''
biasin = ''
filetime = {}
biastime = {}
for i in range(1,7):
filetime[i] = []
biastime[i] = []
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
# are input and output lists the same length?
saltio.comparelists(infiles,outfiles,'Input','output')
# Does master bias frame exist?
# gain, speed, binning and instrument of master bias frame
if subbias:
if os.path.isfile(masterbias):
bstruct = saltio.openfits(masterbias)
else:
message = 'Master bias frame %s does not exist' % masterbias
raise SaltError(message)
else:
bstruct=None
# open each raw image file
for img, oimg in zip(infiles, outfiles):
#open the file
struct = saltio.openfits(img)
#check to see if it has already been bias subtracted
instrume,keyprep,keygain,keybias,keyxtalk,keyslot = saltkey.instrumid(struct)
# has file been biaseded already?
try:
key = struct[0].header[keybias]
message = 'File %s has already been de-biased ' % infile
raise SaltError(message)
except:
pass
#compare with the master bias to make sure they are the same
if subbias:
pass
#subtract the bias
struct=bias(struct,subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function,
order=order, rej_lo=rej_lo, rej_hi=rej_hi, niter=niter,
plotover=plotover, log=log, verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],keybias, 'Images have been de-biased', hist)
# write FITS file
saltio.writefits(struct,oimg, clobber=clobber)
saltio.closefits(struct)
def slotphot(images,outfile,srcfile,newfits=None,phottype='square',
subbacktype='median',sigback=3,mbin=7,sorder=3,niter=5,sigdet=5,
contpix=10,ampperccd=2,ignorexp=6,driftlimit=10.,finddrift=True,
outtype='ascii',reltime=True,clobber=True,logfile='salt.log',
verbose=True):
"""Perform photometry on listed SALT slotmode *images*."""
with logging(logfile,debug) as log:
# set up the variables
entries = []
vig_lo = {}
vig_hi = {}
amp = {}
x = {}
y = {}
x_o = {}
y_o = {}
r = {}
br1 = {}
br2 = {}
hour = 0
min = 0
sec = 0.
time0 = 0.
nframes = 0
bin=mbin
order=sorder
# is the input file specified?
saltsafeio.filedefined('Input',images)
# if the input file is a list, does it exist?
if images[0] == '@':
saltsafeio.listexists('Input',images)
# parse list of input files
infiles=saltsafeio.listparse('Raw image',images,'','','')
# check input files exist
saltsafeio.filesexist(infiles,'','r')
# is the output file specified?
saltsafeio.filedefined('Output',outfile)
# check output file does not exist, optionally remove it if it does exist
if os.path.exists(outfile) and clobber:
os.remove(outfile)
elif os.path.exists(outfile) and not clobber:
raise SaltIOError('File '+outfile+' already exists, use clobber=y')
# open output ascii file
if outtype=='ascii':
try:
lc = open(outfile,'a')
except:
raise SaltIOError('Cannot open ouput file '+outfile)
# is the extraction region defintion file specified?
saltsafeio.filedefined('Extraction region defintion',srcfile)
# check extraction region defintion file exists
srcfile = srcfile.strip()
saltsafeio.fileexists(srcfile)
# read extraction region defintion file
amp, x, y, x_o, y_o, r, br1, br2=slottool.readsrcfile(srcfile)
# set the writenewfits parameter
if not newfits or newfits=='none':
writenewfits=False
else:
writenewfits=newfits
# get time of first exposure and basic information about the observations
infile=infiles[0]
struct=saltsafeio.openfits(infile)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot=saltsafekey.instrumid(struct,infile)
# how many extensions?
nextend=saltsafekey.get('NEXTEND',struct[0],infile)
if nextend < amp['comparison']:
msg='Insufficient number of extensions in %s' % (infile)
raise SaltIOError(msg)
# how many amplifiers?
amplifiers=saltsafekey.get('NCCDS',struct[0],infile)
amplifiers = int(ampperccd*float(amplifiers))
if ampperccd>0:
nframes = int(nextend/amplifiers)
nstep=amplifiers
else:
nframes = nextend
nstep=1
ntotal=nframes*len(infiles)
# image size
naxis1=saltsafekey.get('NAXIS1',struct[amp['comparison']],infile)
naxis2=saltsafekey.get('NAXIS2',struct[amp['comparison']],infile)
# CCD binning
ccdsum=saltsafekey.get('CCDSUM',struct[0],infile)
binx=int(ccdsum.split(' ')[0])
biny=int(ccdsum.split(' ')[1])
# Identify the time of the observations
ext = 1
try:
time0=slottool.getobstime(struct[ext], infile+'['+str(ext)+']')
dateobs=saltsafekey.get('DATE-OBS',struct[ext],infile)
dateobs=dateobs.replace('-','/')
except:
raise SaltIOError('No time or obsdate in first image')
# If a total file is to be written out, create it and update it
if writenewfits:
if os.path.isfile(writenewfits):
if clobber:
saltsafeio.delete(writenewfits)
else:
raise SaltIOError('Newfits file exists, use clobber')
try:
hdu=pyfits.PrimaryHDU()
hdu.header=struct[0].header
hdu.header['NCCDS']=1
hdu.header['NSCIEXT']=ntotal-ignorexp
hdu.header['NEXTEND']=ntotal-ignorexp
hduList=pyfits.HDUList(hdu)
hduList.verify()
hduList.writeto(writenewfits)
except:
raise SaltIOError('Could not create newfits file, '+writenewfits)
# Close image file
saltsafeio.closefits(struct)
# Read newfits file back in for updating
if writenewfits:
try:
hduList=pyfits.open(writenewfits,mode='update')
except:
raise SaltIOError('Cannot open newfits file '+writenewfits+' for updating.')
# set up the arrays
j=0
time=np.zeros(ntotal ,dtype='float')-1.0
dx=np.zeros(ntotal ,dtype='float')
dy=np.zeros(ntotal ,dtype='float')
tflux=np.zeros(ntotal ,dtype='float')
terr =np.zeros(ntotal ,dtype='float')
cflux=np.zeros(ntotal ,dtype='float')
cerr =np.zeros(ntotal ,dtype='float')
ratio=np.zeros(ntotal ,dtype='float')
rerr =np.zeros(ntotal ,dtype='float')
tgt_x=np.zeros(ntotal ,dtype='float')
tgt_y=np.zeros(ntotal ,dtype='float')
cmp_x=np.zeros(ntotal ,dtype='float')
cmp_y=np.zeros(ntotal ,dtype='float')
p_one=100./ntotal # One percent
p_old=-1 # Previous completed percentage
p_new=0 # New completed percentage
p_n=1 # Counter number
for infile in infiles:
# Log
if verbose:
log.message('Starting photometry on file '+infile, with_stdout=False)
struct=pyfits.open(infile)
# Skip through the frames and process each frame individually
for i in range(nframes):
# Show progress
if verbose:
p_new=int(p_n*p_one)
p_n+=1
if p_new!=p_old:
ctext='Percentage Complete: %d\r' % p_new
sys.stdout.write(ctext)
sys.stdout.flush()
p_old=p_new
if not (infile==infiles[0] and i < ignorexp):
ext=amp['comparison']+i*nstep
try:
header=struct[ext].header
array=struct[ext].data
array=array*1.0
except:
msg='Unable to open extension %i in image %s' % (ext, infile)
raise SaltIOError(msg)
# starti the analysis of each frame
# get the time
time[j]=slottool.getobstime(struct[ext],infile+'['+str(ext)+']')
# gain and readout noise
try:
gain=float(header['GAIN'])
except:
gain=1
raise SaltIOError('Gain not specified in image header')
try:
rdnoise=float(header['RDNOISE'])
except:
rdnoise=0
raise SaltIOError('RDNOISE not specified in image header')
# background subtraction
if not subbacktype=='none':
try:
array=subbackground(array, sigback, bin, order, niter, subbacktype)
except SaltError:
log.warning('Image '+infile+' extention '+str(ext)+' is blank, skipping')
continue
# x-y fit to the comparison star and update the x,y values
if finddrift:
carray, fx,fy=slottool.finddrift(array, x['comparison'], y['comparison'], r['comparison'], naxis1, naxis2, sigdet, contpix, sigback, driftlimit, niter)
if fx > -1 and fy > -1:
if fx < naxis1 and fy < naxis2:
dx[j]=x['comparison']-fx
dy[j]=y['comparison']-fy
x['comparison']=fx
y['comparison']=fy
x['target']=x['target']-dx[j]
y['target']=y['target']-dy[j]
else:
dx[j]=0
dy[j]=0
x['comparison']=x_o['comparison']
y['comparison']=y_o['comparison']
x['target']=x_o['target']
y['target']=y_o['target']
else:
msg='No comparison object found in image file ' + infile+' on extension %i skipping.' % ext
log.warning(msg)
pass
# do photometry
try:
tflux[j],terr[j],cflux[j],cerr[j],ratio[j],rerr[j]=slottool.dophot(phottype, array, x, y, r, br1, br2, gain, rdnoise, naxis1, naxis2)
except SaltError, e:
msg='Could not do photometry on extension %i in image %s because %s skipping.' % (ext, infile, e)
log.warning(msg)
tgt_x[j]=x['target']
tgt_y[j]=y['target']
cmp_x[j]=x['comparison']
cmp_y[j]=y['comparison']
# record results
# TODO! This should be removed in favor of the write all to disk in the end
if outtype=='ascii':
slottool.writedataout(lc, j+1, time[j], x, y, tflux[j], terr[j], cflux[j],cerr[j],ratio[j],rerr[j],time0, reltime)
# write newfits file
if writenewfits:
# add original name and extension number to header
try:
hdue=pyfits.ImageHDU(array)
hdue.header=header
hdue.header.update('ONAME',infile,'Original image name')
hdue.header.update('OEXT',ext,'Original extension number')
hduList.append(hdue)
except:
log.warning('Could not update image in newfits '+infile+' '+str(ext))
# increment counter
j+=1
# close FITS file
saltsafeio.closefits(struct)
# close newfits file
if writenewfits:
try:
hduList.flush()
hduList.close()
except:
raise SaltIOError('Cannot close newfits file.')
# write to output
if outtype=='ascii':
# close output ascii file
try:
lc.close()
except:
raise SaltIOError('Cannot close ouput file ' + outfile)
elif outtype=='fits':
print 'writing fits'
try:
c1=pyfits.Column(name='index',format='D',array=np.arange(ntotal))
if reltime:
c2=pyfits.Column(name='time',format='D',array=time-time0)
else:
c2=pyfits.Column(name='time',format='D',array=time)
c3=pyfits.Column(name='tgt_x',format='D',array=tgt_x)
c4=pyfits.Column(name='tgt_y',format='D',array=tgt_y)
c5=pyfits.Column(name='tgt_flux',format='D',array=tflux)
c6=pyfits.Column(name='tgt_err',format='D',array=terr)
c7=pyfits.Column(name='cmp_x',format='D',array=cmp_x)
c8=pyfits.Column(name='cmp_y',format='D',array=cmp_y)
c9=pyfits.Column(name='cmp_flux',format='D',array=cflux)
c10=pyfits.Column(name='cmp_err',format='D',array=cerr)
c11=pyfits.Column(name='flux_ratio',format='D',array=ratio)
c12=pyfits.Column(name='flux_ratio_err',format='D',array=rerr)
tbhdu=pyfits.new_table([c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12])
# Add header information
tbhdu.header.update('RELTIME',str(reltime),'Time relative to first datapoint or absolute.')
tbhdu.writeto(outfile)
print 'fits written to ',outfile
except:
raise SaltIOError('Could not write to fits table.')
