def readsrcfile(srcfile):
"""Read in the src file and return the parameters extracted from that file
"""
#set up some variables
amp = {}
x = {}
y = {}
x_o = {}
y_o = {}
r = {}
br1 = {}
br2 = {}
tgt_btype = 'region'
cmp_btype = 'region'
# check extraction region defintion file exists
srcfile = srcfile.strip()
saltio.fileexists(srcfile)
#try loading the region file
try:
region = np.round(np.loadtxt(srcfile))
except Exception, e:
msg = 'SLOTTOOLs--ERROR: Unable to load array from %s' % srcfile
raise SaltIOError(msg)
def readsrcfile(srcfile):
"""Read in the src file and return the parameters extracted from that file
"""
#set up some variables
amp = {}
x = {}
y = {}
x_o = {}
y_o = {}
r = {}
br1 = {}
br2 = {}
tgt_btype='region'
cmp_btype='region'
# check extraction region defintion file exists
srcfile = srcfile.strip()
saltio.fileexists(srcfile)
#try loading the region file
try:
region=np.round(np.loadtxt(srcfile))
except Exception, e:
msg='SLOTTOOLs--ERROR: Unable to load array from %s' % srcfile
raise SaltIOError(msg)
def saltfpevelocity(infile, existfit, xc, yc, rmax, logfile, verbose):
"""XXX"""
# getting paths for filenames
pathin = os.path.dirname(infile)
basein = os.path.basename(infile)
pathlog = os.path.dirname(logfile)
baselog = os.path.basename(logfile)
print pathin, basein
# start log now that all parameters are set up
with logging(logfile, debug) as log:
# Some basic checks, some tests are done in the FORTRAN code itself
# is the input file specified?
saltsafeio.filedefined('Input', infile)
# if the input file is a file, does it exist?
if basein[0] != '@':
saltsafeio.fileexists(infile)
# if a list is input throw an error
if basein[0] == '@':
raise SaltIOError(basein + ' list input instead of a file')
# The file contains a list of images, check they exist
# format is 4x dummy lines, then a line/lines containing: x,y,wave0,sn,norm,dnorm,filename
testfile = open(infile)
print testfile
for i in range(1, 5):
line = testfile.readline()
line = 'dummy'
infiles = []
while (len(line.strip()) != 0):
line = testfile.readline()
if (len(line.strip()) > 0):
linearray = line.split()
infiles.append(linearray[6])
# check input files exist
saltsafeio.filesexist(infiles, '', 'r')
# convert existfile to a one letter string
if existfit == 'yes':
existfit == 'y'
else:
existfit == 'n'
# If all looks OK, run the FORTRAN code
print infile, 'input filename'
print existfit, xc, yc, rmax
evelocity_wrapper2.evelocity2(infile, existfit, xc, yc, rmax)
def saltfpeprofile(plottype, infile, logfile, verbose):
"""Line profile fitting and display"""
plottype = str(plottype)
plottype = plottype.strip()
if plottype == 'xwindow':
plottype = '/xw'
else:
plottype = '/ps'
# getting paths for filenames
pathin = os.path.dirname(infile)
basein = os.path.basename(infile)
pathlog = os.path.dirname(logfile)
baselog = os.path.basename(logfile)
print pathin, basein
# start log now that all parameters are set up
with logging(logfile, debug) as log:
# Some basic checks, some tests are done in the FORTRAN code itself
# is the input file specified?
saltsafeio.filedefined('Input', infile)
# if the input file is a file, does it exist?
if basein[0] != '@':
saltsafeio.fileexists(infile)
# if a list is input throw an error
if basein[0] == '@':
raise SaltIOError(basein + ' list input instead of a file')
# The file contains a list of images, check they exist
# format is 4x dummy lines, then a line/lines containing: x,y,wave0,sn,norm,dnorm,filename
testfile = open(infile)
print testfile
for i in range(1, 5):
line = testfile.readline()
line = 'dummy'
infiles = []
while (len(line.strip()) != 0):
line = testfile.readline()
if (len(line.strip()) > 0):
linearray = line.split()
infiles.append(linearray[6])
# check input files exist
saltsafeio.filesexist(infiles, '', 'r')
# If all looks OK, run the FORTRAN code
print 'input filename = ', infile
eprofile_wrapper.eprofile(plottype, infile)
def saltfpcalibrate(plottype, infile, outfile, calibratelogfile, logfile,
verbose):
"""Determines the wavelength calibration of the Fabry-Perot
interactively, using the ring measurements from the CALRING program."""
plottype = str(plottype)
plottype = plottype.strip()
if plottype == 'xwindow':
plottype = '/xw'
else:
plottype = '/ps'
# start log now that all parameter are set up
with logging(logfile, debug) as log:
# Some basic checks, many tests are done in the FORTRAN code itself
# is the input file specified?
saltsafeio.filedefined('Input', infile)
# The input file in this case is a file, does it exist?
if infile[0] != '@':
saltsafeio.fileexists(infile)
# is the proposed output file specified?
saltsafeio.filedefined('Output', outfile)
# check whether the proposed output file exists
path = os.path.dirname(infile)
if len(path) > 0:
dir = path + '/'
else:
dir = './'
print dir, ' directory with data'
outfile = outfile.strip()
if os.path.isfile(outfile):
print 'output file exists, appending'
# saltsafeio.delete(outfile)
# check whether the calibrate logfile is defined
saltsafeio.filedefined('Log', calibratelogfile)
# Get current working directory as the Fortran code changes dir
startdir = os.getcwd()
# If all looks OK, run the FORTRAN code
calibrate_wrapper.calibrate(plottype, infile, outfile,
calibratelogfile)
# go back to starting directory
os.chdir(startdir)
def saltfpcalibrate(plottype,infile,outfile,calibratelogfile,logfile,verbose):
"""Determines the wavelength calibration of the Fabry-Perot
interactively, using the ring measurements from the CALRING program."""
plottype = str(plottype)
plottype = plottype.strip()
if plottype == 'xwindow':
plottype = '/xw'
else:
plottype = '/ps'
# start log now that all parameter are set up
with logging(logfile, debug) as log:
# Some basic checks, many tests are done in the FORTRAN code itself
# is the input file specified?
saltsafeio.filedefined('Input',infile)
# The input file in this case is a file, does it exist?
if infile[0] != '@':
saltsafeio.fileexists(infile)
# is the proposed output file specified?
saltsafeio.filedefined('Output',outfile)
# check whether the proposed output file exists
path = os.path.dirname(infile)
if len(path) > 0:
dir = path + '/'
else:
dir = './'
print dir, ' directory with data'
outfile = outfile.strip()
if os.path.isfile(outfile):
print 'output file exists, appending'
# saltsafeio.delete(outfile)
# check whether the calibrate logfile is defined
saltsafeio.filedefined('Log',calibratelogfile)
# Get current working directory as the Fortran code changes dir
startdir = os.getcwd()
# If all looks OK, run the FORTRAN code
calibrate_wrapper.calibrate(plottype,infile, outfile, calibratelogfile)
# go back to starting directory
os.chdir(startdir)
def readlcfile(lcfile):
"""Read in the lightcurve file
"""
saltio.fileexists(lcfile)
try:
id, time, ratio, rerr, tx, ty, tflux, terr, cx, cy, cflux, cerr=np.loadtxt(lcfile, unpack=True)
except Exception, e:
msg='Error processing light curver file because %s' % e
raise SaltIOError(msg)
def readlcfile(lcfile):
"""Read in the lightcurve file
"""
saltio.fileexists(lcfile)
try:
id, time, ratio, rerr, tx, ty, tflux, terr, cx, cy, cflux, cerr = np.loadtxt(
lcfile, unpack=True)
except Exception, e:
msg = 'Error processing light curver file because %s' % e
raise SaltIOError(msg)
def saltmosaic(images,outimages,outpref,geomfile,interp='linear',geotran=True, cleanup=True,clobber=False,logfile=None,verbose=True):
#Start the logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# does CCD geometry definition file exist
geomfilefile = geomfile.strip()
saltio.fileexists(geomfile)
gap = 0
xshift = [0, 0]
yshift = [0, 0]
rotation = [0, 0]
gap, xshift, yshift, rotation=saltio.readccdgeom(geomfile)
# open each raw image file and apply the transformation to it
for img, oimg in zip(infiles, outfiles):
#open the structure
struct = saltio.openfits(img)
#create the mosaic
ostruct=make_mosaic(struct, gap, xshift, yshift, rotation, interp_type=interp, geotran=geotran, cleanup=cleanup, log=log, verbose=verbose)
#update the header information
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(ostruct[0], 'SMOSAIC', 'Images have been mosaicked ', hist)
#write the image out
saltio.writefits(ostruct,oimg, clobber=clobber)
#close the files
saltio.closefits(struct)
saltio.closefits(ostruct)
def saltemail(propcode, obsdate,readme,server='mail.saao.ac.za',username='', password='',
bcc='',sdbhost='sdb.saao',sdbname='sdb',sdbuser='******',
logfile='salt.log',verbose=True):
"""For a given date, look into the database for observations made on that
date. If propcode='all' send an email to all of the PIs that had data taken on that date.
If propcode gives a specific proposal, only send emails to that PI.
"""
# set up
sender = '*****@*****.**'
bcclist = bcc.split(',')
date_obss = []
propids = []
pis = []
pids = []
email = []
with logging(logfile,debug) as log:
#set up the proposal list
pids = saltio.argunpack('propcode',propcode)
#check to see if the argument is defined
saltio.argdefined('obsdate',obsdate)
# check readme file exists
saltio.fileexists(readme)
#connect to the email server
try:
smtp = smtplib.SMTP()
smtp.connect(server)
smtp.ehlo()
smtp.starttls()
smtp.ehlo()
except Exception, e:
message = 'Cannot connect to %s because %s' % (server, e)
log.error(message)
try:
smtp.login(username,password)
except Exception, e:
message = 'Cannot login to %s as %s because %s' % (server, username, e)
log.error(message)
def read_slits_from_fits(simg):
"""Read the slit definitions from a FITS fiel where the slit
definitions have been stored in a table in the FITS file under an
extension with the name
"""
# first check if the file exists
saltio.fileexists(simg)
# open the slit image
struct = saltio.openfits(simg)
# get the extension of the slit table
slitext = saltkey.get('SLITEXT', struct[0])
# extract the order of the fit and the positions of each of the slits
order, slit_positions = mt.read_slits_HDUtable(struct[slitext])
return order, slit_positions
def read_slits_from_fits(simg):
"""Read the slit definitions from a FITS fiel where the slit
definitions have been stored in a table in the FITS file under an
extension with the name
"""
# first check if the file exists
saltio.fileexists(simg)
# open the slit image
struct = saltio.openfits(simg)
# get the extension of the slit table
slitext = saltkey.get('SLITEXT', struct[0])
# extract the order of the fit and the positions of each of the slits
order, slit_positions = mt.read_slits_HDUtable(struct[slitext])
return order, slit_positions
def checkfordata(rawpath, prefix, obsdate, log):
"""Check to see if the data have downloaded correctly"""
lastnum=1
saltio.fileexists(rawpath+'disk.file')
content = saltio.openascii(rawpath+'disk.file','r')
for line in content:
lastnum = saltstring.filenumber(line)
saltio.closeascii(content)
lastfile = saltstring.filename(prefix,obsdate,lastnum-1)
if lastnum==1: return lastnum
#check to see that the data are present
if (os.path.isfile(rawpath+lastfile) or os.path.isfile(rawpath+lastfile.replace('fits','bin'))):
message = 'Data download complete for %s\n' % rawpath
log.message(message)
else:
message = 'Data download incomplete to %s' % rawpath
log.error(message)
return lastnum
def slotmerge(images, outimages, outpref, geomfile, clobber, logfile, verbose):
with logging(logfile, debug) as log:
# are the arguments defined
saltsafeio.argdefined('images', images)
saltsafeio.argdefined('geomfile', geomfile)
saltsafeio.argdefined('logfile', logfile)
# if the input file is a list, does it exist?
if images[0] == '@':
saltsafeio.listexists('Input', images)
# parse list of input files
infiles = saltsafeio.listparse('Raw image', images, '', '', '')
# check input files exist
saltsafeio.filesexist(infiles, '', 'r')
# load output name list: @list, * and comma separated
outimages = outimages.strip()
outpref = outpref.strip()
if len(outpref) == 0 and len(outimages) == 0:
raise SaltIOError('Output file(s) not specified')
# test output @filelist exists
if len(outimages) > 0 and outimages[0] == '@':
saltsafeio.listexists('Output', outimages)
# parse list of output files
outfiles = saltsafeio.listparse('Output image', outimages, outpref,
infiles, '')
# are input and output lists the same length?
saltsafeio.comparelists(infiles, outfiles, 'Input', 'output')
# do the output files already exist?
if not clobber:
saltsafeio.filesexist(outfiles, '', 'w')
# does CCD geometry definition file exist
geomfilefile = geomfile.strip()
saltsafeio.fileexists(geomfile)
# read geometry definition file
gap = 0
xshift = [0, 0]
yshift = [0, 0]
rotation = [0, 0]
gap, xshift, yshift, rotation = saltsafeio.readccdgeom(geomfile)
for ro in rotation:
if ro != 0:
log.warning('SLOTMERGE currently ignores CCD rotation')
# Begin processes each file
for infile, outfile in zip(infiles, outfiles):
# determine the name for the output file
outpath = outfile.rstrip(os.path.basename(outfile))
if (len(outpath) == 0):
outpath = '.'
# open each raw image
struct = saltsafeio.openfits(infile)
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = saltsafekey.instrumid(
struct, infile)
# how many amplifiers?
nccds = saltsafekey.get('NCCDS', struct[0], infile)
amplifiers = nccds * 2
#if (nccds != 2):
# raise SaltError('Can not currently handle more than two CCDs')
# CCD geometry coefficients
if instrume == 'RSS' or instrume == 'PFIS':
xsh = [xshift[0], 0., xshift[1]]
ysh = [yshift[0], 0., yshift[1]]
rot = [rotation[0], 0., rotation[1]]
refid = 1
if instrume == 'SALTICAM':
xsh = [xshift[0], 0.]
ysh = [yshift[0], 0.]
rot = [rotation[0], 0]
refid = 1
# how many extensions?
nextend = saltsafekey.get('NEXTEND', struct[0], infile)
# how many exposures
exposures = nextend / amplifiers
# CCD on-chip binning
xbin, ybin = saltsafekey.ccdbin(struct[0], infile)
gp = int(gap / xbin)
# create output hdu structure
outstruct = [None] * int(exposures + 1)
outstruct[0] = struct[0]
# iterate over exposures, stitch them to produce file of CCD images
for i in range(exposures):
# Determine the total size of the image
xsize = 0
ysize = 0
for j in range(amplifiers):
hdu = i * amplifiers + j + 1
try:
xsize += len(struct[hdu].data[0])
if ysize < len(struct[hdu].data):
ysize = len(struct[hdu].data)
except:
msg = 'Unable to access extension %i ' % hdu
raise SaltIOError(msg)
xsize += gp * (nccds - 1)
maxxsh, minxsh = determineshifts(xsh)
maxysh, minysh = determineshifts(ysh)
xsize += (maxxsh - minxsh)
ysize += (maxysh - minysh)
# Determine the x and y origins for each frame
xdist = 0
ydist = 0
shid = 0
x0 = np.zeros(amplifiers)
y0 = np.zeros(amplifiers)
for j in range(amplifiers):
x0[j] = xdist + xsh[shid] - minxsh
y0[j] = ysh[shid] - minysh
hdu = i * amplifiers + j + 1
darr = struct[hdu].data
xdist += len(darr[0])
if j % 2 == 1:
xdist += gp
shid += 1
# make the out image
outarr = np.zeros((ysize, xsize), np.float64)
# Embed each frame into the output array
for j in range(amplifiers):
hdu = i * amplifiers + j + 1
darr = struct[hdu].data
outarr = salttran.embed(darr, x0[j], y0[j], outarr)
# Add the outimage to the output structure
hdu = i * amplifiers + 1
outhdu = i + 1
outstruct[outhdu] = pyfits.ImageHDU(outarr)
outstruct[outhdu].header = struct[hdu].header
# Fix the headers in each extension
datasec = '[1:%4i,1:%4i]' % (xsize, ysize)
saltsafekey.put('DATASEC', datasec, outstruct[outhdu], outfile)
saltsafekey.rem('DETSIZE', outstruct[outhdu], outfile)
saltsafekey.rem('DETSEC', outstruct[outhdu], outfile)
saltsafekey.rem('CCDSEC', outstruct[outhdu], outfile)
saltsafekey.rem('AMPSEC', outstruct[outhdu], outfile)
# add housekeeping key words
outstruct[outhdu] = addhousekeeping(outstruct[outhdu], outhdu,
outfile)
# close input FITS file
saltsafeio.closefits(struct)
# housekeeping keywords
keymosaic = 'SLOTMERG'
fname, hist = history(level=1, wrap=False)
saltsafekey.housekeeping(struct[0], keymosaic,
'Amplifiers have been mosaiced', hist)
#saltsafekey.history(outstruct[0],hist)
# this is added for later use by
saltsafekey.put('NCCDS', 0.5, outstruct[0])
saltsafekey.put('NSCIEXT', exposures, outstruct[0])
saltsafekey.put('NEXTEND', exposures, outstruct[0])
# write FITS file of mosaiced image
outstruct = pyfits.HDUList(outstruct)
saltsafeio.writefits(outstruct, outfile, clobber=clobber)
def saltfpringfilter(axc,ayc,arad,rxc,ryc,filterfreq,filterwidth,itmax,conv, fitwidth,image,logfile,useconfig,configfile,verbose):
""" Determines the center coordinates of a ring, bins the ring radially and computes its power spectrum, and allows the user to select a smoothing filter for the ring. """
# default parameter values are set up in the pyraf .par file. The values used are then changed if a FORTRAN config file exists and the user elects to override the pyraf .par file.
# Is the input FORTRAN config file specified?
# If it is blank, then it will be ignored.
if useconfig:
configfile = configfile.strip()
if len(configfile) > 0:
#check exists
saltsafeio.fileexists(configfile)
# read updated parameters from the file
array=getpfp(configfile,"axc")
s=len(array)
flag = array[s-1]
if flag == 1:
axc=float(array[0])
array=getpfp(configfile,"ayc")
s=len(array)
flag = array[s-1]
if flag == 1:
ayc=float(array[0])
array=getpfp(configfile,"arad")
s=len(array)
flag = array[s-1]
if flag == 1:
arad=float(array[0])
array=getpfp(configfile,"rxc")
s=len(array)
flag = array[s-1]
if flag == 1:
rxc=float(array[0])
array=getpfp(configfile,"ryc")
s=len(array)
flag = array[s-1]
if flag == 1:
ryc=float(array[0])
array=getpfp(configfile,"calring_filter_width")
s=len(array)
flag = array[s-1]
if flag == 1:
filterwidth=int(array[0])
array=getpfp(configfile,"calring_filter_freq")
s=len(array)
flag = array[s-1]
if flag == 1:
filterfreq=int(array[0])
array=getpfp(configfile,"calring_itmax")
s=len(array)
flag = array[s-1]
if flag == 1:
itmax=int(array[0])
array=getpfp(configfile,"calring_conv")
s=len(array)
flag = array[s-1]
if flag == 1:
conv=float(array[0])
array=getpfp(configfile,"calring_fitwidth")
s=len(array)
flag = array[s-1]
if flag == 1:
fitwidth=float(array[0])
# getting paths for filenames
pathin = os.path.dirname(image)
basein = os.path.basename(image)
pathlog = os.path.dirname(logfile)
baselog = os.path.basename(logfile)
# forcing logfiles to be created in the same directory as the input data
# (we change to this directory once starting the fortran code)
if len(pathin) > 0:
logfile = baselog
# start log now that all parameter are set up
with logging(logfile, debug) as log:
# Some basic checks, many tests are done in the FORTRAN code itself
# is the input file specified?
saltsafeio.filedefined('Input',image)
# if the input file is a file, does it exist?
if basein[0] != '@':
saltsafeio.fileexists(image)
infile = image
# if the input file is a list, throw an error
if basein[0] == '@':
raise SaltIOError(basein + ' list input instead of a file' )
# optionally update the FORTRAN config file with new values - not implemented currently
# If all looks OK, run the FORTRAN code
if len(pathin) > 0:
dir = pathin
else:
dir = './'
infile = basein
print(dir, infile, 'input directory and input file')
# Get current working directory as the Fortran code changes dir
startdir = os.getcwd()
ringfilter_wrapper.ringfilter(dir,axc, ayc,arad, rxc,ryc,filterfreq,filterwidth,itmax,conv,fitwidth,infile)
# go back to starting directory
os.chdir(startdir)
def salteditkey(images,outimages,outpref, keyfile, recfile=None,clobber=False,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
#is key file defined
saltio.argdefined('keyfile',keyfile)
keyfile = keyfile.strip()
saltio.fileexists(keyfile)
# if the data are the same, set up to use update instead of write
openmode='copyonwrite'
if (infiles!=outfiles): openmode='copyonwrite'
# determine the date of the observations
obsdate=saltstring.makeobsdatestr(infiles, 1,9)
if len(obsdate)!=8:
message = 'Either FITS files from multiple dates exist, '
message += 'or raw FITS files exist with non-standard names.'
log.warning(message)
# FITS file columns to record keyword changes
fitcol = []
keycol = []
oldcol = []
newcol = []
# Set up the rules to change the files
keyedits=readkeyfile(keyfile, log=log, verbose=verbose)
#now step through the images
for img, oimg in zip(infiles, outfiles):
#determine the appropriate keyword edits for the image
klist=[]
for frange in keyedits:
if checkfitsfile(img, frange, keyedits[frange]):
klist.append(keyedits[frange][3])
if klist:
#open up the new files
struct = saltio.openfits(img,mode=openmode)
struct.verify('fix')
for kdict in klist:
for keyword in kdict:
#record the changes
value=kdict[keyword]
fitcol.append(img)
keycol.append(keyword)
newcol.append(value)
try:
oldcol.append(struct[0].header[keyword].lstrip())
except:
oldcol.append('None')
#update the keyword
if saltkey.found(keyword, struct[0]):
try:
saltkey.put(keyword,value,struct[0])
message='\tUpdating %s in %s to %s' % (keyword, os.path.basename(img), value)
log.message(message, with_header=False, with_stdout=verbose)
except Exception, e:
message = 'Could not update %s in %s because %s' % (keyword, img, str(e))
raise SaltError(message)
else:
try:
saltkey.new(keyword.strip(),value,'Added Comment',struct[0])
message='\tAdding %s in %s to %s' % (keyword, os.path.basename(img), value)
log.message(message, with_header=False, with_stdout=verbose)
except Exception,e :
message = 'Could not update %s in %s because %s' % (keyword, img, str(e))
raise SaltError(message)
#updat the history keywords
#fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
#saltkey.housekeeping(struct[0],'SAL-EDT', 'Keywords updated by SALTEDITKEY', hist)
#write the file out
if openmode=='update':
saltio.updatefits(struct)
message = 'Updated file ' + os.path.basename(oimg)
else:
saltio.writefits(struct, oimg, clobber)
message = 'Created file ' + os.path.basename(oimg)
log.message(message, with_header=False, with_stdout=True)
struct.close()
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='',
clobber=False, cleanup=True, logfile='salt.log', verbose=True):
"""SALTADVANCE provides advanced data reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
infiles.sort()
# create list of output files
outpath=saltio.abspath(outpath)
#log into the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
obsmode=obsstruct.data.field('OBSMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
propcode=obsstruct.data.field('PROPID')
masktype=obsstruct.data.field('MASKTYP')
#set the bias list of objects
biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')]
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=fits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=fits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the arc data
arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')]
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if nimg in arclist:
#open the image
struct=fits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
obsdate=os.path.basename(img)[1:9]
#print the message
if log:
message='Processing ARC frame %s' % img
log.message(message, with_stdout=verbose)
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
log=log, verbose=verbose)
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the images
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#measure the arcdata
arcimage=outpath+'mbxgp'+nimg
dbfile=outpath+obsdate+'_specid.db'
lamp = obsstruct.data.field('LAMPID')[i]
lamp = lamp.replace(' ', '')
lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
print arcimage, lampfile, os.getcwd()
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod='Matchlines', function='legendre',
order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3,
startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile,
verbose=verbose)
try:
ximg = outpath+'xmbxgp'+os.path.basename(arcimage)
specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except:
pass
#process the science data
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if not (nimg in flatlist or nimg in biaslist or nimg in arclist):
#open the image
struct=fits.open(img)
if struct[0].header['PROPID'].count('CAL_GAIN'): continue
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#Check to see if it is RSS 2x2 and add bias subtraction
instrume=saltkey.get('INSTRUME', struct[0]).strip()
gainset = saltkey.get('GAINSET', struct[0])
rospeed = saltkey.get('ROSPEED', struct[0])
target = saltkey.get('OBJECT', struct[0]).strip()
exptime = saltkey.get('EXPTIME', struct[0])
obsmode = saltkey.get('OBSMODE', struct[0]).strip()
detmode = saltkey.get('DETMODE', struct[0]).strip()
masktype = saltkey.get('MASKTYP', struct[0]).strip()
xbin, ybin = saltkey.ccdbin( struct[0], img)
obsdate=os.path.basename(img)[1:9]
bstruct=None
crtype=None
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=5
subbias=False
if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW':
bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin)
if os.path.exists(bfile):
bstruct=fits.open(bfile)
subbias=True
if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5:
crtype='edge'
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=3
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \
bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise,
bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter,
log=log, verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#if the file is spectroscopic mode, apply the wavelength correction
if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT':
dbfile=outpath+obsdate+'_specid.db'
try:
ximg = outpath+'xmbxgp'+os.path.basename(img)
specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except Exception, e:
log.message('%s' % e)
#clean up the results
if cleanup:
#clean up the bias frames
for i in masterbias_dict.keys():
blist=masterbias_dict[i][1:]
for b in blist: saltio.delete(b)
#clean up the flat frames
for i in masterflat_dict.keys():
flist=masterflat_dict[i][1:]
for f in flist: saltio.delete(f)
def slotmerge(images,outimages,outpref,geomfile,clobber,logfile,verbose):
with logging(logfile,debug) as log:
# are the arguments defined
saltsafeio.argdefined('images',images)
saltsafeio.argdefined('geomfile',geomfile)
saltsafeio.argdefined('logfile',logfile)
# if the input file is a list, does it exist?
if images[0] == '@':
saltsafeio.listexists('Input',images)
# parse list of input files
infiles=saltsafeio.listparse('Raw image',images,'','','')
# check input files exist
saltsafeio.filesexist(infiles,'','r')
# load output name list: @list, * and comma separated
outimages = outimages.strip()
outpref = outpref.strip()
if len(outpref) == 0 and len(outimages) == 0:
raise SaltIOError('Output file(s) not specified')
# test output @filelist exists
if len(outimages) > 0 and outimages[0] == '@':
saltsafeio.listexists('Output',outimages)
# parse list of output files
outfiles=saltsafeio.listparse('Output image',outimages,outpref,infiles,'')
# are input and output lists the same length?
saltsafeio.comparelists(infiles,outfiles,'Input','output')
# do the output files already exist?
if not clobber:
saltsafeio.filesexist(outfiles,'','w')
# does CCD geometry definition file exist
geomfilefile = geomfile.strip()
saltsafeio.fileexists(geomfile)
# read geometry definition file
gap = 0
xshift = [0, 0]
yshift = [0, 0]
rotation = [0, 0]
gap, xshift, yshift, rotation=saltsafeio.readccdgeom(geomfile)
for ro in rotation:
if ro!=0:
log.warning('SLOTMERGE currently ignores CCD rotation')
# Begin processes each file
for infile, outfile in zip(infiles, outfiles):
# determine the name for the output file
outpath = outfile.rstrip(os.path.basename(outfile))
if (len(outpath) == 0):
outpath = '.'
# open each raw image
struct=saltsafeio.openfits(infile)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot=saltsafekey.instrumid(struct,infile)
# how many amplifiers?
nccds=saltsafekey.get('NCCDS',struct[0],infile)
amplifiers = nccds * 2
#if (nccds != 2):
# raise SaltError('Can not currently handle more than two CCDs')
# CCD geometry coefficients
if instrume == 'RSS' or instrume == 'PFIS':
xsh = [xshift[0], 0., xshift[1]]
ysh = [yshift[0], 0., yshift[1]]
rot = [rotation[0], 0., rotation[1]]
refid = 1
if instrume == 'SALTICAM':
xsh = [xshift[0], 0.]
ysh = [yshift[0], 0.]
rot = [rotation[0], 0]
refid = 1
# how many extensions?
nextend=saltsafekey.get('NEXTEND',struct[0],infile)
# how many exposures
exposures = nextend/amplifiers
# CCD on-chip binning
xbin, ybin=saltsafekey.ccdbin(struct[0],infile)
gp = int(gap / xbin)
# create output hdu structure
outstruct = [None] * int(exposures+1)
outstruct[0]=struct[0]
# iterate over exposures, stitch them to produce file of CCD images
for i in range(exposures):
# Determine the total size of the image
xsize=0
ysize=0
for j in range(amplifiers):
hdu=i*amplifiers+j+1
try:
xsize += len(struct[hdu].data[0])
if ysize < len(struct[hdu].data):
ysize=len(struct[hdu].data)
except:
msg='Unable to access extension %i ' % hdu
raise SaltIOError(msg)
xsize += gp* (nccds-1)
maxxsh, minxsh = determineshifts(xsh)
maxysh, minysh = determineshifts(ysh)
xsize += (maxxsh-minxsh)
ysize += (maxysh-minysh)
# Determine the x and y origins for each frame
xdist=0
ydist=0
shid=0
x0=np.zeros(amplifiers)
y0=np.zeros(amplifiers)
for j in range(amplifiers):
x0[j]=xdist+xsh[shid]-minxsh
y0[j]=ysh[shid]-minysh
hdu=i*amplifiers+j+1
darr=struct[hdu].data
xdist += len(darr[0])
if j%2==1:
xdist += gp
shid += 1
# make the out image
outarr=np.zeros((ysize, xsize), np.float64)
# Embed each frame into the output array
for j in range(amplifiers):
hdu=i*amplifiers+j+1
darr=struct[hdu].data
outarr=salttran.embed(darr, x0[j], y0[j], outarr)
# Add the outimage to the output structure
hdu=i*amplifiers+1
outhdu=i+1
outstruct[outhdu] = pyfits.ImageHDU(outarr)
outstruct[outhdu].header=struct[hdu].header
# Fix the headers in each extension
datasec='[1:%4i,1:%4i]' % (xsize, ysize)
saltsafekey.put('DATASEC',datasec, outstruct[outhdu], outfile)
saltsafekey.rem('DETSIZE',outstruct[outhdu],outfile)
saltsafekey.rem('DETSEC',outstruct[outhdu],outfile)
saltsafekey.rem('CCDSEC',outstruct[outhdu],outfile)
saltsafekey.rem('AMPSEC',outstruct[outhdu],outfile)
# add housekeeping key words
outstruct[outhdu]=addhousekeeping(outstruct[outhdu], outhdu, outfile)
# close input FITS file
saltsafeio.closefits(struct)
# housekeeping keywords
keymosaic='SLOTMERG'
fname, hist=history(level=1, wrap=False)
saltsafekey.housekeeping(struct[0],keymosaic,'Amplifiers have been mosaiced', hist)
#saltsafekey.history(outstruct[0],hist)
# this is added for later use by
saltsafekey.put('NCCDS', 0.5, outstruct[0])
saltsafekey.put('NSCIEXT', exposures, outstruct[0])
saltsafekey.put('NEXTEND', exposures, outstruct[0])
# write FITS file of mosaiced image
outstruct=pyfits.HDUList(outstruct)
saltsafeio.writefits(outstruct, outfile, clobber=clobber)
def slotview(newfits,indata , fileout, srcfile, fps=10.0, phottype='square', sigdet=5, contpix=10, \
driftlimit=10, clobber=True,logfile='slotview.log',verbose=True):
#set up the variables
status = 0
entries = []
vig_lo = {}
vig_hi = {}
hour = 0
min = 0
sec = 0.
time0 = 0.
nframes = 0
sleep = 0
with logging(logfile, debug) as log:
#enter in the input data
saltio.fileexists(newfits)
#set the sleep parameter
if fps > 0: sleep = 1.0 / (fps)
# read in the data file
id, time, ratio, rerr, tx, ty, tflux, terr, cx, cy, cflux, cerr = st.readlcfile(
indata)
# read extraction region defintion file
amp, x, y, x_o, y_o, r, br1, br2 = st.readsrcfile(srcfile)
#determine the size of the data arrays
struct = saltio.openfits(newfits)
naxis1 = saltkey.get('NAXIS1', struct[1])
naxis2 = saltkey.get('NAXIS2', struct[1])
# Plot all of the data and the first image
# Create GUI
App = QtGui.QApplication([])
aw=SlotViewWindow(struct, id, tflux, cflux, ratio, time, phottype, sleep, \
tx, ty, cx, cy, r, br1, br2, naxis1, naxis2, sigdet, contpix, driftlimit)
aw.show()
# Start application event loop
app_exit = App.exec_()
# Check if GUI was executed succesfully
if app_exit != 0:
raise SALTError('InterIdentify GUI has unexpected exit status ' +
str(exit))
ratio, tflux, cflux, gframe, newphot = aw.ratio, aw.tflux, aw.cflux, aw.goodframes, aw.newphot
#close the input file
saltio.closefits(struct)
# Update the indata file if necessary
lc = saltio.openascii(fileout, 'w')
for i in range(len(ratio)):
x['target'] = tx[i]
x['comparison'] = cx[i]
y['target'] = ty[i]
y['comparison'] = cy[i]
reltime = False
if gframe[i]:
st.writedataout(lc, id[i], time[i], x, y, tflux[i], terr[i], \
cflux[i], cerr[i], ratio[i], rerr[i], time[0], reltime)
saltio.closeascii(lc)
def vid2fits(inhead, inbin,outfile, config):
"""
Convert bin files made during the video process to
regular fits files
Format python vid2fits.py inhead inbin outfits config
Returns
"""
#Check that the input files exists
saltio.fileexists(inhead)
saltio.fileexists(inbin)
saltio.fileexists(config)
#if output file exists, then delete
if os.path.isfile(outfile): saltio.delete(outfile)
#read in and process the config file
condict=fitsconfig(config)
#read in the header information
infits=saltio.openfits(inhead)
inheader = infits['Primary'].header
instrume=inheader['INSTRUME']
detswv=softwareversion(inheader['DETSWV'])
#create a new image and copy the header to it
try:
hdu = fits.PrimaryHDU()
hdu.header=inheader
hduList = fits.HDUList(hdu)
#hduList.verify()
hduList.writeto(outfile, output_verify='ignore')
except:
message = 'ERROR -- VID2FIT: Could not create new fits file'
raise SaltError(message)
#Now open up the file that you just made and update it from here on
hduList = fits.open(outfile, mode='update')
#open the binary file
bindata = saltio.openbinary(inbin,'rb')
#read in header information from binary file
#some constants that are needed for reading in the binary data
sizeofinteger=struct.calcsize('i')
sizeofunsignshort=struct.calcsize('H')
sizeofdouble=struct.calcsize('d')
sizeoffloat =struct.calcsize('f')
#read in the number of exposures, geometry of image (width and height) and number of amps
nframes= saltio.readbinary(bindata,sizeofinteger,"=i")
if detswv<=4.78 and instrume=='SALTICAM':
fwidth= saltio.readbinary(bindata,sizeofinteger,"=i")
fheight= saltio.readbinary(bindata,sizeofinteger, "=i")
elif (detswv>=7.01 and instrume=='SALTICAM') or (detswv>=4.37 and instrume=='RSS'):
fwidth= saltio.readbinary(bindata,sizeofunsignshort,"=H")
fheight= saltio.readbinary(bindata,sizeofunsignshort, "=H")
pbcols=saltio.readbinary(bindata,sizeofunsignshort, "=H")
pbrows=saltio.readbinary(bindata,sizeofunsignshort,"=H")
else:
message='VID2FITS--Detector Software version %s is not supported' % detswv
raise SaltError(message)
nelements=fwidth*fheight
namps=saltio.readbinary(bindata,sizeofinteger,"=i")
#read in the gain
gain = numpy.zeros(namps,dtype=float)
for i in range(namps):
gain[i]=saltio.readbinary(bindata,sizeofdouble,"=d")
#read in the rdnoise
rdnoise = numpy.zeros(namps, dtype=float)
for i in range(namps):
rdnoise[i]=saltio.readbinary(bindata,sizeofdouble,"=d")
#set the scale parameters
bzero=32768
bscale=1
otime=0
#start the loop to read in the data
for i in range(nframes):
#read in the start of the data,time
starttime= saltio.readbinary(bindata,sizeofdouble,"=d")
date_obs, time_obs= ascii_time(starttime)
#read in the exposure time
exptime= saltio.readbinary(bindata,sizeofdouble,"=d")
#read in the dead time in milliseconds
if (detswv>=7.01 and instrume=='SALTICAM') or (detswv>=4.37 and instrume=='RSS'):
deadtime= saltio.readbinary(bindata,sizeofinteger,"=i")
else:
deadtime=None
#read in the dead time
if (detswv>=7.01 and instrume=='SALTICAM') or (detswv>=4.37 and instrume=='RSS'):
framecnt= saltio.readbinary(bindata,sizeofinteger,"=i")
else:
framecnt=None
otime=starttime
#read in the data
shape = (fheight,fwidth)
imdata = numpy.fromfile(bindata,dtype=numpy.ushort,count=nelements)
imdata = imdata.reshape(shape)
#for each amplifier write it to the image
if namps > 0: awidth=fwidth/namps
for j in range(namps):
###create the new extension ###
#cut each image by the number of amplifiers
y1=j*awidth
y2=y1+awidth
data = imdata[:,y1:y2].astype(numpy.int16)
hdue = fits.ImageHDU(data)
hdue.scale('int16','',bzero=bzero)
#set the header values
datasec,detsec,ccdsec,ampsec,biassec= \
create_header_values(condict,hdu,j,fheight)
#fill in the header data
hdue = write_ext_header(hdue,outfile,hdu,time_obs,date_obs,bscale,bzero, \
exptime,gain[j],rdnoise[j],datasec,detsec,ccdsec,ampsec, \
biassec, deadtime=deadtime, framecnt=framecnt )
#append the extension to the image
hduList.append(hdue)
try:
hduList.flush()
hduList.close()
except Exception, e:
message = 'ERROR: VID2BIN -- Fail to convert %s due to %s' % (outfile, e)
raise SaltError(message)
def saltobsid(propcode,obslog,rawpath,prodpath,outpath,prefix='mbxgp', fprefix='bxgp',clobber=False,logfile='salt.log',verbose=True):
"""Split data into their different data directories
"""
with logging(logfile,debug) as log:
# are the arguments defined
pids = saltio.argunpack('propcode',propcode)
# check observation log file exists
obslog = obslog.strip()
saltio.fileexists(obslog)
#open the observing log
obstruct = saltio.openfits(obslog)
obstab = saltio.readtab(obstruct[1],obslog)
saltio.closefits(obstruct)
#read in the file information
filenames = saltstring.listfunc(obstab.field('filename'),'lstrip')
instrumes = saltstring.listfunc(obstab.field('instrume'),'lstrip')
proposers = saltstring.listfunc(obstab.field('proposer'),'clean')
propids = saltstring.listfunc(obstab.field('propid'),'clean')
ccdtypes = saltstring.listfunc(obstab.field('ccdtype'),'clean')
ccdsums = saltstring.listfunc(obstab.field('ccdsum'),'clean')
gainsets = saltstring.listfunc(obstab.field('gainset'),'clean')
rospeeds = saltstring.listfunc(obstab.field('rospeed'),'clean')
detmodes = saltstring.listfunc(obstab.field('detmode'),'clean')
filters = saltstring.listfunc(obstab.field('filter'),'clean')
gratings = saltstring.listfunc(obstab.field('grating'),'clean')
gr_angles = obstab.field('gr-angle')
ar_angles = obstab.field('ar-angle')
# Create the list of proposals
try:
pids=saltio.cleanpropcode(pids, propids)
except SaltIOError:
#throw a warning adn exit if not data needs to be filterd
log.warning('No data to filter\n', with_stdout=verbose)
return
# check paths exist, end with a "/" and convert them to absolute paths
rawpath = saltio.abspath(rawpath)
prodpath = saltio.abspath(prodpath)
outpath = saltio.abspath(outpath)
#create the symlink raw path
rawsplit=rawpath.strip().split('/')
symrawpath='../../%s/%s/' % (rawsplit[-3], rawsplit[-2])
prodsplit=prodpath.strip().split('/')
symprodpath='../../%s/%s/' % (prodsplit[-3], prodsplit[-2])
# create PI directories
for pid in pids:
saltio.createdir(outpath+pid)
saltio.createdir(outpath+pid+'/raw')
saltio.createdir(outpath+pid+'/product')
#copy the data that belongs to a pid into that directory
log.message('SALTOBSID -- filtering images to proposal directories\n', with_stdout=verbose)
#copy data for a given proposal to the raw and produce directories
for i in range(len(obstab)):
if os.path.exists(outpath+obstab[i]['propid']):
if obstab[i]['object'].upper() not in ['ZERO', 'BIAS']:
fname=obstab[i]['filename']
pdir=obstab[i]['propid']
detmode=obstab[i]['detmode']
linkfiles(fname, pdir,detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#look through the bias/flat/arc/standard data to see if there is any relavent data
log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose)
caldata=['ZERO', 'FLAT', 'ARC']
biasheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED']
flatheader_list=['DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
archeader_list=['OBSMODE', 'DETMODE', 'CCDSUM', 'GAINSET', 'ROSPEED', 'FILTER', 'GRATING', 'GR-ANGLE', 'AR-ANGLE']
calproplist=['CAL_SPST']
#Include bias frames
log.message('SALTOBSID -- filtering bias files to proposal directories\n', with_stdout=verbose)
for i in range(len(obstab)):
fname=obstab[i]['filename']
prop_list=[]
#if it is a zero, check to see what other data have the same settings
if obstab[i]['CCDTYPE'].strip().upper()=='ZERO' or obstab[i]['OBJECT'].strip().upper() in ['BIAS', 'ZERO']:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], biasheader_list):
prop_list.append(obstab[i]['PROPID'])
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
detmode=obstab[i]['detmode']
linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, fprefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#Include calibration frames
log.message('SALTOBSID -- filtering calibration files to proposal directories\n', with_stdout=verbose)
for i in range(len(obstab)):
fname=obstab[i]['filename']
prop_list=[]
#if it is a flat, check to see what other data have the same settings
#this is turned off
if obstab[i]['CCDTYPE'].strip().upper()=='FLAT' and False:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], flatheader_list):
prop_list.append(obstab[j]['PROPID'])
#if it is a arc, check to see what other data have the same settings
#this is turned off
if obstab[i]['CCDTYPE'].strip().upper()=='ARC' and False:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], archeader_list):
prop_list.append(obstab[j]['PROPID'])
#if it is a calibration standard, see what other data have the same settings
if obstab[i]['PROPID'].strip().upper() in calproplist:
for j in range(len(obstab)):
if comparefiles(obstab[i], obstab[j], flatheader_list):
prop_list.append(obstab[j]['PROPID'])
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
if pdir!=obstab[i]['propid']:
detmode=obstab[i]['detmode']
linkfiles(fname, pdir, detmode, symrawpath, symprodpath, outpath, prefix, fprefix, clobber)
message='Copying %s to %s' % (fname, pdir)
log.message(message, with_header=False, with_stdout=verbose)
#Include master (bias or flat) frames
log.message('SALTOBSID -- filtering master calibration files to proposal directories\n', with_stdout=verbose)
masterlist=glob.glob(prodpath+'*Bias*')+glob.glob(prodpath+'*Flat*')
for bimg in masterlist:
struct=pyfits.open(bimg)
bdict={}
prop_list=[]
for k in biasheader_list:
bdict[k]=saltkey.get(k, struct[0])
for i in range(len(obstab)):
if comparefiles(obstab[i], bdict, biasheader_list):
prop_list.append(obstab[i]['PROPID'])
struct.close()
#copy the files over to the directory
prop_list=saltio.removebadpids(set(prop_list))
for pdir in prop_list:
fname=os.path.basename(bimg)
infile = symprodpath+fname
link = outpath+pdir+'/product/'+fname
saltio.symlink(infile,link,clobber)
message='Copying %s to %s' % (fname ,pdir)
log.message(message, with_header=False, with_stdout=verbose)
def saltfpringfilter(axc, ayc, arad, rxc, ryc, filterfreq, filterwidth, itmax,
conv, fitwidth, image, logfile, useconfig, configfile,
verbose):
""" Determines the center coordinates of a ring, bins the ring radially and computes its power spectrum, and allows the user to select a smoothing filter for the ring. """
# default parameter values are set up in the pyraf .par file. The values used are then changed if a FORTRAN config file exists and the user elects to override the pyraf .par file.
# Is the input FORTRAN config file specified?
# If it is blank, then it will be ignored.
if useconfig:
configfile = configfile.strip()
if len(configfile) > 0:
#check exists
saltsafeio.fileexists(configfile)
# read updated parameters from the file
array = getpfp(configfile, "axc")
s = len(array)
flag = array[s - 1]
if flag == 1:
axc = float(array[0])
array = getpfp(configfile, "ayc")
s = len(array)
flag = array[s - 1]
if flag == 1:
ayc = float(array[0])
array = getpfp(configfile, "arad")
s = len(array)
flag = array[s - 1]
if flag == 1:
arad = float(array[0])
array = getpfp(configfile, "rxc")
s = len(array)
flag = array[s - 1]
if flag == 1:
rxc = float(array[0])
array = getpfp(configfile, "ryc")
s = len(array)
flag = array[s - 1]
if flag == 1:
ryc = float(array[0])
array = getpfp(configfile, "calring_filter_width")
s = len(array)
flag = array[s - 1]
if flag == 1:
filterwidth = int(array[0])
array = getpfp(configfile, "calring_filter_freq")
s = len(array)
flag = array[s - 1]
if flag == 1:
filterfreq = int(array[0])
array = getpfp(configfile, "calring_itmax")
s = len(array)
flag = array[s - 1]
if flag == 1:
itmax = int(array[0])
array = getpfp(configfile, "calring_conv")
s = len(array)
flag = array[s - 1]
if flag == 1:
conv = float(array[0])
array = getpfp(configfile, "calring_fitwidth")
s = len(array)
flag = array[s - 1]
if flag == 1:
fitwidth = float(array[0])
# getting paths for filenames
pathin = os.path.dirname(image)
basein = os.path.basename(image)
pathlog = os.path.dirname(logfile)
baselog = os.path.basename(logfile)
# forcing logfiles to be created in the same directory as the input data
# (we change to this directory once starting the fortran code)
if len(pathin) > 0:
logfile = baselog
# start log now that all parameter are set up
with logging(logfile, debug) as log:
# Some basic checks, many tests are done in the FORTRAN code itself
# is the input file specified?
saltsafeio.filedefined('Input', image)
# if the input file is a file, does it exist?
if basein[0] != '@':
saltsafeio.fileexists(image)
infile = image
# if the input file is a list, throw an error
if basein[0] == '@':
raise SaltIOError(basein + ' list input instead of a file')
# optionally update the FORTRAN config file with new values - not implemented currently
# If all looks OK, run the FORTRAN code
if len(pathin) > 0:
dir = pathin
else:
dir = './'
infile = basein
print dir, infile, 'input directory and input file'
# Get current working directory as the Fortran code changes dir
startdir = os.getcwd()
ringfilter_wrapper.ringfilter(dir, axc, ayc, arad, rxc, ryc,
filterfreq, filterwidth, itmax, conv,
fitwidth, infile)
# go back to starting directory
os.chdir(startdir)
def saltclean(images,
outpath,
obslogfile=None,
gaindb=None,
xtalkfile=None,
geomfile=None,
subover=True,
trim=True,
masbias=None,
subbias=False,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover = False
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist = saltio.readgaindb(gaindb)
else:
dblist = []
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict = None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg = 'The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist = filename[ccdtype == 'ZERO']
masterbias_dict = {}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = fits.open(img)
bimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(blist, bkeys)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist = filename[ccdtype == 'FLAT']
masterflat_dict = {}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct = fits.open(img)
fimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict = compareimages(struct,
fimg,
masterflat_dict,
keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys = masterflat_dict[i][0]
flist = masterflat_dict[i][1:]
mflatname = outpath + createmasterflatname(flist, fkeys)
ffiles = ','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg = os.path.basename(img)
#print nimg, nimg in flatlist, nimg in biaslist
if not (nimg in biaslist):
#open the image
struct = fits.open(img)
simg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg = outpath + 'mbxgp' + os.path.basename(img)
saltmosaic(images=simg,
outimages=mimg,
outpref='',
geomfile=geomfile,
interp=interp,
cleanup=True,
clobber=clobber,
logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def saltfpeprofile(plottype,infile,logfile, verbose):
"""Line profile fitting and display"""
plottype = str(plottype)
plottype = plottype.strip()
if plottype == 'xwindow':
plottype = '/xw'
else:
plottype = '/ps'
# getting paths for filenames
pathin = os.path.dirname(infile)
basein = os.path.basename(infile)
pathlog = os.path.dirname(logfile)
baselog = os.path.basename(logfile)
print pathin, basein
# start log now that all parameters are set up
with logging(logfile, debug) as log:
# Some basic checks, some tests are done in the FORTRAN code itself
# is the input file specified?
saltsafeio.filedefined('Input',infile)
# if the input file is a file, does it exist?
if basein[0] != '@':
saltsafeio.fileexists(infile)
# if a list is input throw an error
if basein[0] == '@':
raise SaltIOError(basein + ' list input instead of a file' )
# The file contains a list of images, check they exist
# format is 4x dummy lines, then a line/lines containing: x,y,wave0,sn,norm,dnorm,filename
testfile= open(infile)
print testfile
for i in range(1, 5):
line = testfile.readline()
line = 'dummy'
infiles=[]
while (len(line.strip()) != 0):
line = testfile.readline()
if (len(line.strip()) > 0):
linearray = line.split()
infiles.append(linearray[6])
# check input files exist
saltsafeio.filesexist(infiles,'','r')
# If all looks OK, run the FORTRAN code
print 'input filename = ', infile
eprofile_wrapper.eprofile(plottype,infile)
def slotview(newfits,indata , fileout, srcfile, fps=10.0, phottype='square', sigdet=5, contpix=10, \
driftlimit=10, clobber=True,logfile='slotview.log',verbose=True):
#set up the variables
status = 0
entries = []
vig_lo = {}
vig_hi = {}
hour = 0
min = 0
sec = 0.
time0 = 0.
nframes = 0
sleep=0
with logging(logfile,debug) as log:
#enter in the input data
saltio.fileexists(newfits)
#set the sleep parameter
if fps>0: sleep=1.0/(fps)
# read in the data file
id, time, ratio, rerr, tx, ty, tflux, terr, cx, cy, cflux, cerr=st.readlcfile(indata)
# read extraction region defintion file
amp, x, y, x_o, y_o, r, br1, br2=st.readsrcfile(srcfile)
#determine the size of the data arrays
struct = saltio.openfits(newfits)
naxis1 = saltkey.get('NAXIS1',struct[1])
naxis2 = saltkey.get('NAXIS2',struct[1])
# Plot all of the data and the first image
# Create GUI
App = QtGui.QApplication([])
aw=SlotViewWindow(struct, id, tflux, cflux, ratio, time, phottype, sleep, \
tx, ty, cx, cy, r, br1, br2, naxis1, naxis2, sigdet, contpix, driftlimit)
aw.show()
# Start application event loop
app_exit=App.exec_()
# Check if GUI was executed succesfully
if app_exit!=0:
raise SALTError('InterIdentify GUI has unexpected exit status '+str(exit))
ratio, tflux, cflux, gframe, newphot=aw.ratio, aw.tflux, aw.cflux, aw.goodframes, aw.newphot
#close the input file
saltio.closefits(struct)
# Update the indata file if necessary
lc=saltio.openascii(fileout,'w')
for i in range(len(ratio)):
x['target']=tx[i]
x['comparison']=cx[i]
y['target']=ty[i]
y['comparison']=cy[i]
reltime=False
if gframe[i]:
st.writedataout(lc, id[i], time[i], x, y, tflux[i], terr[i], \
cflux[i], cerr[i], ratio[i], rerr[i], time[0], reltime)
saltio.closeascii(lc)
def saltmosaic(images,
outimages,
outpref,
geomfile,
interp='linear',
geotran=True,
fill=False,
cleanup=True,
clobber=False,
logfile=None,
verbose=True):
# Start the logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
# verify that the input and output lists are the same length
saltio.comparelists(infiles, outfiles, 'Input', 'output')
# does CCD geometry definition file exist
geomfilefile = geomfile.strip()
saltio.fileexists(geomfile)
gap = 0
xshift = [0, 0]
yshift = [0, 0]
rotation = [0, 0]
gap, xshift, yshift, rotation = saltio.readccdgeom(geomfile)
# open each raw image file and apply the transformation to it
for img, oimg in zip(infiles, outfiles):
# open the structure
struct = saltio.openfits(img)
# create the mosaic
ostruct = make_mosaic(struct,
gap,
xshift,
yshift,
rotation,
interp_type=interp,
geotran=geotran,
fill=fill,
cleanup=cleanup,
log=log,
verbose=verbose)
# update the header information
# housekeeping keywords
fname, hist = history(level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(ostruct[0], 'SMOSAIC',
'Images have been mosaicked', hist)
# write the image out
ostruct.writeto(oimg, clobber=clobber, output_verify='ignore')
# close the files
struct.close()
ostruct.close()
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist=filename[ccdtype=='ZERO']
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=pyfits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in flatlist or not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def saltfpcalprofile(axc,ayc,arad,rxc,ryc,filter, filterfreq,filterwidth,plottype,itmax,conv, fitwidth,rlo,rhi,rfixed,cenfixed,images,outfile,comment,cala, calb, calc, cald, calf,calprofilelogfile,logfile,useconfig,configfile,verbose):
"""Fits a Voigt profile to a ring in one or more Fabry-Perot calibration ring images"""
# default parameter values are set up in the pyraf .par file. The values used are then changed if a FORTRAN config file exists and the user elects to override the pyraf .par file.
# Is the input FORTRAN config file specified?
# If it is blank, then it will be ignored.
if useconfig:
configfile = configfile.strip()
if len(configfile) > 0:
#check exists
saltsafeio.fileexists(configfile)
# read updated parameters from the file
array=getpfp(configfile,"axc")
s=len(array)
flag = array[s-1]
if flag == 1:
axc=float(array[0])
array=getpfp(configfile,"ayc")
s=len(array)
flag = array[s-1]
if flag == 1:
ayc=float(array[0])
array=getpfp(configfile,"arad")
s=len(array)
flag = array[s-1]
if flag == 1:
arad=float(array[0])
array=getpfp(configfile,"rxc")
s=len(array)
flag = array[s-1]
if flag == 1:
rxc=float(array[0])
array=getpfp(configfile,"ryc")
s=len(array)
flag = array[s-1]
if flag == 1:
ryc=float(array[0])
array=getpfp(configfile,"calring_filter")
s=len(array)
flag = array[s-1]
if flag == 1:
filter=str(array[0])
array=getpfp(configfile,"calring_filter_width")
s=len(array)
flag = array[s-1]
if flag == 1:
filterwidth=int(array[0])
array=getpfp(configfile,"calring_filter_freq")
s=len(array)
flag = array[s-1]
if flag == 1:
filterfreq=int(array[0])
array=getpfp(configfile,"calring_itmax")
s=len(array)
flag = array[s-1]
if flag == 1:
itmax=int(array[0])
array=getpfp(configfile,"calring_conv")
s=len(array)
flag = array[s-1]
if flag == 1:
conv=float(array[0])
array=getpfp(configfile,"calring_fitwidth")
s=len(array)
flag = array[s-1]
if flag == 1:
fitwidth=float(array[0])
array=getpfp(configfile,"calring_rlo")
s=len(array)
flag = array[s-1]
if flag == 1:
rlo=float(array[0])
array=getpfp(configfile,"calring_rhi")
s=len(array)
flag = array[s-1]
if flag == 1:
rhi=float(array[0])
array=getpfp(configfile,"calring_fixed")
s=len(array)
flag = array[s-1]
if flag == 1:
cenfixed=str(array[0])
array=getpfp(configfile,"calibration_a")
s=len(array)
flag = array[s-1]
if flag == 1:
cala=float(array[0])
array=getpfp(configfile,"calibration_b")
s=len(array)
flag = array[s-1]
if flag == 1:
calb=float(array[0])
array=getpfp(configfile,"calibration_c")
s=len(array)
flag = array[s-1]
if flag == 1:
calc=float(array[0])
array=getpfp(configfile,"calibration_d")
s=len(array)
flag = array[s-1]
if flag == 1:
cald=float(array[0])
array=getpfp(configfile,"calibration_f")
s=len(array)
flag = array[s-1]
if flag == 1:
calf=float(array[0])
cenfixed=str(cenfixed)
if cenfixed[0] == 'y' or cenfixed[0] == 'Y':
cenfixed = 1
else:
cenfixed = 0
cenfixed = bool(cenfixed)
filter = str(filter)
if filter[0] == 'y' or filter[0] == 'Y':
filter = 1
else:
filter = 0
filter = bool(filter)
plottype = str(plottype)
plottype = plottype.strip()
if plottype == 'xwindow':
plottype = '/xw'
else:
plottype = '/ps'
# getting paths for filenames
pathin = os.path.dirname(images)
basein = os.path.basename(images)
pathout = os.path.dirname(outfile)
baseout = os.path.basename(outfile)
pathlog = os.path.dirname(logfile)
baselog = os.path.basename(logfile)
pathcalprofilelog = os.path.dirname(calprofilelogfile)
basecalprofilelog = os.path.basename(calprofilelogfile)
# forcing logfiles to be created in the same directory as the input data
# (we change to this directory once starting the fortran code)
if len(pathin) > 0:
logfile = baselog
calprofilelogfile = basecalprofilelog
# start log now that all parameter are set up
with logging(logfile, debug) as log:
# Some basic checks, many tests are done in the FORTRAN code itself
# is the input file specified?
saltsafeio.filedefined('Input',images)
# if the input file is a file, does it exist?
if basein[0] != '@':
saltsafeio.fileexists(images)
infile = images
# if the input file is a list, does it exist?
if basein[0] == '@':
infile2 = basein.lstrip('@')
if (len(pathin) > 0):
infile = pathin + '/' + infile2
else:
infile = infile2
saltsafeio.listexists('Input',infile)
# parse list of input files
infiles=fpsafeio.fplistparse('Raw image',images,'','','')
# check input files exist
saltsafeio.filesexist(infiles,'','r')
# is the proposed output file specified?
saltsafeio.filedefined('Output',outfile)
# check whether the proposed output file exists
path = os.path.dirname(images)
if len(path) > 0:
dir = path + '/'
else:
dir = './'
# print dir, ' directory with data'
outfile = outfile.strip()
if os.path.isfile(outfile):
print 'output file exists, appending'
# saltsafeio.delete(outfile)
# optionally update the FORTRAN config file with new values - not implemented currently
# If all looks OK, run the FORTRAN code
infile = basein
if len(pathout) > 0 :
if (pathin == pathout):
outfile = baseout
# absolute path then leave alone
elif pathout[0] == '/':
outfile = outfile
else:
# as will move to directory of input data, need to strip this off if its in the path
a = pathout.lstrip(pathin)
b = a.lstrip('/')
outfile = b + '/' + baseout
print dir, infile, outfile, 'input directory, input and output files'
if basein[0] == '@':
print infiles,'infiles'
# Get current working directory as the Fortran code changes dir
startdir = os.getcwd()
calprofile_wrapper.calprofile(dir,calprofilelogfile,outfile,comment, axc, ayc,arad, rxc,ryc,filter,filterfreq,filterwidth,plottype,itmax,conv,fitwidth,rlo,rhi,rfixed, cenfixed,infile, cala, calb, calc, cald, calf)
# go back to starting directory
os.chdir(startdir)
def saltflat(images,outimages,outpref, flatimage,minflat=1, allext=False, clobber=False, \
logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
#verify that the input and output lists are the same length
saltio.comparelists(infiles,outfiles,'Input','output')
# check flatfield image exists
flaimage= flatimage.strip()
saltio.fileexists(flatimage)
flatstruct= saltio.openfits(flatimage)
# Normalize the flat field image
# This requires to go through each science extension and divide it by
# mean of the image. Things that have to be checked:
# that data exist, that it is a science extension
#determine the global mean
fmean=0
fcount=0
#replace bad pixels
for i in range(len(flatstruct)):
if flatstruct[i].data is not None and (flatstruct[i].name=='SCI' or flatstruct[i].name=='PRIMARY'):
data = flatstruct[i].data
mask = (data > minflat)
if (numpy.nan==flatstruct[i].data).sum() or (numpy.inf==flatstruct[i].data).sum():
message = '\nWARNING -- SALTFLAT: %s contains invalid values' % flatimage
log.warning(message,with_stdout=verbose)
flatstruct[i].data[mask==0] = minflat
flatstruct[i].data[flatstruct[i].data==numpy.inf] = minflat
#determine the mean
mask = (data > minflat)
fmean += data[mask].sum()
fcount += data[mask].size
if fcount>0: fmean=fmean/fcount
for i in range(len(flatstruct)):
if flatstruct[i].name=='PRIMARY':
#is it a flat--if not throw a warning
try:
key_ccdtype=saltkey.get('CCDTYPE', flatstruct[i])
except:
key_ccdtype=None
if key_ccdtype!='FLAT':
message = '\nWARNING -- SALTFLAT: %s does not have CCDTYPE=FLAT' % flatimage
log.warning(message,with_stdout=verbose)
#if there are data, normalize it
if flatstruct[i].data is not None:
flatstruct[i].data=flatnormalize(flatstruct[i].data, minflat)
#Noramlize the science extensions
if flatstruct[i].name=='SCI':
if flatstruct[i].data is not None:
if allext is False: fmean=flatstruct[i].data.mean()
flatstruct[i].data=flatnormalize(flatstruct[i].data, minflat, fmean)
#Apply to the variance frames
if saltkey.found('VAREXT', flatstruct[i]):
varext=saltkey.get('VAREXT',flatstruct[i])
flatstruct[varext].data=flatstruct[varext].data/fmean**2
# open each raw image file
for infile, outfile in zip(infiles,outfiles):
struct = saltio.openfits(infile)
# flat field correct the image
outstruct = flat(struct, flatstruct)
try:
pass
except Exception,e:
msg='Unable to flatten %s because %s' % (infile, e)
raise SaltError(msg)
#add any header keywords like history
fname, hist=history(level=1, wrap=False)
saltkey.housekeeping(struct[0],'SFLAT', 'File flatfield corrected', hist)
#write it out and close it
saltio.writefits(outstruct,outfile,clobber=clobber)
saltio.closefits(struct)
#output the information
log.message('Flatfields image %s using %s' % (infile, flatimage), with_header=False, with_stdout=verbose)
#clost the flatfield image
saltio.closefits(flatstruct)
def readsrcfile(srcfile):
"""Read in the src file and return the parameters extracted from that file
"""
#set up some variables
amp = {}
x = {}
y = {}
x_o = {}
y_o = {}
r = {}
br1 = {}
br2 = {}
tgt_btype='region'
cmp_btype='region'
# check extraction region defintion file exists
srcfile = srcfile.strip()
saltio.fileexists(srcfile)
#try loading the region file
try:
region=np.round(np.loadtxt(srcfile))
except Exception as e:
msg='SLOTTOOLs--ERROR: Unable to load array from %s' % srcfile
raise SaltIOError(msg)
# read extraction region defintion file
try:
# Check if region definition file has an acceptable format
if region.shape[1] not in [7,9]:
raise SaltIOError('Region definition file has invalid number of columns')
# Check if all object numbers in the region file are unique
if len(set(region[:,0]))!=len(region[:,0]):
raise SaltIOError('Object numbers in region definition file not unique')
# Find target and comparison
tindex=None
cindex=None
for i in range(region.shape[0]):
if region[i][0]==1:
tindex=i
elif region[i][0]==2:
cindex=i
else:
msg='Object %d not used' % region[i][0]
log.message(msg)
continue
if tindex is None or cindex is None:
raise SaltIOError('Could not find target and comparison in region definition file.')
#set the background type
if len(region[tindex])==7: tgt_btype='annulus'
if len(region[cindex])==7: cmp_btype='annulus'
print(tgt_btype, cmp_btype)
# Check if selected background mode matches defined regions
valid=True
if (tgt_btype=='region' or cmp_btype=='region') and region.shape[1]<9:
valid=False
if tgt_btype=='annulus' and not np.any(region[tindex][5:7]):
valid=False
elif tgt_btype=='region' and not np.any(region[tindex][5:9]):
valid=False
if cmp_btype=='annulus' and not np.any(region[cindex][5:7]):
valid=False
elif cmp_btype=='region' and not np.any(region[cindex][5:91]):
valid=False
if not valid:
raise SaltIOError('Selected background types do not match regions specified in region definition file.')
# Read the parameters
for object in [('target',tindex),('comparison',cindex)]:
amp[object[0]]=int(region[object[1]][1])
x[object[0]]=int(region[object[1]][2])
y[object[0]]=int(region[object[1]][3])
x_o[object[0]]=int(region[object[1]][2])
y_o[object[0]]=int(region[object[1]][3])
r[object[0]]=int(region[object[1]][4])
if tgt_btype=='annulus':
# Ensure br2>=br1
br1['target']=np.min((int(region[tindex][5]),int(region[tindex][6])))
br2['target']=np.max((int(region[tindex][5]),int(region[tindex][6])))
elif tgt_btype=='region':
i=tindex
# Ensure vertices are in proper order
br1['target']=[np.min((int(region[i][5]),int(region[i][7]))),
np.min((int(region[i][6]),int(region[i][8])))]
br2['target']=[np.max((int(region[i][5]),int(region[i][7]))),
np.max((int(region[i][6]),int(region[i][8])))]
if cmp_btype=='annulus':
# Ensure br2>=br1
br1['comparison']=np.min((int(region[cindex][5]),int(region[cindex][6])))
br2['comparison']=np.max((int(region[cindex][5]),int(region[cindex][6])))
elif cmp_btype=='region':
i=cindex
# Ensure vertices are in proper order
br1['comparison']=[np.min((int(region[i][7]),int(region[i][5]))),
np.min((int(region[i][8]),int(region[i][6])))]
br2['comparison']=[np.max((int(region[i][7]),int(region[i][5]))),
np.max((int(region[i][8]),int(region[i][6])))]
except Exception as e:
msg='Error processing region definition file because %s'%e
raise SaltIOError(msg)
if (len(amp) != 2 or len(x) != 2 or len(y) != 2 or len(r) != 2 or len(br1) != 2 or len(br2) != 2):
message = 'ERROR: SALTVIEW -- extraction region definition file '+srcfile
message += 'does not have an acceptable format.'
raise SaltIOError(msg)
return amp, x, y, x_o, y_o, r, br1, br2
def saltfpnightring(axc, ayc, arad, rxc, ryc, filter, filterfreq, filterwidth,
plot, plottype, itmax, conv, fitwidth, images, outfile,
comment, cala, calb, calc, cald, calf, nightringlogfile,
logfile, useconfig, configfile, verbose):
"""Measures the centre coordinates and radius of one or more
night-time calibration rings """
# default parameter values are set up in the pyraf .par file. The values used are then changed if a FORTRAN config file exists and the user elects to override the pyraf .par file.
# Is the input FORTRAN config file specified?
# If it is blank, then it will be ignored.
if useconfig:
configfile = configfile.strip()
if len(configfile) > 0:
#check exists
saltsafeio.fileexists(configfile)
# read updated parameters from the file
array = getpfp(configfile, "axc")
s = len(array)
flag = array[s - 1]
if flag == 1:
axc = float(array[0])
array = getpfp(configfile, "ayc")
s = len(array)
flag = array[s - 1]
if flag == 1:
ayc = float(array[0])
array = getpfp(configfile, "arad")
s = len(array)
flag = array[s - 1]
if flag == 1:
arad = float(array[0])
array = getpfp(configfile, "rxc")
s = len(array)
flag = array[s - 1]
if flag == 1:
rxc = float(array[0])
array = getpfp(configfile, "ryc")
s = len(array)
flag = array[s - 1]
if flag == 1:
ryc = float(array[0])
array = getpfp(configfile, "calring_filter")
s = len(array)
flag = array[s - 1]
if flag == 1:
filter = str(array[0])
array = getpfp(configfile, "calring_filter_width")
s = len(array)
flag = array[s - 1]
if flag == 1:
filterwidth = int(array[0])
array = getpfp(configfile, "calring_filter_freq")
s = len(array)
flag = array[s - 1]
if flag == 1:
filterfreq = int(array[0])
array = getpfp(configfile, "calring_itmax")
s = len(array)
flag = array[s - 1]
if flag == 1:
itmax = int(array[0])
array = getpfp(configfile, "calring_conv")
s = len(array)
flag = array[s - 1]
if flag == 1:
conv = float(array[0])
array = getpfp(configfile, "calring_fitwidth")
s = len(array)
flag = array[s - 1]
if flag == 1:
fitwidth = float(array[0])
array = getpfp(configfile, "calibration_a")
s = len(array)
flag = array[s - 1]
if flag == 1:
cala = float(array[0])
array = getpfp(configfile, "calibration_b")
s = len(array)
flag = array[s - 1]
if flag == 1:
calb = float(array[0])
array = getpfp(configfile, "calibration_c")
s = len(array)
flag = array[s - 1]
if flag == 1:
calc = float(array[0])
array = getpfp(configfile, "calibration_d")
s = len(array)
flag = array[s - 1]
if flag == 1:
cald = float(array[0])
array = getpfp(configfile, "calibration_f")
s = len(array)
flag = array[s - 1]
if flag == 1:
calf = float(array[0])
filter = str(filter)
if filter[0] == 'y' or filter[0] == 'Y':
filter = 1
else:
filter = 0
filter = bool(filter)
plot = str(plot)
if plot[0] == 'y' or plot[0] == 'Y':
plot = 1
else:
plot = 0
plot = bool(plot)
plottype = str(plottype)
plottype = plottype.strip()
if plottype == 'xwindow':
plottype = '/xw'
else:
plottype = '/ps'
# additional verbose command as required in the fortran code too. Set this to whatever is input in the epar file.
verb = str(verbose)
if verb[0] == 'y' or verb[0] == 'Y':
verb = 1
else:
verb = 0
verb = bool(verb)
# getting paths for filenames
pathin = os.path.dirname(images)
basein = os.path.basename(images)
pathout = os.path.dirname(outfile)
baseout = os.path.basename(outfile)
pathlog = os.path.dirname(logfile)
baselog = os.path.basename(logfile)
pathnightringlog = os.path.dirname(nightringlogfile)
basenightringlog = os.path.basename(nightringlogfile)
# forcing logfiles to be created in the same directory as the input data
# (we change to this directory once starting the fortran code)
if len(pathin) > 0:
logfile = baselog
nightringlogfile = basenightringlog
# start log now that all parameter are set up
with logging(logfile, debug) as log:
# Some basic checks, many tests are done in the FORTRAN code itself
# is the input file specified?
saltsafeio.filedefined('Input', images)
# if the input file is a file, does it exist?
if basein[0] != '@':
saltsafeio.fileexists(images)
infile = images
# if the input file is a list, does it exist?
if basein[0] == '@':
infile2 = basein.lstrip('@')
if (len(pathin) > 0):
infile = pathin + '/' + infile2
else:
infile = infile2
saltsafeio.listexists('Input', infile)
# parse list of input files
infiles = fpsafeio.fplistparse('Raw image', images, '', '', '')
# check input files exist
saltsafeio.filesexist(infiles, '', 'r')
# is the proposed output file specified?
saltsafeio.filedefined('Output', outfile)
# check whether the proposed output file exists
path = os.path.dirname(images)
if len(path) > 0:
dir = path + '/'
else:
dir = './'
# print dir, ' directory with data'
outfile = outfile.strip()
if os.path.isfile(outfile):
print 'output file exists, appending'
# saltsafeio.delete(outfile)
# optionally update the FORTRAN config file with new values - not implemented currently
# If all looks OK, run the FORTRAN code
infile = basein
if len(pathout) > 0:
if (pathin == pathout):
outfile = baseout
# absolute path then leave alone
elif pathout[0] == '/':
outfile = outfile
else:
# as will move to directory of input data, need to strip this off if its in the path
a = pathout.lstrip(pathin)
b = a.lstrip('/')
outfile = b + '/' + baseout
print dir, infile, outfile, 'input directory, input and output files'
if basein[0] == '@':
print infiles, 'infiles'
# Get current working directory as the Fortran code changes dir
startdir = os.getcwd()
nightring_wrapper.nightring(dir, nightringlogfile, outfile, comment,
axc, ayc, arad, rxc, ryc, filter,
filterfreq, filterwidth, plot, plottype,
itmax, conv, fitwidth, infile, cala, calb,
calc, cald, calf, verb)
# go back to starting directory
os.chdir(startdir)
def saltfpevelocity(infile,existfit,xc,yc,rmax,logfile, verbose):
"""XXX"""
# getting paths for filenames
pathin = os.path.dirname(infile)
basein = os.path.basename(infile)
pathlog = os.path.dirname(logfile)
baselog = os.path.basename(logfile)
print(pathin, basein)
# start log now that all parameters are set up
with logging(logfile, debug) as log:
# Some basic checks, some tests are done in the FORTRAN code itself
# is the input file specified?
saltsafeio.filedefined('Input',infile)
# if the input file is a file, does it exist?
if basein[0] != '@':
saltsafeio.fileexists(infile)
# if a list is input throw an error
if basein[0] == '@':
raise SaltIOError(basein + ' list input instead of a file' )
# The file contains a list of images, check they exist
# format is 4x dummy lines, then a line/lines containing: x,y,wave0,sn,norm,dnorm,filename
testfile= open(infile)
print(testfile)
for i in range(1, 5):
line = testfile.readline()
line = 'dummy'
infiles=[]
while (len(line.strip()) != 0):
line = testfile.readline()
if (len(line.strip()) > 0):
linearray = line.split()
infiles.append(linearray[6])
# check input files exist
saltsafeio.filesexist(infiles,'','r')
# convert existfile to a one letter string
if existfit == 'yes':
existfit == 'y'
else:
existfit =='n'
# If all looks OK, run the FORTRAN code
print(infile, 'input filename')
print(existfit, xc, yc, rmax)
evelocity_wrapper2.evelocity2(infile,existfit,xc,yc,rmax)
def slotphot(images,outfile,srcfile,newfits=None,phottype='square',
subbacktype='median',sigback=3,mbin=7,sorder=3,niter=5,sigdet=5,
contpix=10,ampperccd=2,ignorexp=6,driftlimit=10.,finddrift=True,
outtype='ascii',reltime=True,clobber=True,logfile='salt.log',
verbose=True):
"""Perform photometry on listed SALT slotmode *images*."""
with logging(logfile,debug) as log:
# set up the variables
entries = []
vig_lo = {}
vig_hi = {}
amp = {}
x = {}
y = {}
x_o = {}
y_o = {}
r = {}
br1 = {}
br2 = {}
hour = 0
min = 0
sec = 0.
time0 = 0.
nframes = 0
bin=mbin
order=sorder
# is the input file specified?
saltsafeio.filedefined('Input',images)
# if the input file is a list, does it exist?
if images[0] == '@':
saltsafeio.listexists('Input',images)
# parse list of input files
infiles=saltsafeio.listparse('Raw image',images,'','','')
# check input files exist
saltsafeio.filesexist(infiles,'','r')
# is the output file specified?
saltsafeio.filedefined('Output',outfile)
# check output file does not exist, optionally remove it if it does exist
if os.path.exists(outfile) and clobber:
os.remove(outfile)
elif os.path.exists(outfile) and not clobber:
raise SaltIOError('File '+outfile+' already exists, use clobber=y')
# open output ascii file
if outtype=='ascii':
try:
lc = open(outfile,'a')
except:
raise SaltIOError('Cannot open ouput file '+outfile)
# is the extraction region defintion file specified?
saltsafeio.filedefined('Extraction region defintion',srcfile)
# check extraction region defintion file exists
srcfile = srcfile.strip()
saltsafeio.fileexists(srcfile)
# read extraction region defintion file
amp, x, y, x_o, y_o, r, br1, br2=slottool.readsrcfile(srcfile)
# set the writenewfits parameter
if not newfits or newfits=='none':
writenewfits=False
else:
writenewfits=newfits
# get time of first exposure and basic information about the observations
infile=infiles[0]
struct=saltsafeio.openfits(infile)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot=saltsafekey.instrumid(struct,infile)
# how many extensions?
nextend=saltsafekey.get('NEXTEND',struct[0],infile)
if nextend < amp['comparison']:
msg='Insufficient number of extensions in %s' % (infile)
raise SaltIOError(msg)
# how many amplifiers?
amplifiers=saltsafekey.get('NCCDS',struct[0],infile)
amplifiers = int(ampperccd*float(amplifiers))
if ampperccd>0:
nframes = int(nextend/amplifiers)
nstep=amplifiers
else:
nframes = nextend
nstep=1
ntotal=nframes*len(infiles)
# image size
naxis1=saltsafekey.get('NAXIS1',struct[amp['comparison']],infile)
naxis2=saltsafekey.get('NAXIS2',struct[amp['comparison']],infile)
# CCD binning
ccdsum=saltsafekey.get('CCDSUM',struct[0],infile)
binx=int(ccdsum.split(' ')[0])
biny=int(ccdsum.split(' ')[1])
# Identify the time of the observations
ext = 1
try:
time0=slottool.getobstime(struct[ext], infile+'['+str(ext)+']')
dateobs=saltsafekey.get('DATE-OBS',struct[ext],infile)
dateobs=dateobs.replace('-','/')
except:
raise SaltIOError('No time or obsdate in first image')
# If a total file is to be written out, create it and update it
if writenewfits:
if os.path.isfile(writenewfits):
if clobber:
saltsafeio.delete(writenewfits)
else:
raise SaltIOError('Newfits file exists, use clobber')
try:
hdu=pyfits.PrimaryHDU()
hdu.header=struct[0].header
hdu.header['NCCDS']=1
hdu.header['NSCIEXT']=ntotal-ignorexp
hdu.header['NEXTEND']=ntotal-ignorexp
hduList=pyfits.HDUList(hdu)
hduList.verify()
hduList.writeto(writenewfits)
except:
raise SaltIOError('Could not create newfits file, '+writenewfits)
# Close image file
saltsafeio.closefits(struct)
# Read newfits file back in for updating
if writenewfits:
try:
hduList=pyfits.open(writenewfits,mode='update')
except:
raise SaltIOError('Cannot open newfits file '+writenewfits+' for updating.')
# set up the arrays
j=0
time=np.zeros(ntotal ,dtype='float')-1.0
dx=np.zeros(ntotal ,dtype='float')
dy=np.zeros(ntotal ,dtype='float')
tflux=np.zeros(ntotal ,dtype='float')
terr =np.zeros(ntotal ,dtype='float')
cflux=np.zeros(ntotal ,dtype='float')
cerr =np.zeros(ntotal ,dtype='float')
ratio=np.zeros(ntotal ,dtype='float')
rerr =np.zeros(ntotal ,dtype='float')
tgt_x=np.zeros(ntotal ,dtype='float')
tgt_y=np.zeros(ntotal ,dtype='float')
cmp_x=np.zeros(ntotal ,dtype='float')
cmp_y=np.zeros(ntotal ,dtype='float')
p_one=100./ntotal # One percent
p_old=-1 # Previous completed percentage
p_new=0 # New completed percentage
p_n=1 # Counter number
for infile in infiles:
# Log
if verbose:
log.message('Starting photometry on file '+infile, with_stdout=False)
struct=pyfits.open(infile)
# Skip through the frames and process each frame individually
for i in range(nframes):
# Show progress
if verbose:
p_new=int(p_n*p_one)
p_n+=1
if p_new!=p_old:
ctext='Percentage Complete: %d\r' % p_new
sys.stdout.write(ctext)
sys.stdout.flush()
p_old=p_new
if not (infile==infiles[0] and i < ignorexp):
ext=amp['comparison']+i*nstep
try:
header=struct[ext].header
array=struct[ext].data
array=array*1.0
except:
msg='Unable to open extension %i in image %s' % (ext, infile)
raise SaltIOError(msg)
# starti the analysis of each frame
# get the time
time[j]=slottool.getobstime(struct[ext],infile+'['+str(ext)+']')
# gain and readout noise
try:
gain=float(header['GAIN'])
except:
gain=1
raise SaltIOError('Gain not specified in image header')
try:
rdnoise=float(header['RDNOISE'])
except:
rdnoise=0
raise SaltIOError('RDNOISE not specified in image header')
# background subtraction
if not subbacktype=='none':
try:
array=subbackground(array, sigback, bin, order, niter, subbacktype)
except SaltError:
log.warning('Image '+infile+' extention '+str(ext)+' is blank, skipping')
continue
# x-y fit to the comparison star and update the x,y values
if finddrift:
carray, fx,fy=slottool.finddrift(array, x['comparison'], y['comparison'], r['comparison'], naxis1, naxis2, sigdet, contpix, sigback, driftlimit, niter)
if fx > -1 and fy > -1:
if fx < naxis1 and fy < naxis2:
dx[j]=x['comparison']-fx
dy[j]=y['comparison']-fy
x['comparison']=fx
y['comparison']=fy
x['target']=x['target']-dx[j]
y['target']=y['target']-dy[j]
else:
dx[j]=0
dy[j]=0
x['comparison']=x_o['comparison']
y['comparison']=y_o['comparison']
x['target']=x_o['target']
y['target']=y_o['target']
else:
msg='No comparison object found in image file ' + infile+' on extension %i skipping.' % ext
log.warning(msg)
pass
# do photometry
try:
tflux[j],terr[j],cflux[j],cerr[j],ratio[j],rerr[j]=slottool.dophot(phottype, array, x, y, r, br1, br2, gain, rdnoise, naxis1, naxis2)
except SaltError, e:
msg='Could not do photometry on extension %i in image %s because %s skipping.' % (ext, infile, e)
log.warning(msg)
tgt_x[j]=x['target']
tgt_y[j]=y['target']
cmp_x[j]=x['comparison']
cmp_y[j]=y['comparison']
# record results
# TODO! This should be removed in favor of the write all to disk in the end
if outtype=='ascii':
slottool.writedataout(lc, j+1, time[j], x, y, tflux[j], terr[j], cflux[j],cerr[j],ratio[j],rerr[j],time0, reltime)
# write newfits file
if writenewfits:
# add original name and extension number to header
try:
hdue=pyfits.ImageHDU(array)
hdue.header=header
hdue.header.update('ONAME',infile,'Original image name')
hdue.header.update('OEXT',ext,'Original extension number')
hduList.append(hdue)
except:
log.warning('Could not update image in newfits '+infile+' '+str(ext))
# increment counter
j+=1
# close FITS file
saltsafeio.closefits(struct)
# close newfits file
if writenewfits:
try:
hduList.flush()
hduList.close()
except:
raise SaltIOError('Cannot close newfits file.')
# write to output
if outtype=='ascii':
# close output ascii file
try:
lc.close()
except:
raise SaltIOError('Cannot close ouput file ' + outfile)
elif outtype=='fits':
print 'writing fits'
try:
c1=pyfits.Column(name='index',format='D',array=np.arange(ntotal))
if reltime:
c2=pyfits.Column(name='time',format='D',array=time-time0)
else:
c2=pyfits.Column(name='time',format='D',array=time)
c3=pyfits.Column(name='tgt_x',format='D',array=tgt_x)
c4=pyfits.Column(name='tgt_y',format='D',array=tgt_y)
c5=pyfits.Column(name='tgt_flux',format='D',array=tflux)
c6=pyfits.Column(name='tgt_err',format='D',array=terr)
c7=pyfits.Column(name='cmp_x',format='D',array=cmp_x)
c8=pyfits.Column(name='cmp_y',format='D',array=cmp_y)
c9=pyfits.Column(name='cmp_flux',format='D',array=cflux)
c10=pyfits.Column(name='cmp_err',format='D',array=cerr)
c11=pyfits.Column(name='flux_ratio',format='D',array=ratio)
c12=pyfits.Column(name='flux_ratio_err',format='D',array=rerr)
tbhdu=pyfits.new_table([c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12])
# Add header information
tbhdu.header.update('RELTIME',str(reltime),'Time relative to first datapoint or absolute.')
tbhdu.writeto(outfile)
print 'fits written to ',outfile
except:
raise SaltIOError('Could not write to fits table.')
