def convertbin(inpath, fitsconfig, logfile, verbose):
if len(glob.glob(inpath+'/*.bin')) > 0:
saltbin2fit(inpath=inpath,outpath=inpath,cleanup=True,fitsconfig=fitsconfig,logfile=logfile,verbose=verbose)
for bfile in glob.glob(inpath+'/*.bin'):
saltio.delete(bfile)
ffile=bfile.replace('bin', 'fits')
slotreadtimefix(ffile, ffile, '', clobber=True, logfile=logfile, verbose=verbose)
def specsky(images,outimages,outpref, method='normal', section=None,
function='polynomial', order=2,
clobber=True,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outfiles=saltio.listparse('Outfile', outimages, outpref,infiles,'')
if method not in ['normal', 'fit']:
msg='%s mode is not supported yet' % method
raise SALTSpecError(msg)
if section is None:
section=saltio.getSection(section)
msg='This mode is not supported yet'
raise SALTSpecError(msg)
else:
section=saltio.getSection(section)
# Identify the lines in each file
for img, ofile in zip(infiles, outfiles):
log.message('Subtracting sky spectrum in image %s into %s' % (img, ofile))
#open the images
hdu=saltio.openfits(img)
#sky subtract the array
hdu=skysubtract(hdu, method=method, section=section, funct=function, order=order)
#write out the image
if clobber and os.path.isfile(ofile): saltio.delete(ofile)
hdu.writeto(ofile)
def saltobslog(images,
outfile,
clobber=False,
logfile='salt.log',
verbose=True):
"""Create the observation log from the input files"""
#start the logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
#create the header dictionary
headerDict = obslog(infiles, log)
#clobber the output if it exists
if (clobber and os.path.isfile(outfile)):
saltio.delete(outfile)
#create the fits file
struct = createobslogfits(headerDict)
# close table file
saltio.writefits(struct, outfile)
#indicate the log was created
log.message('\nSALTLOG -- created observation log ' + outfile)
def saltobslog(images,outfile,clobber=False,logfile='salt.log',verbose=True):
"""Create the observation log from the input files"""
#start the logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
#create the header dictionary
headerDict=obslog(infiles, log)
#clobber the output if it exists
if (clobber and os.path.isfile(outfile)):
saltio.delete(outfile)
#create the fits file
struct=createobslogfits(headerDict)
# close table file
saltio.writefits(struct, outfile)
#indicate the log was created
log.message('\nSALTLOG -- created observation log ' + outfile)
def mkheader(file, keyword, value, comment):
"""create keyword with mkheader IRAF tool i.e. without opening the whole file"""
try:
tmpfile = saltsafeio.tmpfile('.', False)
tmp = saltsafeio.openascii(tmpfile, 'w')
tmp.write('%-8s= \'%-18s\' / %-s\n' % (keyword, value, comment))
saltsafeio.closeascii(tmp)
iraf.noao.artdata.mkheader(file, tmpfile, append='y', verbose='n')
saltsafeio.delete(tmpfile, False)
except:
raise SaltIOError('Cannot edit keyword ' + keyword + ' in ' + file)
def mkheader(file,keyword,value,comment):
"""create keyword with mkheader IRAF tool i.e. without opening the whole file"""
try:
tmpfile=saltsafeio.tmpfile('.',False)
tmp=saltsafeio.openascii(tmpfile,'w')
tmp.write('%-8s= \'%-18s\' / %-s\n' % (keyword,value,comment))
saltsafeio.closeascii(tmp)
iraf.noao.artdata.mkheader(file,tmpfile,append='y',verbose='n')
saltsafeio.delete(tmpfile,False)
except:
raise SaltIOError('Cannot edit keyword '+keyword+' in '+file)
def write_extract_fits(ofile, ap_list, clobber=False):
"""Write out the extracted spectrum to a FITS table. If the file already
exists, this will not overwrite it.
For each spectrum in ap_list, it will add another extension to the
fits file. Each extension will have the first column as wavelength,
the second column as counts, and the third column as sigma on the
counts.
ofile: Output file to write
ap_list: List of extracted spectrum
clobber: delete ofile if it already exists
"""
# delete the file
if os.path.isfile(ofile) and clobber:
saltio.delete(ofile)
# create the primary array
hdu = pyfits.PrimaryHDU()
hdulist = pyfits.HDUList([hdu])
# create the columns and the
for ap in ap_list:
fvar = abs(ap.lvar) ** 0.5
# create the columns
col1 = pyfits.Column(
name='wavelength',
format='D',
unit='Angstroms',
array=ap.wave)
col2 = pyfits.Column(
name='counts',
format='D',
unit='Counts',
array=ap.ldata)
col3 = pyfits.Column(name='counts_err', format='D', array=fvar)
# add to the table
tbhdu = pyfits.new_table([col1, col2, col3])
hdulist.append(tbhdu)
# write it out
hdulist.writeto(ofile)
return
def specsky(images,
outimages,
outpref,
method='normal',
section=None,
function='polynomial',
order=2,
clobber=True,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outfiles = saltio.listparse('Outfile', outimages, outpref, infiles, '')
if method not in ['normal', 'fit']:
msg = '%s mode is not supported yet' % method
raise SALTSpecError(msg)
if section is None:
section = saltio.getSection(section)
msg = 'This mode is not supported yet'
raise SALTSpecError(msg)
else:
section = saltio.getSection(section)
# Identify the lines in each file
for img, ofile in zip(infiles, outfiles):
log.message('Subtracting sky spectrum in image %s into %s' %
(img, ofile))
# open the images
hdu = saltio.openfits(img)
# sky subtract the array
hdu = skysubtract(hdu,
method=method,
section=section,
funct=function,
order=order)
# write out the image
if clobber and os.path.isfile(ofile):
saltio.delete(ofile)
hdu.writeto(ofile)
def write_extract_fits(ofile, ap_list, clobber=False):
"""Write out the extracted spectrum to a FITS table. If the file already
exists, this will not overwrite it.
For each spectrum in ap_list, it will add another extension to the
fits file. Each extension will have the first column as wavelength,
the second column as counts, and the third column as sigma on the
counts.
ofile: Output file to write
ap_list: List of extracted spectrum
clobber: delete ofile if it already exists
"""
# delete the file
if os.path.isfile(ofile) and clobber:
saltio.delete(ofile)
# create the primary array
hdu = pyfits.PrimaryHDU()
hdulist = pyfits.HDUList([hdu])
# create the columns and the
for ap in ap_list:
fvar = abs(ap.lvar)**0.5
# create the columns
col1 = pyfits.Column(name='wavelength',
format='D',
unit='Angstroms',
array=ap.wave)
col2 = pyfits.Column(name='counts',
format='D',
unit='Counts',
array=ap.ldata)
col3 = pyfits.Column(name='counts_err', format='D', array=fvar)
# add to the table
tbhdu = pyfits.new_table([col1, col2, col3])
hdulist.append(tbhdu)
# write it out
hdulist.writeto(ofile)
return
def saltelsdata(propcode, obsdate, elshost, elsname, elsuser, elspass,
sdbhost,sdbname,sdbuser, password, clobber,logfile,verbose):
# set up
proposers = []
propids = []
pids = []
with logging(logfile,debug) as log:
# are the arguments defined
if propcode.strip().upper()=='ALL':
pids=saltmysql.getproposalcodes(str(obsdate), sdbhost,sdbname,sdbuser, password)
else:
pids = saltio.argunpack('propcode',propcode)
#open the database
els=saltmysql.connectelsview(elshost, elsname, elsuser, elspass)
sdb=saltmysql.connectdb(sdbhost,sdbname,sdbuser, password)
#create the values for the entire night
#loop through the proposals
for pid in pids:
outfile='%s_%s_elsdata.fits' % (pid, obsdate)
if clobber and os.path.isfile(outfile):
saltio.delete(outfile)
mintime, maxtime=determinetime(sdb, pid, obsdate)
message='Extracting ELS data for %s from %s to %s' % (pid, mintime, maxtime)
log.message(message, with_stdout=verbose)
getelsdata(els, sdb, outfile, mintime, maxtime)
def slotback(images,outfits,extension,imgtype='image',subbacktype='median',
sigback=3,mbin=7,sorder=3,niter=5,ampperccd=2,ignorexp=6,
clobber=False,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# set up the variables
order=sorder
plotfreq=0.01
ftime=plotfreq
# is the input file specified?
infiles = saltio.argunpack ('Input',images)
# is the output file specified?
saltio.filedefined('Output',outfits)
#open the first file and check its charactistics
struct=saltio.openfits(infiles[0])
# how many extensions?
nextend=saltkey.get('NEXTEND',struct[0])
if nextend < extension:
msg='Insufficient number of extensions in %s' % (infile)
raise SaltIOError(msg)
# how many amplifiers?
amplifiers=saltkey.get('NCCDS',struct[0])
amplifiers = int(ampperccd*float(amplifiers))
if ampperccd>0:
nframes = nextend/amplifiers
nstep=amplifiers
else:
nframes = nextend
nstep=1
ntotal=nframes*len(infiles)
# image size
naxis1=saltkey.get('NAXIS1',struct[extension])
naxis2=saltkey.get('NAXIS2',struct[extension])
# CCD binning
ccdsum=saltkey.get('CCDSUM',struct[0])
binx=int(ccdsum.split(' ')[0])
biny=int(ccdsum.split(' ')[1])
# If a total file is to written out, create it and update it
hdu = 1
# delete the file if clobber is on and the file exists
if os.path.isfile(outfits):
if clobber:
saltio.delete(outfits)
else:
raise SaltIOError('Output fits file '+writenewfits+'already exists. Use Clobber=yes to overwrite file')
# create the output file
try:
hdu = pyfits.PrimaryHDU()
hdu.header=struct[0].header
hdu.header['NCCDS']=1
hdu.header['NSCIEXT']=ntotal-ignorexp
hdu.header['NEXTEND']=ntotal-ignorexp
hduList = pyfits.HDUList(hdu)
hduList.verify()
hduList.writeto(outfits)
except:
raise SaltIOError('Could not create new fits file named '+writenewfits)
# read it back in for updating
hduList = saltio.openfits(outfits,mode='update')
# set up the completeness tracker
if verbose:
j=0
x2=float(j)/float(ntotal)
ctext='Percentage Complete: %3.2f' % x2
sys.stdout.write(ctext)
sys.stdout.flush()
for infile in infiles:
struct=saltio.openfits(infile)
# Skip through the frames and process each frame individually
for i in range(nframes):
if not (infile==infiles[0] and i < ignorexp):
hdu=extension+i*nstep
try:
header=struct[hdu].header
array=struct[hdu].data
array=array*1.0
except Exception as e:
msg='Unable to open extension %i in image %s because %s' % (hdu, infile, e)
raise SaltIOError(msg)
# start the analysis of each frame
# gain and readout noise
try:
gain=float(header['GAIN'])
except:
gain=1
log.warning('Gain not specified in image header')
try:
rdnoise=float(header['RDNOISE'])
except:
rdnoise=0
log.warning('RDNOISE not specified in image header')
# background subtraction
if not subbacktype=='none':
try:
narray=subbackground(array, sigback, mbin, order, niter, subbacktype)
except:
log.warning('Image '+infile+' extention '+str(i)+' is blank, skipping')
continue
# create output array
try:
if imgtype=='background':
oarray=narray-array
else:
oarray=narray
except:
oarray=array
# print progress
if verbose:
x2=float(j)/float(ntotal)
if x2 > ftime:
ctext='\b\b\b\b\b %3.2f' % x2
sys.stdout.write(ctext)
sys.stdout.flush()
ftime += plotfreq
# update the header values with the image name and extension number
try:
hdue = pyfits.ImageHDU(oarray)
hdue.header=header
hdue.header.update('ONAME',infile,'Original image name')
hdue.header.update('OEXT',hdu,'Original extension number')
except Exception as e:
msg='SALTPHOT--WARNING: Could not update image in newfits file for %s ext %i because %s' \
% (infile, hdu, e)
raise SaltIOError(msg)
hduList.append(hdue)
j +=1
# close FITS file
saltio.closefits(struct)
# close the output fits file:
try:
# write out the file
hduList.flush()
hduList.close()
except Exception as e:
raise SaltIOError('Failed to write %s because %s' % (outfits, e))
def make_mosaic(struct, gap, xshift, yshift, rotation, interp_type='linear',
boundary='constant', constant=0, geotran=True, fill=False,
cleanup=True, log=None, verbose=False):
"""Given a SALT image struct, combine each of the individual amplifiers and
apply the geometric CCD transformations to the image
"""
# get the name of the file
infile = saltkey.getimagename(struct[0], base=True)
outpath = './'
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = \
saltkey.instrumid(struct)
# how many amplifiers?
nsciext = saltkey.get('NSCIEXT', struct[0])
nextend = saltkey.get('NEXTEND', struct[0])
nccds = saltkey.get('NCCDS', struct[0])
amplifiers = nccds * 2
if nextend > nsciext:
varframe = True
else:
varframe = False
# CCD geometry coefficients
if (instrume == 'RSS' or instrume == 'PFIS'):
xsh = [0., xshift[0], 0., xshift[1]]
ysh = [0., yshift[0], 0., yshift[1]]
rot = [0., rotation[0], 0., rotation[1]]
elif instrume == 'SALTICAM':
xsh = [0., xshift[0], 0.]
ysh = [0., yshift[0], 0.]
rot = [0., rotation[0], 0]
# how many extensions?
nextend = saltkey.get('NEXTEND', struct[0])
# CCD on-chip binning
xbin, ybin = saltkey.ccdbin(struct[0])
# create temporary primary extension
outstruct = []
outstruct.append(struct[0])
# define temporary FITS file store tiled CCDs
tilefile = saltio.tmpfile(outpath)
tilefile += 'tile.fits'
if varframe:
tilehdu = [None] * (3 * int(nsciext / 2) + 1)
else:
tilehdu = [None] * int(nsciext / 2 + 1)
tilehdu[0] = fits.PrimaryHDU()
#tilehdu[0].header = struct[0].header
if log:
log.message('', with_stdout=verbose)
# iterate over amplifiers, stich them to produce file of CCD images
for i in range(int(nsciext / 2)):
hdu = i * 2 + 1
# amplifier = hdu%amplifiers
# if (amplifier == 0): amplifier = amplifiers
# read DATASEC keywords
datasec1 = saltkey.get('DATASEC', struct[hdu])
datasec2 = saltkey.get('DATASEC', struct[hdu + 1])
xdsec1, ydsec1 = saltstring.secsplit(datasec1)
xdsec2, ydsec2 = saltstring.secsplit(datasec2)
# read images
imdata1 = saltio.readimage(struct, hdu)
imdata2 = saltio.readimage(struct, hdu + 1)
# tile 2n amplifiers to yield n CCD images
outdata = numpy.zeros((ydsec1[1] +
abs(ysh[i +
1] /
ybin), xdsec1[1] +
xdsec2[1] +
abs(xsh[i +
1] /
xbin)), numpy.float32)
# set up the variance frame
if varframe:
vardata = outdata.copy()
vdata1 = saltio.readimage(struct, struct[hdu].header['VAREXT'])
vdata2 = saltio.readimage(struct, struct[hdu + 1].header['VAREXT'])
bpmdata = outdata.copy()
bdata1 = saltio.readimage(struct, struct[hdu].header['BPMEXT'])
bdata2 = saltio.readimage(struct, struct[hdu + 1].header['BPMEXT'])
x1 = xdsec1[0] - 1
if x1 != 0:
msg = 'The data in %s have not been trimmed prior to mosaicking.' \
% infile
log.error(msg)
if xsh[i + 1] < 0:
x1 += abs(xsh[i + 1] / xbin)
x2 = x1 + xdsec1[1]
y1 = ydsec1[0] - 1
if ysh[i + 1] < 0:
y1 += abs(ysh[i + 1] / ybin)
y2 = y1 + ydsec1[1]
outdata[y1:y2, x1:x2] =\
imdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
x1 = x2
x2 = x1 + xdsec2[1]
y1 = ydsec2[0] - 1
if ysh[i + 1] < 0:
y1 += abs(ysh[i + 1] / ybin)
y2 = y1 + ydsec2[1]
outdata[y1:y2, x1:x2] =\
imdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
# size of new image
naxis1 = str(xdsec1[1] + xdsec2[1])
naxis2 = str(ydsec1[1])
# add image and keywords to HDU list
tilehdu[i + 1] = fits.ImageHDU(outdata)
tilehdu[i + 1].header = struct[hdu].header
#tilehdu[
# i + 1].header['DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
if varframe:
vext = i + 1 + int(nsciext / 2.)
tilehdu[vext] = fits.ImageHDU(vardata)
#tilehdu[vext].header = struct[struct[hdu].header['VAREXT']].header
#tilehdu[vext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
bext = i + 1 + 2 * int(nsciext / 2.)
tilehdu[bext] = fits.ImageHDU(bpmdata)
#tilehdu[bext].header = struct[struct[hdu].header['BPMEXT']].header
#tilehdu[bext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
# image tile log message #1
if log:
message = os.path.basename(infile) + '[' + str(hdu) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
message = os.path.basename(infile) + '[' + str(hdu + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[1] + 1) + ':' + \
str(xdsec1[1] + xdsec2[1]) + ','
message += str(ydsec2[0]) + ':' + str(ydsec2[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
# write temporary file of tiled CCDs
hdulist = fits.HDUList(tilehdu)
hdulist.writeto(tilefile)
# iterate over CCDs, transform and rotate images
yrot = [None] * 4
xrot = [None] * 4
tranfile = [' ']
tranhdu = [0]
if varframe:
tranfile = [''] * (3 * int(nsciext / 2) + 1)
tranhdu = [0] * (3 * int(nsciext / 2) + 1)
else:
tranfile = [''] * int(nsciext / 2 + 1)
tranhdu = [0] * int(nsciext / 2 + 1)
# this is hardwired for SALT where the second CCD is considered the
# fiducial
for hdu in range(1, int(nsciext / 2 + 1)):
tranfile[hdu] = saltio.tmpfile(outpath)
tranfile[hdu] += 'tran.fits'
if varframe:
tranfile[hdu + nccds] = saltio.tmpfile(outpath) + 'tran.fits'
tranfile[hdu + 2 * nccds] = saltio.tmpfile(outpath) + 'tran.fits'
ccd = hdu % nccds
if (ccd == 0):
ccd = nccds
# correct rotation for CCD binning
yrot[ccd] = rot[ccd] * ybin / xbin
xrot[ccd] = rot[ccd] * xbin / ybin
dxshift = xbin * int(float(int(gap) / xbin) + 0.5) - gap
# transformation using geotran IRAF task
# if (ccd == 1):
if (ccd != 2):
if geotran:
message = '\nSALTMOSAIC -- geotran ' + tilefile + \
'[' + str(ccd) + '] ' + tranfile[hdu]
message += ' \"\" \"\" xshift=' + \
str((xsh[ccd] + (2 - ccd) * dxshift) / xbin) + ' '
message += 'yshift=' + \
str(ysh[ccd] / ybin) + ' xrotation=' + str(xrot[ccd]) + ' '
message += 'yrotation=' + \
str(yrot[ccd]) + ' xmag=1 ymag=1 xmin=\'INDEF\''
message += 'xmax=\'INDEF\' ymin=\'INDEF\' ymax=\'INDEF\' '
message += 'ncols=\'INDEF\' '
message += 'nlines=\'INDEF\' verbose=\'no\' '
message += 'fluxconserve=\'yes\' nxblock=2048 '
message += 'nyblock=2048 interpolant=\'' + \
interp_type + '\' boundary=\'constant\' constant=0'
log.message(message, with_stdout=verbose)
yd, xd = tilehdu[ccd].data.shape
ncols = 'INDEF' # ncols=xd+abs(xsh[ccd]/xbin)
nlines = 'INDEF' # nlines=yd+abs(ysh[ccd]/ybin)
geo_xshift = xsh[ccd] + (2 - ccd) * dxshift / xbin
geo_yshift = ysh[ccd] / ybin
iraf.images.immatch.geotran(tilefile + "[" + str(ccd) + "]",
tranfile[hdu],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1, ymag=1, xmin='INDEF',
xmax='INDEF', ymin='INDEF',
ymax='INDEF', ncols=ncols,
nlines=nlines, verbose='no',
fluxconserve='yes', nxblock=2048,
nyblock=2048, interpolant="linear",
boundary="constant", constant=0)
if varframe:
var_infile = tilefile + "[" + str(ccd + nccds) + "]"
iraf.images.immatch.geotran(var_infile,
tranfile[hdu + nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1, ymag=1, xmin='INDEF',
xmax='INDEF', ymin='INDEF',
ymax='INDEF', ncols=ncols,
nlines=nlines, verbose='no',
fluxconserve='yes',
nxblock=2048, nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
var2_infile = tilefile + "[" + str(ccd + 2 * nccds) + "]"
iraf.images.immatch.geotran(var2_infile,
tranfile[hdu + 2 * nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1, ymag=1, xmin='INDEF',
xmax='INDEF', ymin='INDEF',
ymax='INDEF', ncols=ncols,
nlines=nlines, verbose='no',
fluxconserve='yes',
nxblock=2048, nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
# open the file and copy the data to tranhdu
tstruct = fits.open(tranfile[hdu])
tranhdu[hdu] = tstruct[0].data
tstruct.close()
if varframe:
tranhdu[
hdu +
nccds] = fits.open(
tranfile[
hdu +
nccds])[0].data
tranhdu[
hdu +
2 *
nccds] = fits.open(
tranfile[
hdu +
2 *
nccds])[0].data
else:
log.message(
"Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd,
xsh[ccd] /
2.0,
ysh[ccd] /
2.0,
xrot[ccd]),
with_stdout=verbose,
with_header=False)
tranhdu[hdu] = geometric_transform(
tilehdu[ccd].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(
xsh[ccd] / 2,
ysh[ccd] / 2,
1,
1,
xrot[ccd],
yrot[ccd]))
tstruct = fits.PrimaryHDU(tranhdu[hdu])
tstruct.writeto(tranfile[hdu])
if varframe:
tranhdu[hdu + nccds] = geometric_transform(
tilehdu[hdu + 3].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(
xsh[ccd] / 2, ysh[ccd] / 2,
1, 1,
xrot[ccd], yrot[ccd]))
tranhdu[hdu + 2 * nccds] = geometric_transform(
tilehdu[hdu + 6].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(
xsh[ccd] / 2, ysh[ccd] / 2,
1, 1,
xrot[ccd], yrot[ccd]))
else:
log.message(
"Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd, 0, 0, 0), with_stdout=verbose, with_header=False)
tranhdu[hdu] = tilehdu[ccd].data
if varframe:
tranhdu[hdu + nccds] = tilehdu[ccd + nccds].data
tranhdu[hdu + 2 * nccds] = tilehdu[ccd + 2 * nccds].data
# open outfile
if varframe:
outlist = 4 * [None]
else:
outlist = 2 * [None]
#outlist[0] = struct[0].copy()
outlist[0] = fits.PrimaryHDU()
outlist[0].header = struct[0].header
naxis1 = int(gap / xbin * (nccds - 1))
naxis2 = 0
for i in range(1, nccds + 1):
yw, xw = tranhdu[i].shape
naxis1 += xw + int(abs(xsh[ccd] / xbin)) + 1
naxis2 = max(naxis2, yw)
outdata = numpy.zeros((naxis2, naxis1), numpy.float32)
outdata.shape = naxis2, naxis1
if varframe:
vardata = outdata * 0
bpmdata = outdata * 0 + 1
# iterate over CCDs, stich them to produce a full image
hdu = 0
totxshift = 0
for hdu in range(1, nccds + 1):
# read DATASEC keywords
ydsec, xdsec = tranhdu[hdu].shape
# define size and shape of final image
# tile CCDs to yield mosaiced image
x1 = int((hdu - 1) * (xdsec + gap / xbin)) + int(totxshift)
x2 = xdsec + x1
y1 = int(0)
y2 = int(ydsec)
outdata[y1:y2, x1:x2] = tranhdu[hdu]
totxshift += int(abs(xsh[hdu] / xbin)) + 1
if varframe:
vardata[y1:y2, x1:x2] = tranhdu[hdu + nccds]
bpmdata[y1:y2, x1:x2] = tranhdu[hdu + 2 * nccds]
# make sure to cover up all the gaps include bad areas
if varframe:
baddata = (outdata == 0)
baddata = nd.maximum_filter(baddata, size=3)
bpmdata[baddata] = 1
# fill in the gaps if requested
if fill:
if varframe:
outdata = fill_gaps(outdata, 0)
else:
outdata = fill_gaps(outdata, 0)
# add to the file
outlist[1] = fits.ImageHDU(outdata)
if varframe:
outlist[2] = fits.ImageHDU(vardata,name='VAR')
outlist[3] = fits.ImageHDU(bpmdata,name='BPM')
# create the image structure
outstruct = fits.HDUList(outlist)
# update the head informaation
# housekeeping keywords
saltkey.put('NEXTEND', 2, outstruct[0])
saltkey.new('EXTNAME', 'SCI', 'Extension name', outstruct[1])
saltkey.new('EXTVER', 1, 'Extension number', outstruct[1])
if varframe:
saltkey.new('VAREXT', 2, 'Variance frame extension', outstruct[1])
saltkey.new('BPMEXT', 3, 'BPM Extension', outstruct[1])
try:
saltkey.copy(struct[1], outstruct[1], 'CCDSUM')
except:
pass
# Add keywords associated with geometry
saltkey.new('SGEOMGAP', gap, 'SALT Chip Gap', outstruct[0])
c1str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[0],
yshift[0],
rotation[0])
saltkey.new('SGEOM1', c1str, 'SALT Chip 1 Transform', outstruct[0])
c2str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[1],
yshift[1],
rotation[1])
saltkey.new('SGEOM2', c2str, 'SALT Chip 2 Transform', outstruct[0])
# WCS keywords
saltkey.new('CRPIX1', 0, 'WCS: X reference pixel', outstruct[1])
saltkey.new('CRPIX2', 0, 'WCS: Y reference pixel', outstruct[1])
saltkey.new(
'CRVAL1',
float(xbin),
'WCS: X reference coordinate value',
outstruct[1])
saltkey.new(
'CRVAL2',
float(ybin),
'WCS: Y reference coordinate value',
outstruct[1])
saltkey.new('CDELT1', float(xbin), 'WCS: X pixel size', outstruct[1])
saltkey.new('CDELT2', float(ybin), 'WCS: Y pixel size', outstruct[1])
saltkey.new('CTYPE1', 'pixel', 'X type', outstruct[1])
saltkey.new('CTYPE2', 'pixel', 'Y type', outstruct[1])
# cleanup temporary files
if cleanup:
for tfile in tranfile:
if os.path.isfile(tfile):
saltio.delete(tfile)
if os.path.isfile(tilefile):
status = saltio.delete(tilefile)
# return the file
return outstruct
def saltfpringfit(images, outfile, section=None, bthresh=5, niter=5,
displayimage=True, clobber=True,logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# read in the section
if section is None:
section=saltio.getSection(section)
msg='This mode is not supported yet'
raise SaltError(msg)
else:
section=saltio.getSection(section)
print section
# open each raw image file
for img in infiles:
#open the fits file
struct=saltio.openfits(img)
data=struct[0].data
#only keep the bright pixels
y1,y2,x1,x2=section
bmean, bmedian, bstd=iterstat(data[y1:y2,x1:x2], sig=bthresh, niter=niter, verbose=False)
message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \
('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd)
log.message(message, with_stdout=verbose)
mdata=data*(data-bmean>bthresh*bstd)
#prepare the first guess for the image
ring_list=findrings(data, thresh=5, niter=5, minsize=10)
if displayimage:
regfile=img.replace('.fits', '.reg')
print regfile
if clobber and os.path.isfile(regfile): fout=saltio.delete(regfile)
fout=open(regfile, 'w')
fout.write("""# Region file format: DS9 version 4.1
# Filename: %s
global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1
physical
""" % img)
for ring in ring_list:
print ring
fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad))
fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad-5*ring.sigma))
fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad+5*ring.sigma))
fout.close()
display(img, catname=regfile, rformat='reg')
#write out the result for viewing
struct[0].data=mdata
saltio.writefits(struct, 'out.fits', clobber=True)
message = 'Ring Parameters'
log.message(message)
def vid2fits(inhead, inbin,outfile, config):
"""
Convert bin files made during the video process to
regular fits files
Format python vid2fits.py inhead inbin outfits config
Returns
"""
#Check that the input files exists
saltio.fileexists(inhead)
saltio.fileexists(inbin)
saltio.fileexists(config)
#if output file exists, then delete
if os.path.isfile(outfile): saltio.delete(outfile)
#read in and process the config file
condict=fitsconfig(config)
#read in the header information
infits=saltio.openfits(inhead)
inheader = infits['Primary'].header
instrume=inheader['INSTRUME']
detswv=softwareversion(inheader['DETSWV'])
#create a new image and copy the header to it
try:
hdu = fits.PrimaryHDU()
hdu.header=inheader
hduList = fits.HDUList(hdu)
#hduList.verify()
hduList.writeto(outfile, output_verify='ignore')
except:
message = 'ERROR -- VID2FIT: Could not create new fits file'
raise SaltError(message)
#Now open up the file that you just made and update it from here on
hduList = fits.open(outfile, mode='update')
#open the binary file
bindata = saltio.openbinary(inbin,'rb')
#read in header information from binary file
#some constants that are needed for reading in the binary data
sizeofinteger=struct.calcsize('i')
sizeofunsignshort=struct.calcsize('H')
sizeofdouble=struct.calcsize('d')
sizeoffloat =struct.calcsize('f')
#read in the number of exposures, geometry of image (width and height) and number of amps
nframes= saltio.readbinary(bindata,sizeofinteger,"=i")
if detswv<=4.78 and instrume=='SALTICAM':
fwidth= saltio.readbinary(bindata,sizeofinteger,"=i")
fheight= saltio.readbinary(bindata,sizeofinteger, "=i")
elif (detswv>=7.01 and instrume=='SALTICAM') or (detswv>=4.37 and instrume=='RSS'):
fwidth= saltio.readbinary(bindata,sizeofunsignshort,"=H")
fheight= saltio.readbinary(bindata,sizeofunsignshort, "=H")
pbcols=saltio.readbinary(bindata,sizeofunsignshort, "=H")
pbrows=saltio.readbinary(bindata,sizeofunsignshort,"=H")
else:
message='VID2FITS--Detector Software version %s is not supported' % detswv
raise SaltError(message)
nelements=fwidth*fheight
namps=saltio.readbinary(bindata,sizeofinteger,"=i")
#read in the gain
gain = numpy.zeros(namps,dtype=float)
for i in range(namps):
gain[i]=saltio.readbinary(bindata,sizeofdouble,"=d")
#read in the rdnoise
rdnoise = numpy.zeros(namps, dtype=float)
for i in range(namps):
rdnoise[i]=saltio.readbinary(bindata,sizeofdouble,"=d")
#set the scale parameters
bzero=32768
bscale=1
otime=0
#start the loop to read in the data
for i in range(nframes):
#read in the start of the data,time
starttime= saltio.readbinary(bindata,sizeofdouble,"=d")
date_obs, time_obs= ascii_time(starttime)
#read in the exposure time
exptime= saltio.readbinary(bindata,sizeofdouble,"=d")
#read in the dead time in milliseconds
if (detswv>=7.01 and instrume=='SALTICAM') or (detswv>=4.37 and instrume=='RSS'):
deadtime= saltio.readbinary(bindata,sizeofinteger,"=i")
else:
deadtime=None
#read in the dead time
if (detswv>=7.01 and instrume=='SALTICAM') or (detswv>=4.37 and instrume=='RSS'):
framecnt= saltio.readbinary(bindata,sizeofinteger,"=i")
else:
framecnt=None
otime=starttime
#read in the data
shape = (fheight,fwidth)
imdata = numpy.fromfile(bindata,dtype=numpy.ushort,count=nelements)
imdata = imdata.reshape(shape)
#for each amplifier write it to the image
if namps > 0: awidth=fwidth/namps
for j in range(namps):
###create the new extension ###
#cut each image by the number of amplifiers
y1=j*awidth
y2=y1+awidth
data = imdata[:,y1:y2].astype(numpy.int16)
hdue = fits.ImageHDU(data)
hdue.scale('int16','',bzero=bzero)
#set the header values
datasec,detsec,ccdsec,ampsec,biassec= \
create_header_values(condict,hdu,j,fheight)
#fill in the header data
hdue = write_ext_header(hdue,outfile,hdu,time_obs,date_obs,bscale,bzero, \
exptime,gain[j],rdnoise[j],datasec,detsec,ccdsec,ampsec, \
biassec, deadtime=deadtime, framecnt=framecnt )
#append the extension to the image
hduList.append(hdue)
try:
hduList.flush()
hduList.close()
except Exception, e:
message = 'ERROR: VID2BIN -- Fail to convert %s due to %s' % (outfile, e)
raise SaltError(message)
def hrsclean(images, outpath, obslogfile=None, subover=True, trim=True, masbias=None,
subbias=True, median=False, function='polynomial', order=5, rej_lo=3, rej_hi=3,
niter=5, interp='linear', clobber=False, logfile='salt.log',verbose=True):
"""Convert MEF HRS data into a single image. If variance frames and BPMs, then
convert them to the same format as well. Returns an MEF image but that is
combined into a single frame
"""
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
if saltio.checkfornone(obslogfile) is None:
raise SaltError('Obslog file is required')
# Delete the obslog file if it already exists
if (os.path.isfile(obslogfile) and clobber) or not os.path.isfile(obslogfile):
if os.path.isfile(obslogfile): saltio.delete(obslogfile)
#read in the obsveration log or create it
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
else:
obsstruct=saltio.openfits(obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('OBJECT')
biaslist=filename[ccdtype=='Bias']
masterbias_dict={}
if log: log.message('Processing Bias Frames')
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bgph'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose, with_header=False)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
subover=subover, trim=trim, subbias=False, imstack=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist)
saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=hrsbiasheader_list)
#create the master bias frame
for i in list(masterbias_dict.keys()):
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys, x1=5, x2=13)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#apply full reductions to the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'mbgph'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#get master bias frame
masterbias=get_masterbias(struct, masterbias_dict, keylist=hrsbiasheader_list)
if masterbias:
subbias=True
bstruct=saltio.openfits(masterbias)
else:
subbias=False
bstruct=None
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
subover=subover, trim=trim, subbias=subbias, imstack=True,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'HPREPARE', 'Images have been prepared', hist)
saltkey.new('HGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
return
def saltfpringfit(images,
outfile,
section=None,
bthresh=5,
niter=5,
displayimage=True,
clobber=True,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# read in the section
if section is None:
section = saltio.getSection(section)
msg = 'This mode is not supported yet'
raise SaltError(msg)
else:
section = saltio.getSection(section)
print section
# open each raw image file
for img in infiles:
#open the fits file
struct = saltio.openfits(img)
data = struct[0].data
#only keep the bright pixels
y1, y2, x1, x2 = section
bmean, bmedian, bstd = iterstat(data[y1:y2, x1:x2],
sig=bthresh,
niter=niter,
verbose=False)
message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \
('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd)
log.message(message, with_stdout=verbose)
mdata = data * (data - bmean > bthresh * bstd)
#prepare the first guess for the image
ring_list = findrings(data, thresh=5, niter=5, minsize=10)
if displayimage:
regfile = img.replace('.fits', '.reg')
print regfile
if clobber and os.path.isfile(regfile):
fout = saltio.delete(regfile)
fout = open(regfile, 'w')
fout.write("""# Region file format: DS9 version 4.1
# Filename: %s
global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1
physical
""" % img)
for ring in ring_list:
print ring
fout.write('circle(%f, %f, %f)\n' %
(ring.xc, ring.yc, ring.prad))
fout.write('circle(%f, %f, %f)\n' %
(ring.xc, ring.yc, ring.prad - 5 * ring.sigma))
fout.write('circle(%f, %f, %f)\n' %
(ring.xc, ring.yc, ring.prad + 5 * ring.sigma))
fout.close()
display(img, catname=regfile, rformat='reg')
#write out the result for viewing
struct[0].data = mdata
saltio.writefits(struct, 'out.fits', clobber=True)
message = 'Ring Parameters'
log.message(message)
def make_mosaic(struct,
gap,
xshift,
yshift,
rotation,
interp_type='linear',
boundary='constant',
constant=0,
geotran=True,
fill=False,
cleanup=True,
log=None,
verbose=False):
"""Given a SALT image struct, combine each of the individual amplifiers and
apply the geometric CCD transformations to the image
"""
# get the name of the file
infile = saltkey.getimagename(struct[0], base=True)
outpath = './'
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = \
saltkey.instrumid(struct)
# how many amplifiers?
nsciext = saltkey.get('NSCIEXT', struct[0])
nextend = saltkey.get('NEXTEND', struct[0])
nccds = saltkey.get('NCCDS', struct[0])
amplifiers = nccds * 2
if nextend > nsciext:
varframe = True
else:
varframe = False
# CCD geometry coefficients
if (instrume == 'RSS' or instrume == 'PFIS'):
xsh = [0., xshift[0], 0., xshift[1]]
ysh = [0., yshift[0], 0., yshift[1]]
rot = [0., rotation[0], 0., rotation[1]]
elif instrume == 'SALTICAM':
xsh = [0., xshift[0], 0.]
ysh = [0., yshift[0], 0.]
rot = [0., rotation[0], 0]
# how many extensions?
nextend = saltkey.get('NEXTEND', struct[0])
# CCD on-chip binning
xbin, ybin = saltkey.ccdbin(struct[0])
# create temporary primary extension
outstruct = []
outstruct.append(struct[0])
# define temporary FITS file store tiled CCDs
tilefile = saltio.tmpfile(outpath)
tilefile += 'tile.fits'
if varframe:
tilehdu = [None] * (3 * int(nsciext / 2) + 1)
else:
tilehdu = [None] * int(nsciext / 2 + 1)
tilehdu[0] = fits.PrimaryHDU()
#tilehdu[0].header = struct[0].header
if log:
log.message('', with_stdout=verbose)
# iterate over amplifiers, stich them to produce file of CCD images
for i in range(int(nsciext / 2)):
hdu = i * 2 + 1
# amplifier = hdu%amplifiers
# if (amplifier == 0): amplifier = amplifiers
# read DATASEC keywords
datasec1 = saltkey.get('DATASEC', struct[hdu])
datasec2 = saltkey.get('DATASEC', struct[hdu + 1])
xdsec1, ydsec1 = saltstring.secsplit(datasec1)
xdsec2, ydsec2 = saltstring.secsplit(datasec2)
# read images
imdata1 = saltio.readimage(struct, hdu)
imdata2 = saltio.readimage(struct, hdu + 1)
# tile 2n amplifiers to yield n CCD images
outdata = numpy.zeros(
(int(ydsec1[1] + abs(ysh[i + 1] / ybin)),
int(xdsec1[1] + xdsec2[1] + abs(xsh[i + 1] / xbin))),
numpy.float32)
# set up the variance frame
if varframe:
vardata = outdata.copy()
vdata1 = saltio.readimage(struct, struct[hdu].header['VAREXT'])
vdata2 = saltio.readimage(struct, struct[hdu + 1].header['VAREXT'])
bpmdata = outdata.copy()
bdata1 = saltio.readimage(struct, struct[hdu].header['BPMEXT'])
bdata2 = saltio.readimage(struct, struct[hdu + 1].header['BPMEXT'])
x1 = xdsec1[0] - 1
if x1 != 0:
msg = 'The data in %s have not been trimmed prior to mosaicking.' \
% infile
log.error(msg)
if xsh[i + 1] < 0:
x1 += int(abs(xsh[i + 1] / xbin))
x2 = x1 + xdsec1[1]
y1 = ydsec1[0] - 1
if ysh[i + 1] < 0:
y1 += int(abs(ysh[i + 1] / ybin))
y2 = y1 + ydsec1[1]
outdata[y1:y2, x1:x2] =\
imdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
x1 = x2
x2 = x1 + xdsec2[1]
y1 = ydsec2[0] - 1
if ysh[i + 1] < 0:
y1 += abs(ysh[i + 1] / ybin)
y2 = y1 + ydsec2[1]
outdata[y1:y2, x1:x2] =\
imdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
# size of new image
naxis1 = str(xdsec1[1] + xdsec2[1])
naxis2 = str(ydsec1[1])
# add image and keywords to HDU list
tilehdu[i + 1] = fits.ImageHDU(outdata)
tilehdu[i + 1].header = struct[hdu].header
#tilehdu[
# i + 1].header['DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
if varframe:
vext = i + 1 + int(nsciext / 2.)
tilehdu[vext] = fits.ImageHDU(vardata)
#tilehdu[vext].header = struct[struct[hdu].header['VAREXT']].header
#tilehdu[vext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
bext = i + 1 + 2 * int(nsciext / 2.)
tilehdu[bext] = fits.ImageHDU(bpmdata)
#tilehdu[bext].header = struct[struct[hdu].header['BPMEXT']].header
#tilehdu[bext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
# image tile log message #1
if log:
message = os.path.basename(infile) + '[' + str(hdu) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
message = os.path.basename(infile) + '[' + str(hdu + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[1] + 1) + ':' + \
str(xdsec1[1] + xdsec2[1]) + ','
message += str(ydsec2[0]) + ':' + str(ydsec2[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
# write temporary file of tiled CCDs
hdulist = fits.HDUList(tilehdu)
hdulist.writeto(tilefile)
# iterate over CCDs, transform and rotate images
yrot = [None] * 4
xrot = [None] * 4
tranfile = [' ']
tranhdu = [0]
if varframe:
tranfile = [''] * (3 * int(nsciext / 2) + 1)
tranhdu = [0] * (3 * int(nsciext / 2) + 1)
else:
tranfile = [''] * int(nsciext / 2 + 1)
tranhdu = [0] * int(nsciext / 2 + 1)
# this is hardwired for SALT where the second CCD is considered the
# fiducial
for hdu in range(1, int(nsciext / 2 + 1)):
tranfile[hdu] = saltio.tmpfile(outpath)
tranfile[hdu] += 'tran.fits'
if varframe:
tranfile[hdu + nccds] = saltio.tmpfile(outpath) + 'tran.fits'
tranfile[hdu + 2 * nccds] = saltio.tmpfile(outpath) + 'tran.fits'
ccd = hdu % nccds
if (ccd == 0):
ccd = nccds
# correct rotation for CCD binning
yrot[ccd] = rot[ccd] * ybin / xbin
xrot[ccd] = rot[ccd] * xbin / ybin
dxshift = xbin * int(float(int(gap) / xbin) + 0.5) - gap
# transformation using geotran IRAF task
# if (ccd == 1):
if (ccd != 2):
if geotran:
message = '\nSALTMOSAIC -- geotran ' + tilefile + \
'[' + str(ccd) + '] ' + tranfile[hdu]
message += ' \"\" \"\" xshift=' + \
str((xsh[ccd] + (2 - ccd) * dxshift) / xbin) + ' '
message += 'yshift=' + \
str(ysh[ccd] / ybin) + ' xrotation=' + str(xrot[ccd]) + ' '
message += 'yrotation=' + \
str(yrot[ccd]) + ' xmag=1 ymag=1 xmin=\'INDEF\''
message += 'xmax=\'INDEF\' ymin=\'INDEF\' ymax=\'INDEF\' '
message += 'ncols=\'INDEF\' '
message += 'nlines=\'INDEF\' verbose=\'no\' '
message += 'fluxconserve=\'yes\' nxblock=2048 '
message += 'nyblock=2048 interpolant=\'' + \
interp_type + '\' boundary=\'constant\' constant=0'
log.message(message, with_stdout=verbose)
yd, xd = tilehdu[ccd].data.shape
ncols = 'INDEF' # ncols=xd+abs(xsh[ccd]/xbin)
nlines = 'INDEF' # nlines=yd+abs(ysh[ccd]/ybin)
geo_xshift = xsh[ccd] + (2 - ccd) * dxshift / xbin
geo_yshift = ysh[ccd] / ybin
iraf.images.immatch.geotran(tilefile + "[" + str(ccd) + "]",
tranfile[hdu],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1,
ymag=1,
xmin='INDEF',
xmax='INDEF',
ymin='INDEF',
ymax='INDEF',
ncols=ncols,
nlines=nlines,
verbose='no',
fluxconserve='yes',
nxblock=2048,
nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
if varframe:
var_infile = tilefile + "[" + str(ccd + nccds) + "]"
iraf.images.immatch.geotran(var_infile,
tranfile[hdu + nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1,
ymag=1,
xmin='INDEF',
xmax='INDEF',
ymin='INDEF',
ymax='INDEF',
ncols=ncols,
nlines=nlines,
verbose='no',
fluxconserve='yes',
nxblock=2048,
nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
var2_infile = tilefile + "[" + str(ccd + 2 * nccds) + "]"
iraf.images.immatch.geotran(var2_infile,
tranfile[hdu + 2 * nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1,
ymag=1,
xmin='INDEF',
xmax='INDEF',
ymin='INDEF',
ymax='INDEF',
ncols=ncols,
nlines=nlines,
verbose='no',
fluxconserve='yes',
nxblock=2048,
nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
# open the file and copy the data to tranhdu
tstruct = fits.open(tranfile[hdu])
tranhdu[hdu] = tstruct[0].data
tstruct.close()
if varframe:
tranhdu[hdu + nccds] = fits.open(tranfile[hdu +
nccds])[0].data
tranhdu[hdu + 2 * nccds] = fits.open(
tranfile[hdu + 2 * nccds])[0].data
else:
log.message("Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd, xsh[ccd] / 2.0, ysh[ccd] / 2.0, xrot[ccd]),
with_stdout=verbose,
with_header=False)
tranhdu[hdu] = geometric_transform(
tilehdu[ccd].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1,
xrot[ccd], yrot[ccd]))
tstruct = fits.PrimaryHDU(tranhdu[hdu])
tstruct.writeto(tranfile[hdu])
if varframe:
tranhdu[hdu + nccds] = geometric_transform(
tilehdu[hdu + 3].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1,
xrot[ccd], yrot[ccd]))
tranhdu[hdu + 2 * nccds] = geometric_transform(
tilehdu[hdu + 6].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1,
xrot[ccd], yrot[ccd]))
else:
log.message("Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd, 0, 0, 0),
with_stdout=verbose,
with_header=False)
tranhdu[hdu] = tilehdu[ccd].data
if varframe:
tranhdu[hdu + nccds] = tilehdu[ccd + nccds].data
tranhdu[hdu + 2 * nccds] = tilehdu[ccd + 2 * nccds].data
# open outfile
if varframe:
outlist = 4 * [None]
else:
outlist = 2 * [None]
#outlist[0] = struct[0].copy()
outlist[0] = fits.PrimaryHDU()
outlist[0].header = struct[0].header
naxis1 = int(gap / xbin * (nccds - 1))
naxis2 = 0
for i in range(1, nccds + 1):
yw, xw = tranhdu[i].shape
naxis1 += xw + int(abs(xsh[ccd] / xbin)) + 1
naxis2 = max(naxis2, yw)
outdata = numpy.zeros((naxis2, naxis1), numpy.float32)
outdata.shape = naxis2, naxis1
if varframe:
vardata = outdata * 0
bpmdata = outdata * 0 + 1
# iterate over CCDs, stich them to produce a full image
hdu = 0
totxshift = 0
for hdu in range(1, nccds + 1):
# read DATASEC keywords
ydsec, xdsec = tranhdu[hdu].shape
# define size and shape of final image
# tile CCDs to yield mosaiced image
x1 = int((hdu - 1) * (xdsec + gap / xbin)) + int(totxshift)
x2 = xdsec + x1
y1 = int(0)
y2 = int(ydsec)
outdata[y1:y2, x1:x2] = tranhdu[hdu]
totxshift += int(abs(xsh[hdu] / xbin)) + 1
if varframe:
vardata[y1:y2, x1:x2] = tranhdu[hdu + nccds]
bpmdata[y1:y2, x1:x2] = tranhdu[hdu + 2 * nccds]
# make sure to cover up all the gaps include bad areas
if varframe:
baddata = (outdata == 0)
baddata = nd.maximum_filter(baddata, size=3)
bpmdata[baddata] = 1
# fill in the gaps if requested
if fill:
if varframe:
outdata = fill_gaps(outdata, 0)
else:
outdata = fill_gaps(outdata, 0)
# add to the file
outlist[1] = fits.ImageHDU(outdata)
if varframe:
outlist[2] = fits.ImageHDU(vardata, name='VAR')
outlist[3] = fits.ImageHDU(bpmdata, name='BPM')
# create the image structure
outstruct = fits.HDUList(outlist)
# update the head informaation
# housekeeping keywords
saltkey.put('NEXTEND', 2, outstruct[0])
saltkey.new('EXTNAME', 'SCI', 'Extension name', outstruct[1])
saltkey.new('EXTVER', 1, 'Extension number', outstruct[1])
if varframe:
saltkey.new('VAREXT', 2, 'Variance frame extension', outstruct[1])
saltkey.new('BPMEXT', 3, 'BPM Extension', outstruct[1])
try:
saltkey.copy(struct[1], outstruct[1], 'CCDSUM')
except:
pass
# Add keywords associated with geometry
saltkey.new('SGEOMGAP', gap, 'SALT Chip Gap', outstruct[0])
c1str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[0], yshift[0], rotation[0])
saltkey.new('SGEOM1', c1str, 'SALT Chip 1 Transform', outstruct[0])
c2str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[1], yshift[1], rotation[1])
saltkey.new('SGEOM2', c2str, 'SALT Chip 2 Transform', outstruct[0])
# WCS keywords
saltkey.new('CRPIX1', 0, 'WCS: X reference pixel', outstruct[1])
saltkey.new('CRPIX2', 0, 'WCS: Y reference pixel', outstruct[1])
saltkey.new('CRVAL1', float(xbin), 'WCS: X reference coordinate value',
outstruct[1])
saltkey.new('CRVAL2', float(ybin), 'WCS: Y reference coordinate value',
outstruct[1])
saltkey.new('CDELT1', float(xbin), 'WCS: X pixel size', outstruct[1])
saltkey.new('CDELT2', float(ybin), 'WCS: Y pixel size', outstruct[1])
saltkey.new('CTYPE1', 'pixel', 'X type', outstruct[1])
saltkey.new('CTYPE2', 'pixel', 'Y type', outstruct[1])
# cleanup temporary files
if cleanup:
for tfile in tranfile:
if os.path.isfile(tfile):
saltio.delete(tfile)
if os.path.isfile(tilefile):
status = saltio.delete(tilefile)
# return the file
return outstruct
def saltadvance(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', sdbhost='',sdbname='',sdbuser='', password='',
clobber=False, cleanup=True, logfile='salt.log', verbose=True):
"""SALTADVANCE provides advanced data reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
infiles.sort()
# create list of output files
outpath=saltio.abspath(outpath)
#log into the database
sdb=saltmysql.connectdb(sdbhost, sdbname, sdbuser, password)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
obsmode=obsstruct.data.field('OBSMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
propcode=obsstruct.data.field('PROPID')
masktype=obsstruct.data.field('MASKTYP')
#set the bias list of objects
biaslist=filename[(ccdtype=='ZERO')*(propcode=='CAL_BIAS')]
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=fits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=fits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the arc data
arclist=filename[(ccdtype=='ARC') * (obsmode=='SPECTROSCOPY') * (masktype=='LONGSLIT')]
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if nimg in arclist:
#open the image
struct=fits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
obsdate=os.path.basename(img)[1:9]
#print the message
if log:
message='Processing ARC frame %s' % img
log.message(message, with_stdout=verbose)
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
log=log, verbose=verbose)
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the images
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#measure the arcdata
arcimage=outpath+'mbxgp'+nimg
dbfile=outpath+obsdate+'_specid.db'
lamp = obsstruct.data.field('LAMPID')[i]
lamp = lamp.replace(' ', '')
lampfile = iraf.osfn("pysalt$data/linelists/%s.salt" % lamp)
print arcimage, lampfile, os.getcwd()
specidentify(arcimage, lampfile, dbfile, guesstype='rss',
guessfile='', automethod='Matchlines', function='legendre',
order=3, rstep=100, rstart='middlerow', mdiff=20, thresh=3,
startext=0, niter=5, smooth=3, inter=False, clobber=True, logfile=logfile,
verbose=verbose)
try:
ximg = outpath+'xmbxgp'+os.path.basename(arcimage)
specrectify(images=arcimage, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except:
pass
#process the science data
for i, img in enumerate(infiles):
nimg=os.path.basename(img)
if not (nimg in flatlist or nimg in biaslist or nimg in arclist):
#open the image
struct=fits.open(img)
if struct[0].header['PROPID'].count('CAL_GAIN'): continue
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#Check to see if it is RSS 2x2 and add bias subtraction
instrume=saltkey.get('INSTRUME', struct[0]).strip()
gainset = saltkey.get('GAINSET', struct[0])
rospeed = saltkey.get('ROSPEED', struct[0])
target = saltkey.get('OBJECT', struct[0]).strip()
exptime = saltkey.get('EXPTIME', struct[0])
obsmode = saltkey.get('OBSMODE', struct[0]).strip()
detmode = saltkey.get('DETMODE', struct[0]).strip()
masktype = saltkey.get('MASKTYP', struct[0]).strip()
xbin, ybin = saltkey.ccdbin( struct[0], img)
obsdate=os.path.basename(img)[1:9]
bstruct=None
crtype=None
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=5
subbias=False
if instrume=='RSS' and gainset=='FAINT' and rospeed=='SLOW':
bfile='P%sBiasNM%ix%iFASL.fits' % (obsdate, xbin, ybin)
if os.path.exists(bfile):
bstruct=fits.open(bfile)
subbias=True
if detmode=='Normal' and target!='ARC' and xbin < 5 and ybin < 5:
crtype='edge'
thresh=5
mbox=11
bthresh=5.0,
flux_ratio=0.2
bbox=25
gain=1.0
rdnoise=5.0
fthresh=5.0
bfactor=2
gbox=3
maxiter=3
#process the image
struct=clean(struct, createvar=True, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=subbias,
bstruct=bstruct, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover,
crtype=crtype,thresh=thresh,mbox=mbox, bbox=bbox, \
bthresh=bthresh, flux_ratio=flux_ratio, gain=gain, rdnoise=rdnoise,
bfactor=bfactor, fthresh=fthresh, gbox=gbox, maxiter=maxiter,
log=log, verbose=verbose)
#update the database
updatedq(os.path.basename(img), struct, sdb)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,fill=True, cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
#if the file is spectroscopic mode, apply the wavelength correction
if obsmode == 'SPECTROSCOPY' and masktype.strip()=='LONGSLIT':
dbfile=outpath+obsdate+'_specid.db'
try:
ximg = outpath+'xmbxgp'+os.path.basename(img)
specrectify(images=mimg, outimages=ximg, outpref='', solfile=dbfile, caltype='line',
function='legendre', order=3, inttype='interp', w1=None, w2=None, dw=None,
nw=None, blank=0.0, conserve=True, nearest=True, clobber=True,
logfile=logfile, verbose=verbose)
except Exception, e:
log.message('%s' % e)
#clean up the results
if cleanup:
#clean up the bias frames
for i in masterbias_dict.keys():
blist=masterbias_dict[i][1:]
for b in blist: saltio.delete(b)
#clean up the flat frames
for i in masterflat_dict.keys():
flist=masterflat_dict[i][1:]
for f in flist: saltio.delete(f)
def saltclean(images, outpath, obslogfile=None, gaindb=None,xtalkfile=None,
geomfile=None,subover=True,trim=True,masbias=None,
subbias=False, median=False, function='polynomial', order=5,rej_lo=3,
rej_hi=3,niter=5,interp='linear', clobber=False, logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover=False
#start logging
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
# create list of output files
outpath=saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist= saltio.readgaindb(gaindb)
else:
dblist=[]
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict=None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg='The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict=obslog(infiles, log)
obsstruct=createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename=obsstruct.data.field('FILENAME')
detmode=obsstruct.data.field('DETMODE')
ccdtype=obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist=filename[ccdtype=='ZERO']
masterbias_dict={}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct=pyfits.open(img)
bimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict=compareimages(struct, bimg, masterbias_dict, keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys=masterbias_dict[i][0]
blist=masterbias_dict[i][1:]
mbiasname=outpath+createmasterbiasname(blist, bkeys)
bfiles=','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist=filename[ccdtype=='FLAT']
masterflat_dict={}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct=pyfits.open(img)
fimg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict=compareimages(struct, fimg, masterflat_dict, keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys=masterflat_dict[i][0]
flist=masterflat_dict[i][1:]
mflatname=outpath+createmasterflatname(flist, fkeys)
ffiles=','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg=os.path.basename(img)
if not nimg in flatlist or not nimg in biaslist:
#open the image
struct=pyfits.open(img)
simg=outpath+'bxgp'+os.path.basename(img)
#print the message
if log:
message='Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct=clean(struct, createvar=False, badpixelstruct=None, mult=True,
dblist=dblist, xdict=xdict, subover=subover, trim=trim, subbias=False,
bstruct=None, median=median, function=function, order=order,
rej_lo=rej_lo, rej_hi=rej_hi, niter=niter, plotover=plotover, log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist=history(level=1, wrap=False, exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0],'SPREPARE', 'Images have been prepared', hist)
saltkey.new('SGAIN',time.asctime(time.localtime()),'Images have been gain corrected',struct[0])
saltkey.new('SXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('SBIAS',time.asctime(time.localtime()),'Images have been de-biased',struct[0])
# write FITS file
saltio.writefits(struct,simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg=outpath+'mbxgp'+os.path.basename(img)
saltmosaic(images=simg, outimages=mimg,outpref='',geomfile=geomfile,
interp=interp,cleanup=True,clobber=clobber,logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def hrsclean(images,
outpath,
obslogfile=None,
subover=True,
trim=True,
masbias=None,
subbias=True,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""Convert MEF HRS data into a single image. If variance frames and BPMs, then
convert them to the same format as well. Returns an MEF image but that is
combined into a single frame
"""
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
if saltio.checkfornone(obslogfile) is None:
raise SaltError('Obslog file is required')
# Delete the obslog file if it already exists
if (os.path.isfile(obslogfile)
and clobber) or not os.path.isfile(obslogfile):
if os.path.isfile(obslogfile): saltio.delete(obslogfile)
#read in the obsveration log or create it
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
else:
obsstruct = saltio.openfits(obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('OBJECT')
biaslist = filename[ccdtype == 'Bias']
masterbias_dict = {}
if log: log.message('Processing Bias Frames')
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = pyfits.open(img)
bimg = outpath + 'bgph' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message,
with_stdout=verbose,
with_header=False)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
subover=subover,
trim=trim,
subbias=False,
imstack=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'HPREPARE',
'Images have been prepared', hist)
saltkey.new('HGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=hrsbiasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(
blist, bkeys, x1=5, x2=13)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#apply full reductions to the science data
for img in infiles:
nimg = os.path.basename(img)
if not nimg in biaslist:
#open the image
struct = pyfits.open(img)
simg = outpath + 'mbgph' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#get master bias frame
masterbias = get_masterbias(struct,
masterbias_dict,
keylist=hrsbiasheader_list)
if masterbias:
subbias = True
bstruct = saltio.openfits(masterbias)
else:
subbias = False
bstruct = None
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
subover=subover,
trim=trim,
subbias=subbias,
imstack=True,
bstruct=bstruct,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'HPREPARE',
'Images have been prepared', hist)
saltkey.new('HGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
#saltkey.new('HXTALK',time.asctime(time.localtime()),'Images have been xtalk corrected',struct[0])
saltkey.new('HBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
return
def saltclean(images,
outpath,
obslogfile=None,
gaindb=None,
xtalkfile=None,
geomfile=None,
subover=True,
trim=True,
masbias=None,
subbias=False,
median=False,
function='polynomial',
order=5,
rej_lo=3,
rej_hi=3,
niter=5,
interp='linear',
clobber=False,
logfile='salt.log',
verbose=True):
"""SALTCLEAN will provide basic CCD reductions for a set of data. It will
sort the data, and first process the biases, flats, and then the science
frames. It will record basic quality control information about each of
the steps.
"""
plotover = False
#start logging
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
# create list of output files
outpath = saltio.abspath(outpath)
#does the gain database file exist
if gaindb:
dblist = saltio.readgaindb(gaindb)
else:
dblist = []
# does crosstalk coefficient data exist
if xtalkfile:
xtalkfile = xtalkfile.strip()
xdict = saltio.readxtalkcoeff(xtalkfile)
else:
xdict = None
#does the mosaic file exist--raise error if no
saltio.fileexists(geomfile)
# Delete the obslog file if it already exists
if os.path.isfile(obslogfile) and clobber: saltio.delete(obslogfile)
#read in the obsveration log or create it
if os.path.isfile(obslogfile):
msg = 'The observing log already exists. Please either delete it or run saltclean with clobber=yes'
raise SaltError(msg)
else:
headerDict = obslog(infiles, log)
obsstruct = createobslogfits(headerDict)
saltio.writefits(obsstruct, obslogfile)
#create the list of bias frames and process them
filename = obsstruct.data.field('FILENAME')
detmode = obsstruct.data.field('DETMODE')
ccdtype = obsstruct.data.field('CCDTYPE')
#set the bias list of objects
biaslist = filename[ccdtype == 'ZERO']
masterbias_dict = {}
for img in infiles:
if os.path.basename(img) in biaslist:
#open the image
struct = fits.open(img)
bimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Zero frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, bimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterbias_dict = compareimages(struct,
bimg,
masterbias_dict,
keylist=biasheader_list)
#create the master bias frame
for i in masterbias_dict.keys():
bkeys = masterbias_dict[i][0]
blist = masterbias_dict[i][1:]
mbiasname = outpath + createmasterbiasname(blist, bkeys)
bfiles = ','.join(blist)
saltcombine(bfiles, mbiasname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#create the list of flatfields and process them
flatlist = filename[ccdtype == 'FLAT']
masterflat_dict = {}
for img in infiles:
if os.path.basename(img) in flatlist:
#open the image
struct = fits.open(img)
fimg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing Flat frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, fimg, clobber=clobber)
saltio.closefits(struct)
#add files to the master bias list
masterflat_dict = compareimages(struct,
fimg,
masterflat_dict,
keylist=flatheader_list)
#create the master flat frame
for i in masterflat_dict.keys():
fkeys = masterflat_dict[i][0]
flist = masterflat_dict[i][1:]
mflatname = outpath + createmasterflatname(flist, fkeys)
ffiles = ','.join(flist)
saltcombine(ffiles, mflatname, method='median', reject='sigclip', mask=False,
weight=False, blank=0, scale=None, statsec=None, lthresh=3, \
hthresh=3, clobber=False, logfile=logfile,verbose=verbose)
#process the science data
for img in infiles:
nimg = os.path.basename(img)
#print nimg, nimg in flatlist, nimg in biaslist
if not (nimg in biaslist):
#open the image
struct = fits.open(img)
simg = outpath + 'bxgp' + os.path.basename(img)
#print the message
if log:
message = 'Processing science frame %s' % img
log.message(message, with_stdout=verbose)
#process the image
struct = clean(struct,
createvar=False,
badpixelstruct=None,
mult=True,
dblist=dblist,
xdict=xdict,
subover=subover,
trim=trim,
subbias=False,
bstruct=None,
median=median,
function=function,
order=order,
rej_lo=rej_lo,
rej_hi=rej_hi,
niter=niter,
plotover=plotover,
log=log,
verbose=verbose)
#write the file out
# housekeeping keywords
fname, hist = history(
level=1,
wrap=False,
exclude=['images', 'outimages', 'outpref'])
saltkey.housekeeping(struct[0], 'SPREPARE',
'Images have been prepared', hist)
saltkey.new('SGAIN', time.asctime(time.localtime()),
'Images have been gain corrected', struct[0])
saltkey.new('SXTALK', time.asctime(time.localtime()),
'Images have been xtalk corrected', struct[0])
saltkey.new('SBIAS', time.asctime(time.localtime()),
'Images have been de-biased', struct[0])
# write FITS file
saltio.writefits(struct, simg, clobber=clobber)
saltio.closefits(struct)
#mosaic the files--currently not in the proper format--will update when it is
if not saltkey.fastmode(saltkey.get('DETMODE', struct[0])):
mimg = outpath + 'mbxgp' + os.path.basename(img)
saltmosaic(images=simg,
outimages=mimg,
outpref='',
geomfile=geomfile,
interp=interp,
cleanup=True,
clobber=clobber,
logfile=logfile,
verbose=verbose)
#remove the intermediate steps
saltio.delete(simg)
def slotback(images,
outfits,
extension,
imgtype='image',
subbacktype='median',
sigback=3,
mbin=7,
sorder=3,
niter=5,
ampperccd=2,
ignorexp=6,
clobber=False,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# set up the variables
order = sorder
plotfreq = 0.01
ftime = plotfreq
# is the input file specified?
infiles = saltio.argunpack('Input', images)
# is the output file specified?
saltio.filedefined('Output', outfits)
#open the first file and check its charactistics
struct = saltio.openfits(infiles[0])
# how many extensions?
nextend = saltkey.get('NEXTEND', struct[0])
if nextend < extension:
msg = 'Insufficient number of extensions in %s' % (infile)
raise SaltIOError(msg)
# how many amplifiers?
amplifiers = saltkey.get('NCCDS', struct[0])
amplifiers = int(ampperccd * float(amplifiers))
if ampperccd > 0:
nframes = nextend / amplifiers
nstep = amplifiers
else:
nframes = nextend
nstep = 1
ntotal = nframes * len(infiles)
# image size
naxis1 = saltkey.get('NAXIS1', struct[extension])
naxis2 = saltkey.get('NAXIS2', struct[extension])
# CCD binning
ccdsum = saltkey.get('CCDSUM', struct[0])
binx = int(ccdsum.split(' ')[0])
biny = int(ccdsum.split(' ')[1])
# If a total file is to written out, create it and update it
hdu = 1
# delete the file if clobber is on and the file exists
if os.path.isfile(outfits):
if clobber:
saltio.delete(outfits)
else:
raise SaltIOError(
'Output fits file ' + writenewfits +
'already exists. Use Clobber=yes to overwrite file')
# create the output file
try:
hdu = pyfits.PrimaryHDU()
hdu.header = struct[0].header
hdu.header['NCCDS'] = 1
hdu.header['NSCIEXT'] = ntotal - ignorexp
hdu.header['NEXTEND'] = ntotal - ignorexp
hduList = pyfits.HDUList(hdu)
hduList.verify()
hduList.writeto(outfits)
except:
raise SaltIOError('Could not create new fits file named ' +
writenewfits)
# read it back in for updating
hduList = saltio.openfits(outfits, mode='update')
# set up the completeness tracker
if verbose:
j = 0
x2 = float(j) / float(ntotal)
ctext = 'Percentage Complete: %3.2f' % x2
sys.stdout.write(ctext)
sys.stdout.flush()
for infile in infiles:
struct = saltio.openfits(infile)
# Skip through the frames and process each frame individually
for i in range(nframes):
if not (infile == infiles[0] and i < ignorexp):
hdu = extension + i * nstep
try:
header = struct[hdu].header
array = struct[hdu].data
array = array * 1.0
except Exception, e:
msg = 'Unable to open extension %i in image %s because %s' % (
hdu, infile, e)
raise SaltIOError(msg)
# start the analysis of each frame
# gain and readout noise
try:
gain = float(header['GAIN'])
except:
gain = 1
log.warning('Gain not specified in image header')
try:
rdnoise = float(header['RDNOISE'])
except:
rdnoise = 0
log.warning('RDNOISE not specified in image header')
# background subtraction
if not subbacktype == 'none':
try:
narray = subbackground(array, sigback, mbin, order,
niter, subbacktype)
except:
log.warning('Image ' + infile + ' extention ' +
str(i) + ' is blank, skipping')
continue
# create output array
try:
if imgtype == 'background':
oarray = narray - array
else:
oarray = narray
except:
oarray = array
# print progress
if verbose:
x2 = float(j) / float(ntotal)
if x2 > ftime:
ctext = '\b\b\b\b\b %3.2f' % x2
sys.stdout.write(ctext)
sys.stdout.flush()
ftime += plotfreq
# update the header values with the image name and extension number
try:
hdue = pyfits.ImageHDU(oarray)
hdue.header = header
hdue.header.update('ONAME', infile,
'Original image name')
hdue.header.update('OEXT', hdu,
'Original extension number')
except Exception, e:
msg='SALTPHOT--WARNING: Could not update image in newfits file for %s ext %i because %s' \
% (infile, hdu, e)
raise SaltIOError(msg)
hduList.append(hdue)
j += 1
# close FITS file
saltio.closefits(struct)
def saltfpringfind(images,
method=None,
section=None,
thresh=5,
minsize=10,
niter=5,
conv=0.05,
displayimage=True,
clobber=False,
logfile='salt.log',
verbose=True):
with logging(logfile, debug) as log:
# Check the input images
infiles = saltio.argunpack('Input', images)
#check the method
method = saltio.checkfornone(method)
# read in the section
section = saltio.checkfornone(section)
if section is None:
pass
else:
section = saltio.getSection(section)
# open each raw image file
for img in infiles:
#open the fits file
struct = saltio.openfits(img)
data = struct[0].data
#determine the background value for the image
if section is None:
#if section is none, just use all pixels greater than zero
bdata = data[data > 0]
else:
y1, y2, x1, x2 = section
bdata = data[y1:y2, x1:x2]
bmean, bmedian, bstd = iterstat(bdata,
sig=thresh,
niter=niter,
verbose=False)
message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \
('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd)
log.message(message, with_stdout=verbose)
mdata = data * (data - bmean > thresh * bstd)
#prepare the first guess for the image
ring_list = findrings(data,
thresh=thresh,
niter=niter,
minsize=minsize)
#if specified, find the center of the ring
if method is not None:
for i in range(len(ring_list)):
ring_list[i] = findcenter(data,
ring_list[i],
method,
niter=niter,
conv=conv)
#if one peak: no rings. If two peaks: one ring, if two peaks: four rings
if len(ring_list) == 1:
msg = "One ring dected in image"
else:
msg = "%i rings found in image" % len(ring_list)
log.message(message, with_stdout=verbose)
if displayimage:
regfile = img.replace('.fits', '.reg')
if clobber and os.path.isfile(regfile):
fout = saltio.delete(regfile)
fout = open(regfile, 'w')
fout.write("""# Region file format: DS9 version 4.1
# Filename: %s
global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1
physical
""" % img)
for ring in ring_list:
fout.write('circle(%f, %f, %f)\n' %
(ring.xc, ring.yc, ring.prad))
fout.write('circle(%f, %f, %f)\n' %
(ring.xc, ring.yc, ring.prad - 3 * ring.sigma))
fout.write('circle(%f, %f, %f)\n' %
(ring.xc, ring.yc, ring.prad + 3 * ring.sigma))
fout.close()
display(img, catname=regfile, rformat='reg')
message = 'Ring Parameters\n%30s %6s %6s %6s\n' % ('Image', 'XC',
'YC', 'Radius')
log.message(message, with_stdout=verbose)
for ring in ring_list:
msg = '%30s %6.2f %6.2f %6.2f\n' % (img, ring.xc, ring.yc,
ring.prad)
log.message(msg, with_header=False, with_stdout=verbose)
def saltfpringfind(images, method=None, section=None, thresh=5, minsize=10, niter=5, conv=0.05,
displayimage=True, clobber=False, logfile='salt.log',verbose=True):
with logging(logfile,debug) as log:
# Check the input images
infiles = saltio.argunpack ('Input',images)
#check the method
method=saltio.checkfornone(method)
# read in the section
section=saltio.checkfornone(section)
if section is None:
pass
else:
section=saltio.getSection(section)
# open each raw image file
for img in infiles:
#open the fits file
struct=saltio.openfits(img)
data=struct[0].data
#determine the background value for the image
if section is None:
#if section is none, just use all pixels greater than zero
bdata=data[data>0]
else:
y1,y2,x1,x2=section
bdata=data[y1:y2,x1:x2]
bmean, bmedian, bstd=iterstat(bdata, sig=thresh, niter=niter, verbose=False)
message="Image Background Statistics\n%30s %6s %8s %8s\n%30s %5.4f %5.4f %5.4f\n" % \
('Image', 'Mean', 'Median', 'Std',img, bmean, bmedian, bstd)
log.message(message, with_stdout=verbose)
mdata=data*(data-bmean>thresh*bstd)
#prepare the first guess for the image
ring_list=findrings(data, thresh=thresh, niter=niter, minsize=minsize)
#if specified, find the center of the ring
if method is not None:
for i in range(len(ring_list)):
ring_list[i]=findcenter(data, ring_list[i], method, niter=niter, conv=conv)
#if one peak: no rings. If two peaks: one ring, if two peaks: four rings
if len(ring_list)==1:
msg="One ring dected in image"
else:
msg="%i rings found in image" % len(ring_list)
log.message(message, with_stdout=verbose)
if displayimage:
regfile=img.replace('.fits', '.reg')
if clobber and os.path.isfile(regfile): fout=saltio.delete(regfile)
fout=open(regfile, 'w')
fout.write("""# Region file format: DS9 version 4.1
# Filename: %s
global color=green dashlist=8 3 width=1 font="helvetica 10 normal roman" select=1 highlite=1 dash=0 fixed=0 edit=1 move=1 delete=1 include=1 source=1
physical
""" % img)
for ring in ring_list:
fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad))
fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad-3*ring.sigma))
fout.write('circle(%f, %f, %f)\n' % (ring.xc,ring.yc,ring.prad+3*ring.sigma))
fout.close()
display(img, catname=regfile, rformat='reg')
message = 'Ring Parameters\n%30s %6s %6s %6s\n' % ('Image', 'XC', 'YC', 'Radius')
log.message(message, with_stdout=verbose)
for ring in ring_list:
msg='%30s %6.2f %6.2f %6.2f\n' % (img, ring.xc, ring.yc, ring.prad)
log.message(msg, with_header=False, with_stdout=verbose)
def slotphot(images,outfile,srcfile,newfits=None,phottype='square',
subbacktype='median',sigback=3,mbin=7,sorder=3,niter=5,sigdet=5,
contpix=10,ampperccd=2,ignorexp=6,driftlimit=10.,finddrift=True,
outtype='ascii',reltime=True,clobber=True,logfile='salt.log',
verbose=True):
"""Perform photometry on listed SALT slotmode *images*."""
with logging(logfile,debug) as log:
# set up the variables
entries = []
vig_lo = {}
vig_hi = {}
amp = {}
x = {}
y = {}
x_o = {}
y_o = {}
r = {}
br1 = {}
br2 = {}
hour = 0
min = 0
sec = 0.
time0 = 0.
nframes = 0
bin=mbin
order=sorder
# is the input file specified?
saltsafeio.filedefined('Input',images)
# if the input file is a list, does it exist?
if images[0] == '@':
saltsafeio.listexists('Input',images)
# parse list of input files
infiles=saltsafeio.listparse('Raw image',images,'','','')
# check input files exist
saltsafeio.filesexist(infiles,'','r')
# is the output file specified?
saltsafeio.filedefined('Output',outfile)
# check output file does not exist, optionally remove it if it does exist
if os.path.exists(outfile) and clobber:
os.remove(outfile)
elif os.path.exists(outfile) and not clobber:
raise SaltIOError('File '+outfile+' already exists, use clobber=y')
# open output ascii file
if outtype=='ascii':
try:
lc = open(outfile,'a')
except:
raise SaltIOError('Cannot open ouput file '+outfile)
# is the extraction region defintion file specified?
saltsafeio.filedefined('Extraction region defintion',srcfile)
# check extraction region defintion file exists
srcfile = srcfile.strip()
saltsafeio.fileexists(srcfile)
# read extraction region defintion file
amp, x, y, x_o, y_o, r, br1, br2=slottool.readsrcfile(srcfile)
# set the writenewfits parameter
if not newfits or newfits=='none':
writenewfits=False
else:
writenewfits=newfits
# get time of first exposure and basic information about the observations
infile=infiles[0]
struct=saltsafeio.openfits(infile)
# identify instrument
instrume,keyprep,keygain,keybias,keyxtalk,keyslot=saltsafekey.instrumid(struct,infile)
# how many extensions?
nextend=saltsafekey.get('NEXTEND',struct[0],infile)
if nextend < amp['comparison']:
msg='Insufficient number of extensions in %s' % (infile)
raise SaltIOError(msg)
# how many amplifiers?
amplifiers=saltsafekey.get('NCCDS',struct[0],infile)
amplifiers = int(ampperccd*float(amplifiers))
if ampperccd>0:
nframes = int(nextend/amplifiers)
nstep=amplifiers
else:
nframes = nextend
nstep=1
ntotal=nframes*len(infiles)
# image size
naxis1=saltsafekey.get('NAXIS1',struct[amp['comparison']],infile)
naxis2=saltsafekey.get('NAXIS2',struct[amp['comparison']],infile)
# CCD binning
ccdsum=saltsafekey.get('CCDSUM',struct[0],infile)
binx=int(ccdsum.split(' ')[0])
biny=int(ccdsum.split(' ')[1])
# Identify the time of the observations
ext = 1
try:
time0=slottool.getobstime(struct[ext], infile+'['+str(ext)+']')
dateobs=saltsafekey.get('DATE-OBS',struct[ext],infile)
dateobs=dateobs.replace('-','/')
except:
raise SaltIOError('No time or obsdate in first image')
# If a total file is to be written out, create it and update it
if writenewfits:
if os.path.isfile(writenewfits):
if clobber:
saltsafeio.delete(writenewfits)
else:
raise SaltIOError('Newfits file exists, use clobber')
try:
hdu=pyfits.PrimaryHDU()
hdu.header=struct[0].header
hdu.header['NCCDS']=1
hdu.header['NSCIEXT']=ntotal-ignorexp
hdu.header['NEXTEND']=ntotal-ignorexp
hduList=pyfits.HDUList(hdu)
hduList.verify()
hduList.writeto(writenewfits)
except:
raise SaltIOError('Could not create newfits file, '+writenewfits)
# Close image file
saltsafeio.closefits(struct)
# Read newfits file back in for updating
if writenewfits:
try:
hduList=pyfits.open(writenewfits,mode='update')
except:
raise SaltIOError('Cannot open newfits file '+writenewfits+' for updating.')
# set up the arrays
j=0
time=np.zeros(ntotal ,dtype='float')-1.0
dx=np.zeros(ntotal ,dtype='float')
dy=np.zeros(ntotal ,dtype='float')
tflux=np.zeros(ntotal ,dtype='float')
terr =np.zeros(ntotal ,dtype='float')
cflux=np.zeros(ntotal ,dtype='float')
cerr =np.zeros(ntotal ,dtype='float')
ratio=np.zeros(ntotal ,dtype='float')
rerr =np.zeros(ntotal ,dtype='float')
tgt_x=np.zeros(ntotal ,dtype='float')
tgt_y=np.zeros(ntotal ,dtype='float')
cmp_x=np.zeros(ntotal ,dtype='float')
cmp_y=np.zeros(ntotal ,dtype='float')
p_one=100./ntotal # One percent
p_old=-1 # Previous completed percentage
p_new=0 # New completed percentage
p_n=1 # Counter number
for infile in infiles:
# Log
if verbose:
log.message('Starting photometry on file '+infile, with_stdout=False)
struct=pyfits.open(infile)
# Skip through the frames and process each frame individually
for i in range(nframes):
# Show progress
if verbose:
p_new=int(p_n*p_one)
p_n+=1
if p_new!=p_old:
ctext='Percentage Complete: %d\r' % p_new
sys.stdout.write(ctext)
sys.stdout.flush()
p_old=p_new
if not (infile==infiles[0] and i < ignorexp):
ext=amp['comparison']+i*nstep
try:
header=struct[ext].header
array=struct[ext].data
array=array*1.0
except:
msg='Unable to open extension %i in image %s' % (ext, infile)
raise SaltIOError(msg)
# starti the analysis of each frame
# get the time
time[j]=slottool.getobstime(struct[ext],infile+'['+str(ext)+']')
# gain and readout noise
try:
gain=float(header['GAIN'])
except:
gain=1
raise SaltIOError('Gain not specified in image header')
try:
rdnoise=float(header['RDNOISE'])
except:
rdnoise=0
raise SaltIOError('RDNOISE not specified in image header')
# background subtraction
if not subbacktype=='none':
try:
array=subbackground(array, sigback, bin, order, niter, subbacktype)
except SaltError:
log.warning('Image '+infile+' extention '+str(ext)+' is blank, skipping')
continue
# x-y fit to the comparison star and update the x,y values
if finddrift:
carray, fx,fy=slottool.finddrift(array, x['comparison'], y['comparison'], r['comparison'], naxis1, naxis2, sigdet, contpix, sigback, driftlimit, niter)
if fx > -1 and fy > -1:
if fx < naxis1 and fy < naxis2:
dx[j]=x['comparison']-fx
dy[j]=y['comparison']-fy
x['comparison']=fx
y['comparison']=fy
x['target']=x['target']-dx[j]
y['target']=y['target']-dy[j]
else:
dx[j]=0
dy[j]=0
x['comparison']=x_o['comparison']
y['comparison']=y_o['comparison']
x['target']=x_o['target']
y['target']=y_o['target']
else:
msg='No comparison object found in image file ' + infile+' on extension %i skipping.' % ext
log.warning(msg)
pass
# do photometry
try:
tflux[j],terr[j],cflux[j],cerr[j],ratio[j],rerr[j]=slottool.dophot(phottype, array, x, y, r, br1, br2, gain, rdnoise, naxis1, naxis2)
except SaltError, e:
msg='Could not do photometry on extension %i in image %s because %s skipping.' % (ext, infile, e)
log.warning(msg)
tgt_x[j]=x['target']
tgt_y[j]=y['target']
cmp_x[j]=x['comparison']
cmp_y[j]=y['comparison']
# record results
# TODO! This should be removed in favor of the write all to disk in the end
if outtype=='ascii':
slottool.writedataout(lc, j+1, time[j], x, y, tflux[j], terr[j], cflux[j],cerr[j],ratio[j],rerr[j],time0, reltime)
# write newfits file
if writenewfits:
# add original name and extension number to header
try:
hdue=pyfits.ImageHDU(array)
hdue.header=header
hdue.header.update('ONAME',infile,'Original image name')
hdue.header.update('OEXT',ext,'Original extension number')
hduList.append(hdue)
except:
log.warning('Could not update image in newfits '+infile+' '+str(ext))
# increment counter
j+=1
# close FITS file
saltsafeio.closefits(struct)
# close newfits file
if writenewfits:
try:
hduList.flush()
hduList.close()
except:
raise SaltIOError('Cannot close newfits file.')
# write to output
if outtype=='ascii':
# close output ascii file
try:
lc.close()
except:
raise SaltIOError('Cannot close ouput file ' + outfile)
elif outtype=='fits':
print 'writing fits'
try:
c1=pyfits.Column(name='index',format='D',array=np.arange(ntotal))
if reltime:
c2=pyfits.Column(name='time',format='D',array=time-time0)
else:
c2=pyfits.Column(name='time',format='D',array=time)
c3=pyfits.Column(name='tgt_x',format='D',array=tgt_x)
c4=pyfits.Column(name='tgt_y',format='D',array=tgt_y)
c5=pyfits.Column(name='tgt_flux',format='D',array=tflux)
c6=pyfits.Column(name='tgt_err',format='D',array=terr)
c7=pyfits.Column(name='cmp_x',format='D',array=cmp_x)
c8=pyfits.Column(name='cmp_y',format='D',array=cmp_y)
c9=pyfits.Column(name='cmp_flux',format='D',array=cflux)
c10=pyfits.Column(name='cmp_err',format='D',array=cerr)
c11=pyfits.Column(name='flux_ratio',format='D',array=ratio)
c12=pyfits.Column(name='flux_ratio_err',format='D',array=rerr)
tbhdu=pyfits.new_table([c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12])
# Add header information
tbhdu.header.update('RELTIME',str(reltime),'Time relative to first datapoint or absolute.')
tbhdu.writeto(outfile)
print 'fits written to ',outfile
except:
raise SaltIOError('Could not write to fits table.')
