def prepare(struct,createvar=False, badpixelstruct=None, namps=2):
#set up the file name
infile=saltkey.getimagename(struct[0])
#include information about which headers are science extensoisn
#and check to make sure each array is a data array
nextend=len(struct)-1
nsciext=nextend
for i in range(1,nextend+1):
try:
struct[i].header['XTENSION'] == 'IMAGE'
except:
msg='Extension %i in %s is not an image data'
raise SaltError(msg)
saltkey.new('EXTNAME','SCI','Extension name',struct[i])
saltkey.new('EXTVER',i,'Extension number',struct[i])
#check the number of amplifiers
#TODO: This is current hardwired at a value of 2
nccds = saltkey.get('NCCDS',struct[0])
if (nextend%(nccds*namps) != 0):
message = 'ERROR -- SALTPREPARE: Number of file extensions and '
message += 'number of amplifiers are not consistent in ' + infile
raise SaltError(message)
#Add the inv. variance frame if requested--assumes no variance frame
#and the science frames are in the first n frames
if createvar:
#create the inv. variance frames
for i in range(1, nsciext+1):
try:
hdu=CreateVariance(struct[i], i, nextend+i)
except Exception as e:
msg='Cannot create variance frame in extension %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header.update('VAREXT',nextend+i, comment='Extension for Variance Frame')
struct.append(hdu)
nextend+=nsciext
#create the badpixelframes
for i in range(1, nsciext+1):
try:
hdu=createbadpixel(struct, badpixelstruct, i, nextend+i)
except Exception as e:
msg='Could not create bad pixel extension in ext %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header.update('BPMEXT',nextend+i, comment='Extension for Bad Pixel Mask')
struct.append(hdu)
nextend+=nsciext
#update the number of extensions
saltkey.new('NSCIEXT',nsciext,'Number of science extensions', struct[0])
saltkey.new('NEXTEND',nextend,'Number of data extensions', struct[0])
return struct
def prepare(struct,createvar=False, badpixelstruct=None, namps=2):
#set up the file name
infile=saltkey.getimagename(struct[0])
#include information about which headers are science extensoisn
#and check to make sure each array is a data array
nextend=len(struct)-1
nsciext=nextend
for i in range(1,nextend+1):
try:
struct[i].header['XTENSION'] == 'IMAGE'
except:
msg='Extension %i in %s is not an image data'
raise SaltError(msg)
saltkey.new('EXTNAME','SCI','Extension name',struct[i])
saltkey.new('EXTVER',i,'Extension number',struct[i])
#check the number of amplifiers
#TODO: This is current hardwired at a value of 2
nccds = saltkey.get('NCCDS',struct[0])
if (nextend%(nccds*namps) != 0):
message = 'ERROR -- SALTPREPARE: Number of file extensions and '
message += 'number of amplifiers are not consistent in ' + infile
raise SaltError(message)
#Add the inv. variance frame if requested--assumes no variance frame
#and the science frames are in the first n frames
if createvar:
#create the inv. variance frames
for i in range(1, nsciext+1):
try:
hdu=CreateVariance(struct[i], i, nextend+i)
except Exception, e:
msg='Cannot create variance frame in extension %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header.update('VAREXT',nextend+i, comment='Extension for Variance Frame')
struct.append(hdu)
nextend+=nsciext
#create the badpixelframes
for i in range(1, nsciext+1):
try:
hdu=createbadpixel(struct, badpixelstruct, i, nextend+i)
except Exception, e:
msg='Could not create bad pixel extension in ext %i of %s because %s' % (nextend+i, infile, e)
raise SaltError(msg)
struct[i].header.update('BPMEXT',nextend+i, comment='Extension for Bad Pixel Mask')
struct.append(hdu)
def bias(struct,subover=True,trim=True, subbias=False, bstruct=None,
median=False, function='polynomial',order=3,rej_lo=3,rej_hi=3,niter=10,
plotover=False, log=None, verbose=True):
"""Bias subtracts the bias levels from a frame. It will fit and subtract the overscan
region, trim the images, and subtract a master bias if required.
struct--image structure
subover--subtract the overscan region
trim--trim the image
subbias--subtract master bias
bstruct--master bias image structure
median--use the median instead of mean in image statistics
function--form to fit to the overscan region
order--order for the function
rej_lo--sigma of low points to reject in the fit
rej_hi--sigma of high points to reject in the fit
niter--number of iterations
log--saltio log for recording information
verbose--whether to print to stdout
"""
infile=saltkey.getimagename(struct[0])
# how many extensions?
nsciext = saltkey.get('NSCIEXT',struct[0])
nextend = saltkey.get('NEXTEND',struct[0])
nccd = saltkey.get('NCCDS',struct[0])
# how many amplifiers?--this is hard wired
amplifiers = 2 * nccd
#log the process
if subover and log:
message = '%28s %7s %5s %4s %6s' % \
('HDU','Overscan','Order','RMS','Niter')
log.message('\n --------------------------------------------------',
with_header=False, with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' --------------------------------------------------',
with_header=False, with_stdout=verbose)
if (plotover):
plt.figure(1)
plt.axes([0.1,0.1,0.8,0.8])
plt.xlabel('CCD Column')
plt.ylabel('Pixel Counts (e-)')
plt.ion()
#loop through the extensions and subtract the bias
for i in range(1,nsciext+1):
if struct[i].name=='SCI':
#get the bias section
biassec = saltkey.get('BIASSEC',struct[i])
y1,y2,x1,x2 = saltio.getSection(biassec, iraf_format=True)
#get the data section
datasec = saltkey.get('DATASEC',struct[i])
dy1,dy2, dx1, dx2 = saltio.getSection(datasec, iraf_format=True)
#setup the overscan region
if subover:
yarr=np.arange(y1,y2, dtype=float)
data=struct[i].data
odata=struct[i].data[y1:y2,x1:x2]
if median:
odata=np.median((struct[i].data[y1:y2,x1:x2]),axis=1)
olevel=np.median((struct[i].data[y1:y2,x1:x2]))
saltkey.new('OVERSCAN','%f' % (olevel),'Overscan median value', struct[i])
else:
odata=np.mean((struct[i].data[y1:y2,x1:x2]),axis=1)
olevel=np.mean((struct[i].data[y1:y2,x1:x2]))
saltkey.new('OVERSCAN','%f' % (olevel),'Overscan mean value', struct[i])
#fit the overscan region
ifit=saltfit.interfit(yarr, odata, function=function, \
order=order, thresh=rej_hi, niter=niter)
try:
ifit.interfit()
coeffs=ifit.coef
ofit=ifit(yarr)
omean, omed, osigma=saltstat.iterstat((odata-ofit), sig=3, niter=5)
except ValueError:
#catch the error if it is a zero array
ofit=np.array(yarr)*0.0
osigma=0.0
except TypeError:
#catch the error if it is a zero array
ofit=np.array(yarr)*0.0
osigma=0.0
#if it hasn't been already, convert image to
#double format
struct[i].data = 1.0 * struct[i].data
try:
struct[i].header.remove('BZERO')
struct[i].header.remove('BSCALE')
except:
pass
#subtract the overscan region
for j in range(len(struct[i].data[0])):
struct[i].data[y1:y2,j] -= ofit
#report the information
if log:
message = '%25s[%1d] %8.2f %3d %7.2f %3d' % \
(infile, i, olevel, order, osigma, niter)
log.message(message, with_stdout=verbose, with_header=False)
#add the statistics to the image header
saltkey.new('OVERRMS','%f' % (osigma),'Overscan RMS value', struct[i])
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu=saltkey.get('VAREXT', struct[i])
try:
vdata=struct[vhdu].data
#The bias level should not be included in the noise from the signal
for j in range(len(struct[i].data[0])):
vdata[y1:y2,j] -= ofit
#add a bit to make sure that the minimum error is the rednoise
rdnoise= saltkey.get('RDNOISE',struct[i])
vdata[vdata<rdnoise**2]=rdnoise**2
struct[vhdu].data=vdata+osigma**2
except Exception, e:
msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e)
raise SaltError(msg)
#plot the overscan region
if plotover:
plt.plot(yarr, odata)
plt.plot(yarr, ofit)
#trim the data and update the headers
if trim:
struct[i].data=struct[i].data[dy1:dy2,dx1:dx2]
datasec = '[1:'+str(dx2-dx1)+',1:'+str(dy2-dy1)+']'
saltkey.put('DATASEC',datasec,struct[i])
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu=saltkey.get('VAREXT', struct[i])
struct[vhdu].data=struct[vhdu].data[dy1:dy2,dx1:dx2]
datasec = '[1:'+str(dx2-dx1)+',1:'+str(dy2-dy1)+']'
saltkey.put('DATASEC',datasec,struct[vhdu])
#update the BPM frame
if saltkey.found('BPMEXT', struct[i]):
bhdu=saltkey.get('BPMEXT', struct[i])
struct[bhdu].data=struct[bhdu].data[dy1:dy2,dx1:dx2]
datasec = '[1:'+str(dx2-dx1)+',1:'+str(dy2-dy1)+']'
saltkey.put('DATASEC',datasec,struct[bhdu])
#subtract the master bias if necessary
if subbias and bstruct:
struct[i].data -= bstruct[i].data
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu=saltkey.get('VAREXT', struct[i])
try:
vdata=struct[vhdu].data
struct[vhdu].data=vdata+bstruct[vhdu].data
except Exception, e:
msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e)
raise SaltError(msg)
def make_mosaic(struct, gap, xshift, yshift, rotation, interp_type='linear',
boundary='constant', constant=0, geotran=True, fill=False,
cleanup=True, log=None, verbose=False):
"""Given a SALT image struct, combine each of the individual amplifiers and
apply the geometric CCD transformations to the image
"""
# get the name of the file
infile = saltkey.getimagename(struct[0], base=True)
outpath = './'
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = \
saltkey.instrumid(struct)
# how many amplifiers?
nsciext = saltkey.get('NSCIEXT', struct[0])
nextend = saltkey.get('NEXTEND', struct[0])
nccds = saltkey.get('NCCDS', struct[0])
amplifiers = nccds * 2
if nextend > nsciext:
varframe = True
else:
varframe = False
# CCD geometry coefficients
if (instrume == 'RSS' or instrume == 'PFIS'):
xsh = [0., xshift[0], 0., xshift[1]]
ysh = [0., yshift[0], 0., yshift[1]]
rot = [0., rotation[0], 0., rotation[1]]
elif instrume == 'SALTICAM':
xsh = [0., xshift[0], 0.]
ysh = [0., yshift[0], 0.]
rot = [0., rotation[0], 0]
# how many extensions?
nextend = saltkey.get('NEXTEND', struct[0])
# CCD on-chip binning
xbin, ybin = saltkey.ccdbin(struct[0])
# create temporary primary extension
outstruct = []
outstruct.append(struct[0])
# define temporary FITS file store tiled CCDs
tilefile = saltio.tmpfile(outpath)
tilefile += 'tile.fits'
if varframe:
tilehdu = [None] * (3 * int(nsciext / 2) + 1)
else:
tilehdu = [None] * int(nsciext / 2 + 1)
tilehdu[0] = fits.PrimaryHDU()
#tilehdu[0].header = struct[0].header
if log:
log.message('', with_stdout=verbose)
# iterate over amplifiers, stich them to produce file of CCD images
for i in range(int(nsciext / 2)):
hdu = i * 2 + 1
# amplifier = hdu%amplifiers
# if (amplifier == 0): amplifier = amplifiers
# read DATASEC keywords
datasec1 = saltkey.get('DATASEC', struct[hdu])
datasec2 = saltkey.get('DATASEC', struct[hdu + 1])
xdsec1, ydsec1 = saltstring.secsplit(datasec1)
xdsec2, ydsec2 = saltstring.secsplit(datasec2)
# read images
imdata1 = saltio.readimage(struct, hdu)
imdata2 = saltio.readimage(struct, hdu + 1)
# tile 2n amplifiers to yield n CCD images
outdata = numpy.zeros((ydsec1[1] +
abs(ysh[i +
1] /
ybin), xdsec1[1] +
xdsec2[1] +
abs(xsh[i +
1] /
xbin)), numpy.float32)
# set up the variance frame
if varframe:
vardata = outdata.copy()
vdata1 = saltio.readimage(struct, struct[hdu].header['VAREXT'])
vdata2 = saltio.readimage(struct, struct[hdu + 1].header['VAREXT'])
bpmdata = outdata.copy()
bdata1 = saltio.readimage(struct, struct[hdu].header['BPMEXT'])
bdata2 = saltio.readimage(struct, struct[hdu + 1].header['BPMEXT'])
x1 = xdsec1[0] - 1
if x1 != 0:
msg = 'The data in %s have not been trimmed prior to mosaicking.' \
% infile
log.error(msg)
if xsh[i + 1] < 0:
x1 += abs(xsh[i + 1] / xbin)
x2 = x1 + xdsec1[1]
y1 = ydsec1[0] - 1
if ysh[i + 1] < 0:
y1 += abs(ysh[i + 1] / ybin)
y2 = y1 + ydsec1[1]
outdata[y1:y2, x1:x2] =\
imdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
x1 = x2
x2 = x1 + xdsec2[1]
y1 = ydsec2[0] - 1
if ysh[i + 1] < 0:
y1 += abs(ysh[i + 1] / ybin)
y2 = y1 + ydsec2[1]
outdata[y1:y2, x1:x2] =\
imdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
# size of new image
naxis1 = str(xdsec1[1] + xdsec2[1])
naxis2 = str(ydsec1[1])
# add image and keywords to HDU list
tilehdu[i + 1] = fits.ImageHDU(outdata)
tilehdu[i + 1].header = struct[hdu].header
#tilehdu[
# i + 1].header['DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
if varframe:
vext = i + 1 + int(nsciext / 2.)
tilehdu[vext] = fits.ImageHDU(vardata)
#tilehdu[vext].header = struct[struct[hdu].header['VAREXT']].header
#tilehdu[vext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
bext = i + 1 + 2 * int(nsciext / 2.)
tilehdu[bext] = fits.ImageHDU(bpmdata)
#tilehdu[bext].header = struct[struct[hdu].header['BPMEXT']].header
#tilehdu[bext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
# image tile log message #1
if log:
message = os.path.basename(infile) + '[' + str(hdu) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
message = os.path.basename(infile) + '[' + str(hdu + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[1] + 1) + ':' + \
str(xdsec1[1] + xdsec2[1]) + ','
message += str(ydsec2[0]) + ':' + str(ydsec2[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
# write temporary file of tiled CCDs
hdulist = fits.HDUList(tilehdu)
hdulist.writeto(tilefile)
# iterate over CCDs, transform and rotate images
yrot = [None] * 4
xrot = [None] * 4
tranfile = [' ']
tranhdu = [0]
if varframe:
tranfile = [''] * (3 * int(nsciext / 2) + 1)
tranhdu = [0] * (3 * int(nsciext / 2) + 1)
else:
tranfile = [''] * int(nsciext / 2 + 1)
tranhdu = [0] * int(nsciext / 2 + 1)
# this is hardwired for SALT where the second CCD is considered the
# fiducial
for hdu in range(1, int(nsciext / 2 + 1)):
tranfile[hdu] = saltio.tmpfile(outpath)
tranfile[hdu] += 'tran.fits'
if varframe:
tranfile[hdu + nccds] = saltio.tmpfile(outpath) + 'tran.fits'
tranfile[hdu + 2 * nccds] = saltio.tmpfile(outpath) + 'tran.fits'
ccd = hdu % nccds
if (ccd == 0):
ccd = nccds
# correct rotation for CCD binning
yrot[ccd] = rot[ccd] * ybin / xbin
xrot[ccd] = rot[ccd] * xbin / ybin
dxshift = xbin * int(float(int(gap) / xbin) + 0.5) - gap
# transformation using geotran IRAF task
# if (ccd == 1):
if (ccd != 2):
if geotran:
message = '\nSALTMOSAIC -- geotran ' + tilefile + \
'[' + str(ccd) + '] ' + tranfile[hdu]
message += ' \"\" \"\" xshift=' + \
str((xsh[ccd] + (2 - ccd) * dxshift) / xbin) + ' '
message += 'yshift=' + \
str(ysh[ccd] / ybin) + ' xrotation=' + str(xrot[ccd]) + ' '
message += 'yrotation=' + \
str(yrot[ccd]) + ' xmag=1 ymag=1 xmin=\'INDEF\''
message += 'xmax=\'INDEF\' ymin=\'INDEF\' ymax=\'INDEF\' '
message += 'ncols=\'INDEF\' '
message += 'nlines=\'INDEF\' verbose=\'no\' '
message += 'fluxconserve=\'yes\' nxblock=2048 '
message += 'nyblock=2048 interpolant=\'' + \
interp_type + '\' boundary=\'constant\' constant=0'
log.message(message, with_stdout=verbose)
yd, xd = tilehdu[ccd].data.shape
ncols = 'INDEF' # ncols=xd+abs(xsh[ccd]/xbin)
nlines = 'INDEF' # nlines=yd+abs(ysh[ccd]/ybin)
geo_xshift = xsh[ccd] + (2 - ccd) * dxshift / xbin
geo_yshift = ysh[ccd] / ybin
iraf.images.immatch.geotran(tilefile + "[" + str(ccd) + "]",
tranfile[hdu],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1, ymag=1, xmin='INDEF',
xmax='INDEF', ymin='INDEF',
ymax='INDEF', ncols=ncols,
nlines=nlines, verbose='no',
fluxconserve='yes', nxblock=2048,
nyblock=2048, interpolant="linear",
boundary="constant", constant=0)
if varframe:
var_infile = tilefile + "[" + str(ccd + nccds) + "]"
iraf.images.immatch.geotran(var_infile,
tranfile[hdu + nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1, ymag=1, xmin='INDEF',
xmax='INDEF', ymin='INDEF',
ymax='INDEF', ncols=ncols,
nlines=nlines, verbose='no',
fluxconserve='yes',
nxblock=2048, nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
var2_infile = tilefile + "[" + str(ccd + 2 * nccds) + "]"
iraf.images.immatch.geotran(var2_infile,
tranfile[hdu + 2 * nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1, ymag=1, xmin='INDEF',
xmax='INDEF', ymin='INDEF',
ymax='INDEF', ncols=ncols,
nlines=nlines, verbose='no',
fluxconserve='yes',
nxblock=2048, nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
# open the file and copy the data to tranhdu
tstruct = fits.open(tranfile[hdu])
tranhdu[hdu] = tstruct[0].data
tstruct.close()
if varframe:
tranhdu[
hdu +
nccds] = fits.open(
tranfile[
hdu +
nccds])[0].data
tranhdu[
hdu +
2 *
nccds] = fits.open(
tranfile[
hdu +
2 *
nccds])[0].data
else:
log.message(
"Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd,
xsh[ccd] /
2.0,
ysh[ccd] /
2.0,
xrot[ccd]),
with_stdout=verbose,
with_header=False)
tranhdu[hdu] = geometric_transform(
tilehdu[ccd].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(
xsh[ccd] / 2,
ysh[ccd] / 2,
1,
1,
xrot[ccd],
yrot[ccd]))
tstruct = fits.PrimaryHDU(tranhdu[hdu])
tstruct.writeto(tranfile[hdu])
if varframe:
tranhdu[hdu + nccds] = geometric_transform(
tilehdu[hdu + 3].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(
xsh[ccd] / 2, ysh[ccd] / 2,
1, 1,
xrot[ccd], yrot[ccd]))
tranhdu[hdu + 2 * nccds] = geometric_transform(
tilehdu[hdu + 6].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(
xsh[ccd] / 2, ysh[ccd] / 2,
1, 1,
xrot[ccd], yrot[ccd]))
else:
log.message(
"Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd, 0, 0, 0), with_stdout=verbose, with_header=False)
tranhdu[hdu] = tilehdu[ccd].data
if varframe:
tranhdu[hdu + nccds] = tilehdu[ccd + nccds].data
tranhdu[hdu + 2 * nccds] = tilehdu[ccd + 2 * nccds].data
# open outfile
if varframe:
outlist = 4 * [None]
else:
outlist = 2 * [None]
#outlist[0] = struct[0].copy()
outlist[0] = fits.PrimaryHDU()
outlist[0].header = struct[0].header
naxis1 = int(gap / xbin * (nccds - 1))
naxis2 = 0
for i in range(1, nccds + 1):
yw, xw = tranhdu[i].shape
naxis1 += xw + int(abs(xsh[ccd] / xbin)) + 1
naxis2 = max(naxis2, yw)
outdata = numpy.zeros((naxis2, naxis1), numpy.float32)
outdata.shape = naxis2, naxis1
if varframe:
vardata = outdata * 0
bpmdata = outdata * 0 + 1
# iterate over CCDs, stich them to produce a full image
hdu = 0
totxshift = 0
for hdu in range(1, nccds + 1):
# read DATASEC keywords
ydsec, xdsec = tranhdu[hdu].shape
# define size and shape of final image
# tile CCDs to yield mosaiced image
x1 = int((hdu - 1) * (xdsec + gap / xbin)) + int(totxshift)
x2 = xdsec + x1
y1 = int(0)
y2 = int(ydsec)
outdata[y1:y2, x1:x2] = tranhdu[hdu]
totxshift += int(abs(xsh[hdu] / xbin)) + 1
if varframe:
vardata[y1:y2, x1:x2] = tranhdu[hdu + nccds]
bpmdata[y1:y2, x1:x2] = tranhdu[hdu + 2 * nccds]
# make sure to cover up all the gaps include bad areas
if varframe:
baddata = (outdata == 0)
baddata = nd.maximum_filter(baddata, size=3)
bpmdata[baddata] = 1
# fill in the gaps if requested
if fill:
if varframe:
outdata = fill_gaps(outdata, 0)
else:
outdata = fill_gaps(outdata, 0)
# add to the file
outlist[1] = fits.ImageHDU(outdata)
if varframe:
outlist[2] = fits.ImageHDU(vardata,name='VAR')
outlist[3] = fits.ImageHDU(bpmdata,name='BPM')
# create the image structure
outstruct = fits.HDUList(outlist)
# update the head informaation
# housekeeping keywords
saltkey.put('NEXTEND', 2, outstruct[0])
saltkey.new('EXTNAME', 'SCI', 'Extension name', outstruct[1])
saltkey.new('EXTVER', 1, 'Extension number', outstruct[1])
if varframe:
saltkey.new('VAREXT', 2, 'Variance frame extension', outstruct[1])
saltkey.new('BPMEXT', 3, 'BPM Extension', outstruct[1])
try:
saltkey.copy(struct[1], outstruct[1], 'CCDSUM')
except:
pass
# Add keywords associated with geometry
saltkey.new('SGEOMGAP', gap, 'SALT Chip Gap', outstruct[0])
c1str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[0],
yshift[0],
rotation[0])
saltkey.new('SGEOM1', c1str, 'SALT Chip 1 Transform', outstruct[0])
c2str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[1],
yshift[1],
rotation[1])
saltkey.new('SGEOM2', c2str, 'SALT Chip 2 Transform', outstruct[0])
# WCS keywords
saltkey.new('CRPIX1', 0, 'WCS: X reference pixel', outstruct[1])
saltkey.new('CRPIX2', 0, 'WCS: Y reference pixel', outstruct[1])
saltkey.new(
'CRVAL1',
float(xbin),
'WCS: X reference coordinate value',
outstruct[1])
saltkey.new(
'CRVAL2',
float(ybin),
'WCS: Y reference coordinate value',
outstruct[1])
saltkey.new('CDELT1', float(xbin), 'WCS: X pixel size', outstruct[1])
saltkey.new('CDELT2', float(ybin), 'WCS: Y pixel size', outstruct[1])
saltkey.new('CTYPE1', 'pixel', 'X type', outstruct[1])
saltkey.new('CTYPE2', 'pixel', 'Y type', outstruct[1])
# cleanup temporary files
if cleanup:
for tfile in tranfile:
if os.path.isfile(tfile):
saltio.delete(tfile)
if os.path.isfile(tilefile):
status = saltio.delete(tilefile)
# return the file
return outstruct
def xtalk(struct,xcoeff,namps=2, log=None,verbose=False):
"""xtalk cross-talk corrects the amplifies. It takes
"""
infile=saltkey.getimagename(struct[0])
# how many extensions?
nsciext = saltkey.get('NSCIEXT',struct[0])
nextend = saltkey.get('NEXTEND',struct[0])
nccd = saltkey.get('NCCDS',struct[0])
# how many amplifiers?--this is hard wired
amplifiers = namps * nccd
#setup the log
if log:
message='%28s %23s' % ('HDU','Correction')
message += '\n ----------------------------------------------'
log.message(message, with_header=False, with_stdout=verbose)
#Loop over the image extensions and subtract one
#set from the other--still hardwired at 2
for i in range(1,nsciext, 2):
if struct[i].name=='SCI' and struct[i+1].name=='SCI':
#set up the first amplifier
dat1=struct[i].data.copy()
ext1=saltkey.get('EXTVER', struct[i])
if xcoeff:
j=(ext1-1)%amplifiers
xc1=float(xcoeff[j])
else:
xc1=1.0/saltkey.get('XTALK', struct[i])
#set up the second amplifier
dat2=struct[i+1].data.copy()
ext2=saltkey.get('EXTVER', struct[i+1])
if xcoeff:
j=(ext2-1)%amplifiers
xc2=float(xcoeff[j])
else:
xc2=1.0/saltkey.get('XTALK', struct[i+1])
#subtract one from the other
struct[i].data=struct[i].data-xc2*dat2[:,::-1]
struct[i+1].data=struct[i+1].data-xc1*dat1[:,::-1]
#correct the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu1=saltkey.get('VAREXT', struct[i])
vhdu2=saltkey.get('VAREXT', struct[i+1])
try:
struct[vhdu1].data+=xc2*struct[vhdu2].data
struct[vhdu2].data+=xc1*struct[vhdu1].data
except Exception, e:
msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu1, e)
raise SaltError(msg)
#print the message
if log:
message = '%25s[%1d] Amp%1d - Amp%1d * %8.6f' % \
(infile, i, ext1, ext2, xc2)
log.message(message, with_header=False, with_stdout=verbose)
message = '%25s[%1d] Amp%1d - Amp%1d * %8.6f' % \
(infile, i+1, ext2, ext1, xc1)
log.message(message, with_header=False, with_stdout=verbose)
def crclean(struct, crtype='fast', thresh=5, mbox=5, bbox=11, bthresh=3, flux_ratio=0.2, \
gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=1, update=True, log=None, verbose=True):
"""CRCLEAN cleans SALT-like data of cosmic rays. The user has
three different choices for the type of cosmic ray cleaning being
fast, median, and edge.
crytpe--type of cosmic ray cleaning. Either fast, median, or
thresh--threshold for detecting cosmic rays
mbox--box for median cleaning
bbox--background box for median measurement
bthresh--threshold for iterating on background calculation
flux_ratio--ratio of fluxes for 'fast' method
gain--gain of images--set to None to read in from header
rdnoise--read noise of images--set to None to read in from header
bfactor--block replication factor for 'edge' method
fthresh--threshold for excluding compact sources (edge only)
gbox--Window size to grow sources. gbox=0 for no growth of cosmic rays
maxiter--maximum number of iterations
return struct
"""
#setup the image name
infile=saltkey.getimagename(struct[0])
#count the CR
totcr=0
#print out the header for the log
if log:
message = '%28s %11s' % ('HDU','COSMICRAYS')
log.message('\n ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
#cosmic ray clean each extension
for i in range(len(struct)):
#only clean the cosmic rays if it is a SCI extension or a single extension
if struct[i].name=='SCI' or len(struct)==1:
#for the edge method, get the gain and rdnoise from the fits header
#if they are not set
if crtype=='edge':
if gain==None: gain=saltkey.get('GAIN',struct[i])
if rdnoise==None: rdnoise=saltkey.get('RDNOISE',struct[i])
#get all the cosmic rays from an array
crarr=cleancosmicrays(struct[i].data, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \
gain, rdnoise, bfactor, fthresh, gbox, maxiter)
#update the frame for the various values
mask=(crarr>0)
if update: struct[i].data[mask]=crarr[mask]
#track the number of cosmic rays
ncr=mask.sum()
totcr += ncr
#if verbose print out information
if log:
message='%25s[%1d] %i' % (infile, i, ncr)
log.message(message, with_header=False, with_stdout=verbose)
#correct the BPM frame
if saltkey.found('BPMEXT', struct[i]):
b_i=saltkey.get('BPMEXT', struct[i])
try:
struct[b_i].data[mask]=1
except Exception, e:
msg='Cannot update the BPM frame in %s[%i] because %s' % (infile, b_i, e)
raise SaltError(msg)
def multicrclean(struct, crtype='fast', thresh=5, mbox=5, bbox=11, bthresh=3, flux_ratio=0.2, \
gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=1, update=True, log=None, verbose=True):
"""MULTICRCLEAN cleans SALT-like data of cosmic rays. The user has
three different choices for the type of cosmic ray cleaning being
fast, median, and edge. The process is set up to use multithreading for
quick processing of the data.
crytpe--type of cosmic ray cleaning. Either fast, median, or
thresh--threshold for detecting cosmic rays
mbox--box for median cleaning
bbox--background box for median measurement
bthresh--threshold for iterating on background calculation
flux_ratio--ratio of fluxes for 'fast' method
gain--gain of images--set to None to read in from header
rdnoise--read noise of images--set to None to read in from header
bfactor--block replication factor for 'edge' method
fthresh--threshold for excluding compact sources (edge only)
gbox--Window size to grow sources. gbox=0 for no growth of cosmic rays
maxiter--maximum number of iterations
return struct
"""
#setup the image name
infile=saltkey.getimagename(struct[0])
#count the CR
totcr=0
#print out the header for the log
if log:
message = '%28s %11s' % ('HDU','COSMICRAYS')
log.message('\n ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
task_list=[]
nproc=0
for hdu in struct:
if hdu.name=='SCI':
task_list.append((hdu.data, crtype, thresh, mbox, bbox, bthresh, flux_ratio, gain, rdnoise, bfactor , fthresh, gbox, maxiter))
nproc+=1
#set up the multi-thread
p=mp.Pool()
results=[p.apply_async(cleancosmicrays, i) for i in task_list]
p.close()
for i, hdu in enumerate(struct):
if hdu.name=='SCI':
#set up the cosmic ray array
crarr=results[i-1].get()
#update the frame for the various values
mask=(crarr>0)
if update: struct[i].data[mask]=crarr[mask]
#track the number of cosmic rays
ncr=mask.sum()
totcr += ncr
#if verbose print out information
if log:
message='%25s[%1d] %i' % (infile, i, ncr)
log.message(message, with_header=False, with_stdout=verbose)
#correct the BPM frame
if saltkey.found('BPMEXT', struct[i]):
b_i=saltkey.get('BPMEXT', struct[i])
try:
struct[b_i].data[mask]=1
except Exception, e:
msg='Cannot update the BPM frame in %s[%i] because %s' % (infile, b_i, e)
raise SaltError(msg)
def make_mosaic(struct,
gap,
xshift,
yshift,
rotation,
interp_type='linear',
boundary='constant',
constant=0,
geotran=True,
fill=False,
cleanup=True,
log=None,
verbose=False):
"""Given a SALT image struct, combine each of the individual amplifiers and
apply the geometric CCD transformations to the image
"""
# get the name of the file
infile = saltkey.getimagename(struct[0], base=True)
outpath = './'
# identify instrument
instrume, keyprep, keygain, keybias, keyxtalk, keyslot = \
saltkey.instrumid(struct)
# how many amplifiers?
nsciext = saltkey.get('NSCIEXT', struct[0])
nextend = saltkey.get('NEXTEND', struct[0])
nccds = saltkey.get('NCCDS', struct[0])
amplifiers = nccds * 2
if nextend > nsciext:
varframe = True
else:
varframe = False
# CCD geometry coefficients
if (instrume == 'RSS' or instrume == 'PFIS'):
xsh = [0., xshift[0], 0., xshift[1]]
ysh = [0., yshift[0], 0., yshift[1]]
rot = [0., rotation[0], 0., rotation[1]]
elif instrume == 'SALTICAM':
xsh = [0., xshift[0], 0.]
ysh = [0., yshift[0], 0.]
rot = [0., rotation[0], 0]
# how many extensions?
nextend = saltkey.get('NEXTEND', struct[0])
# CCD on-chip binning
xbin, ybin = saltkey.ccdbin(struct[0])
# create temporary primary extension
outstruct = []
outstruct.append(struct[0])
# define temporary FITS file store tiled CCDs
tilefile = saltio.tmpfile(outpath)
tilefile += 'tile.fits'
if varframe:
tilehdu = [None] * (3 * int(nsciext / 2) + 1)
else:
tilehdu = [None] * int(nsciext / 2 + 1)
tilehdu[0] = fits.PrimaryHDU()
#tilehdu[0].header = struct[0].header
if log:
log.message('', with_stdout=verbose)
# iterate over amplifiers, stich them to produce file of CCD images
for i in range(int(nsciext / 2)):
hdu = i * 2 + 1
# amplifier = hdu%amplifiers
# if (amplifier == 0): amplifier = amplifiers
# read DATASEC keywords
datasec1 = saltkey.get('DATASEC', struct[hdu])
datasec2 = saltkey.get('DATASEC', struct[hdu + 1])
xdsec1, ydsec1 = saltstring.secsplit(datasec1)
xdsec2, ydsec2 = saltstring.secsplit(datasec2)
# read images
imdata1 = saltio.readimage(struct, hdu)
imdata2 = saltio.readimage(struct, hdu + 1)
# tile 2n amplifiers to yield n CCD images
outdata = numpy.zeros(
(int(ydsec1[1] + abs(ysh[i + 1] / ybin)),
int(xdsec1[1] + xdsec2[1] + abs(xsh[i + 1] / xbin))),
numpy.float32)
# set up the variance frame
if varframe:
vardata = outdata.copy()
vdata1 = saltio.readimage(struct, struct[hdu].header['VAREXT'])
vdata2 = saltio.readimage(struct, struct[hdu + 1].header['VAREXT'])
bpmdata = outdata.copy()
bdata1 = saltio.readimage(struct, struct[hdu].header['BPMEXT'])
bdata2 = saltio.readimage(struct, struct[hdu + 1].header['BPMEXT'])
x1 = xdsec1[0] - 1
if x1 != 0:
msg = 'The data in %s have not been trimmed prior to mosaicking.' \
% infile
log.error(msg)
if xsh[i + 1] < 0:
x1 += int(abs(xsh[i + 1] / xbin))
x2 = x1 + xdsec1[1]
y1 = ydsec1[0] - 1
if ysh[i + 1] < 0:
y1 += int(abs(ysh[i + 1] / ybin))
y2 = y1 + ydsec1[1]
outdata[y1:y2, x1:x2] =\
imdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata1[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
x1 = x2
x2 = x1 + xdsec2[1]
y1 = ydsec2[0] - 1
if ysh[i + 1] < 0:
y1 += abs(ysh[i + 1] / ybin)
y2 = y1 + ydsec2[1]
outdata[y1:y2, x1:x2] =\
imdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
if varframe:
vardata[y1:y2, x1:x2] =\
vdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
bpmdata[y1:y2, x1:x2] =\
bdata2[ydsec1[0] - 1:ydsec1[1], xdsec1[0] - 1:xdsec1[1]]
# size of new image
naxis1 = str(xdsec1[1] + xdsec2[1])
naxis2 = str(ydsec1[1])
# add image and keywords to HDU list
tilehdu[i + 1] = fits.ImageHDU(outdata)
tilehdu[i + 1].header = struct[hdu].header
#tilehdu[
# i + 1].header['DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
if varframe:
vext = i + 1 + int(nsciext / 2.)
tilehdu[vext] = fits.ImageHDU(vardata)
#tilehdu[vext].header = struct[struct[hdu].header['VAREXT']].header
#tilehdu[vext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
bext = i + 1 + 2 * int(nsciext / 2.)
tilehdu[bext] = fits.ImageHDU(bpmdata)
#tilehdu[bext].header = struct[struct[hdu].header['BPMEXT']].header
#tilehdu[bext].header[
# 'DATASEC'] = '[1:' + naxis1 + ',1:' + naxis2 + ']'
# image tile log message #1
if log:
message = os.path.basename(infile) + '[' + str(hdu) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
message = os.path.basename(infile) + '[' + str(hdu + 1) + ']['
message += str(xdsec1[0]) + ':' + str(xdsec1[1]) + ','
message += str(ydsec1[0]) + ':' + str(ydsec1[1]) + '] --> '
message += os.path.basename(tilefile) + '[' + str(i + 1) + ']['
message += str(xdsec1[1] + 1) + ':' + \
str(xdsec1[1] + xdsec2[1]) + ','
message += str(ydsec2[0]) + ':' + str(ydsec2[1]) + ']'
log.message(message, with_stdout=verbose, with_header=False)
# write temporary file of tiled CCDs
hdulist = fits.HDUList(tilehdu)
hdulist.writeto(tilefile)
# iterate over CCDs, transform and rotate images
yrot = [None] * 4
xrot = [None] * 4
tranfile = [' ']
tranhdu = [0]
if varframe:
tranfile = [''] * (3 * int(nsciext / 2) + 1)
tranhdu = [0] * (3 * int(nsciext / 2) + 1)
else:
tranfile = [''] * int(nsciext / 2 + 1)
tranhdu = [0] * int(nsciext / 2 + 1)
# this is hardwired for SALT where the second CCD is considered the
# fiducial
for hdu in range(1, int(nsciext / 2 + 1)):
tranfile[hdu] = saltio.tmpfile(outpath)
tranfile[hdu] += 'tran.fits'
if varframe:
tranfile[hdu + nccds] = saltio.tmpfile(outpath) + 'tran.fits'
tranfile[hdu + 2 * nccds] = saltio.tmpfile(outpath) + 'tran.fits'
ccd = hdu % nccds
if (ccd == 0):
ccd = nccds
# correct rotation for CCD binning
yrot[ccd] = rot[ccd] * ybin / xbin
xrot[ccd] = rot[ccd] * xbin / ybin
dxshift = xbin * int(float(int(gap) / xbin) + 0.5) - gap
# transformation using geotran IRAF task
# if (ccd == 1):
if (ccd != 2):
if geotran:
message = '\nSALTMOSAIC -- geotran ' + tilefile + \
'[' + str(ccd) + '] ' + tranfile[hdu]
message += ' \"\" \"\" xshift=' + \
str((xsh[ccd] + (2 - ccd) * dxshift) / xbin) + ' '
message += 'yshift=' + \
str(ysh[ccd] / ybin) + ' xrotation=' + str(xrot[ccd]) + ' '
message += 'yrotation=' + \
str(yrot[ccd]) + ' xmag=1 ymag=1 xmin=\'INDEF\''
message += 'xmax=\'INDEF\' ymin=\'INDEF\' ymax=\'INDEF\' '
message += 'ncols=\'INDEF\' '
message += 'nlines=\'INDEF\' verbose=\'no\' '
message += 'fluxconserve=\'yes\' nxblock=2048 '
message += 'nyblock=2048 interpolant=\'' + \
interp_type + '\' boundary=\'constant\' constant=0'
log.message(message, with_stdout=verbose)
yd, xd = tilehdu[ccd].data.shape
ncols = 'INDEF' # ncols=xd+abs(xsh[ccd]/xbin)
nlines = 'INDEF' # nlines=yd+abs(ysh[ccd]/ybin)
geo_xshift = xsh[ccd] + (2 - ccd) * dxshift / xbin
geo_yshift = ysh[ccd] / ybin
iraf.images.immatch.geotran(tilefile + "[" + str(ccd) + "]",
tranfile[hdu],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1,
ymag=1,
xmin='INDEF',
xmax='INDEF',
ymin='INDEF',
ymax='INDEF',
ncols=ncols,
nlines=nlines,
verbose='no',
fluxconserve='yes',
nxblock=2048,
nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
if varframe:
var_infile = tilefile + "[" + str(ccd + nccds) + "]"
iraf.images.immatch.geotran(var_infile,
tranfile[hdu + nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1,
ymag=1,
xmin='INDEF',
xmax='INDEF',
ymin='INDEF',
ymax='INDEF',
ncols=ncols,
nlines=nlines,
verbose='no',
fluxconserve='yes',
nxblock=2048,
nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
var2_infile = tilefile + "[" + str(ccd + 2 * nccds) + "]"
iraf.images.immatch.geotran(var2_infile,
tranfile[hdu + 2 * nccds],
"",
"",
xshift=geo_xshift,
yshift=geo_yshift,
xrotation=xrot[ccd],
yrotation=yrot[ccd],
xmag=1,
ymag=1,
xmin='INDEF',
xmax='INDEF',
ymin='INDEF',
ymax='INDEF',
ncols=ncols,
nlines=nlines,
verbose='no',
fluxconserve='yes',
nxblock=2048,
nyblock=2048,
interpolant="linear",
boundary="constant",
constant=0)
# open the file and copy the data to tranhdu
tstruct = fits.open(tranfile[hdu])
tranhdu[hdu] = tstruct[0].data
tstruct.close()
if varframe:
tranhdu[hdu + nccds] = fits.open(tranfile[hdu +
nccds])[0].data
tranhdu[hdu + 2 * nccds] = fits.open(
tranfile[hdu + 2 * nccds])[0].data
else:
log.message("Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd, xsh[ccd] / 2.0, ysh[ccd] / 2.0, xrot[ccd]),
with_stdout=verbose,
with_header=False)
tranhdu[hdu] = geometric_transform(
tilehdu[ccd].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1,
xrot[ccd], yrot[ccd]))
tstruct = fits.PrimaryHDU(tranhdu[hdu])
tstruct.writeto(tranfile[hdu])
if varframe:
tranhdu[hdu + nccds] = geometric_transform(
tilehdu[hdu + 3].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1,
xrot[ccd], yrot[ccd]))
tranhdu[hdu + 2 * nccds] = geometric_transform(
tilehdu[hdu + 6].data,
tran_func,
prefilter=False,
order=1,
extra_arguments=(xsh[ccd] / 2, ysh[ccd] / 2, 1, 1,
xrot[ccd], yrot[ccd]))
else:
log.message("Transform CCD #%i using dx=%s, dy=%s, rot=%s" %
(ccd, 0, 0, 0),
with_stdout=verbose,
with_header=False)
tranhdu[hdu] = tilehdu[ccd].data
if varframe:
tranhdu[hdu + nccds] = tilehdu[ccd + nccds].data
tranhdu[hdu + 2 * nccds] = tilehdu[ccd + 2 * nccds].data
# open outfile
if varframe:
outlist = 4 * [None]
else:
outlist = 2 * [None]
#outlist[0] = struct[0].copy()
outlist[0] = fits.PrimaryHDU()
outlist[0].header = struct[0].header
naxis1 = int(gap / xbin * (nccds - 1))
naxis2 = 0
for i in range(1, nccds + 1):
yw, xw = tranhdu[i].shape
naxis1 += xw + int(abs(xsh[ccd] / xbin)) + 1
naxis2 = max(naxis2, yw)
outdata = numpy.zeros((naxis2, naxis1), numpy.float32)
outdata.shape = naxis2, naxis1
if varframe:
vardata = outdata * 0
bpmdata = outdata * 0 + 1
# iterate over CCDs, stich them to produce a full image
hdu = 0
totxshift = 0
for hdu in range(1, nccds + 1):
# read DATASEC keywords
ydsec, xdsec = tranhdu[hdu].shape
# define size and shape of final image
# tile CCDs to yield mosaiced image
x1 = int((hdu - 1) * (xdsec + gap / xbin)) + int(totxshift)
x2 = xdsec + x1
y1 = int(0)
y2 = int(ydsec)
outdata[y1:y2, x1:x2] = tranhdu[hdu]
totxshift += int(abs(xsh[hdu] / xbin)) + 1
if varframe:
vardata[y1:y2, x1:x2] = tranhdu[hdu + nccds]
bpmdata[y1:y2, x1:x2] = tranhdu[hdu + 2 * nccds]
# make sure to cover up all the gaps include bad areas
if varframe:
baddata = (outdata == 0)
baddata = nd.maximum_filter(baddata, size=3)
bpmdata[baddata] = 1
# fill in the gaps if requested
if fill:
if varframe:
outdata = fill_gaps(outdata, 0)
else:
outdata = fill_gaps(outdata, 0)
# add to the file
outlist[1] = fits.ImageHDU(outdata)
if varframe:
outlist[2] = fits.ImageHDU(vardata, name='VAR')
outlist[3] = fits.ImageHDU(bpmdata, name='BPM')
# create the image structure
outstruct = fits.HDUList(outlist)
# update the head informaation
# housekeeping keywords
saltkey.put('NEXTEND', 2, outstruct[0])
saltkey.new('EXTNAME', 'SCI', 'Extension name', outstruct[1])
saltkey.new('EXTVER', 1, 'Extension number', outstruct[1])
if varframe:
saltkey.new('VAREXT', 2, 'Variance frame extension', outstruct[1])
saltkey.new('BPMEXT', 3, 'BPM Extension', outstruct[1])
try:
saltkey.copy(struct[1], outstruct[1], 'CCDSUM')
except:
pass
# Add keywords associated with geometry
saltkey.new('SGEOMGAP', gap, 'SALT Chip Gap', outstruct[0])
c1str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[0], yshift[0], rotation[0])
saltkey.new('SGEOM1', c1str, 'SALT Chip 1 Transform', outstruct[0])
c2str = '{:3.2f} {:3.2f} {:3.4f}'.format(xshift[1], yshift[1], rotation[1])
saltkey.new('SGEOM2', c2str, 'SALT Chip 2 Transform', outstruct[0])
# WCS keywords
saltkey.new('CRPIX1', 0, 'WCS: X reference pixel', outstruct[1])
saltkey.new('CRPIX2', 0, 'WCS: Y reference pixel', outstruct[1])
saltkey.new('CRVAL1', float(xbin), 'WCS: X reference coordinate value',
outstruct[1])
saltkey.new('CRVAL2', float(ybin), 'WCS: Y reference coordinate value',
outstruct[1])
saltkey.new('CDELT1', float(xbin), 'WCS: X pixel size', outstruct[1])
saltkey.new('CDELT2', float(ybin), 'WCS: Y pixel size', outstruct[1])
saltkey.new('CTYPE1', 'pixel', 'X type', outstruct[1])
saltkey.new('CTYPE2', 'pixel', 'Y type', outstruct[1])
# cleanup temporary files
if cleanup:
for tfile in tranfile:
if os.path.isfile(tfile):
saltio.delete(tfile)
if os.path.isfile(tilefile):
status = saltio.delete(tilefile)
# return the file
return outstruct
def bias(struct,
subover=True,
trim=True,
subbias=False,
bstruct=None,
median=False,
function='polynomial',
order=3,
rej_lo=3,
rej_hi=3,
niter=10,
plotover=False,
log=None,
verbose=True):
"""Bias subtracts the bias levels from a frame. It will fit and subtract the overscan
region, trim the images, and subtract a master bias if required.
struct--image structure
subover--subtract the overscan region
trim--trim the image
subbias--subtract master bias
bstruct--master bias image structure
median--use the median instead of mean in image statistics
function--form to fit to the overscan region
order--order for the function
rej_lo--sigma of low points to reject in the fit
rej_hi--sigma of high points to reject in the fit
niter--number of iterations
log--saltio log for recording information
verbose--whether to print to stdout
"""
infile = saltkey.getimagename(struct[0])
# how many extensions?
nsciext = saltkey.get('NSCIEXT', struct[0])
nextend = saltkey.get('NEXTEND', struct[0])
nccd = saltkey.get('NCCDS', struct[0])
# how many amplifiers?--this is hard wired
amplifiers = 2 * nccd
#log the process
if subover and log:
message = '%28s %7s %5s %4s %6s' % \
('HDU','Overscan','Order','RMS','Niter')
log.message(
'\n --------------------------------------------------',
with_header=False,
with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' --------------------------------------------------',
with_header=False,
with_stdout=verbose)
if (plotover):
plt.figure(1)
plt.axes([0.1, 0.1, 0.8, 0.8])
plt.xlabel('CCD Column')
plt.ylabel('Pixel Counts (e-)')
plt.ion()
#loop through the extensions and subtract the bias
for i in range(1, nsciext + 1):
if struct[i].name == 'SCI':
#get the bias section
biassec = saltkey.get('BIASSEC', struct[i])
y1, y2, x1, x2 = saltio.getSection(biassec, iraf_format=True)
#get the data section
datasec = saltkey.get('DATASEC', struct[i])
dy1, dy2, dx1, dx2 = saltio.getSection(datasec, iraf_format=True)
#setup the overscan region
if subover:
yarr = np.arange(y1, y2, dtype=float)
data = struct[i].data
odata = struct[i].data[y1:y2, x1:x2]
if median:
odata = np.median((struct[i].data[y1:y2, x1:x2]), axis=1)
olevel = np.median((struct[i].data[y1:y2, x1:x2]))
saltkey.new('OVERSCAN', '%f' % (olevel),
'Overscan median value', struct[i])
else:
odata = np.mean((struct[i].data[y1:y2, x1:x2]), axis=1)
olevel = np.mean((struct[i].data[y1:y2, x1:x2]))
saltkey.new('OVERSCAN', '%f' % (olevel),
'Overscan mean value', struct[i])
#fit the overscan region
ifit=saltfit.interfit(yarr, odata, function=function, \
order=order, thresh=rej_hi, niter=niter)
try:
ifit.interfit()
coeffs = ifit.coef
ofit = ifit(yarr)
omean, omed, osigma = saltstat.iterstat((odata - ofit),
sig=3,
niter=5)
except ValueError:
#catch the error if it is a zero array
ofit = np.array(yarr) * 0.0
osigma = 0.0
except TypeError:
#catch the error if it is a zero array
ofit = np.array(yarr) * 0.0
osigma = 0.0
#if it hasn't been already, convert image to
#double format
struct[i].data = 1.0 * struct[i].data
try:
struct[i].header.remove('BZERO')
struct[i].header.remove('BSCALE')
except:
pass
#subtract the overscan region
for j in range(len(struct[i].data[0])):
struct[i].data[y1:y2, j] -= ofit
#report the information
if log:
message = '%25s[%1d] %8.2f %3d %7.2f %3d' % \
(infile, i, olevel, order, osigma, niter)
log.message(message,
with_stdout=verbose,
with_header=False)
#add the statistics to the image header
saltkey.new('OVERRMS', '%f' % (osigma), 'Overscan RMS value',
struct[i])
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu = saltkey.get('VAREXT', struct[i])
try:
vdata = struct[vhdu].data
#The bias level should not be included in the noise from the signal
for j in range(len(struct[i].data[0])):
vdata[y1:y2, j] -= ofit
#add a bit to make sure that the minimum error is the rednoise
rdnoise = saltkey.get('RDNOISE', struct[i])
vdata[vdata < rdnoise**2] = rdnoise**2
struct[vhdu].data = vdata + osigma**2
except Exception, e:
msg = 'Cannot update the variance frame in %s[%i] because %s' % (
infile, vhdu, e)
raise SaltError(msg)
#plot the overscan region
if plotover:
plt.plot(yarr, odata)
plt.plot(yarr, ofit)
#trim the data and update the headers
if trim:
struct[i].data = struct[i].data[dy1:dy2, dx1:dx2]
datasec = '[1:' + str(dx2 - dx1) + ',1:' + str(dy2 - dy1) + ']'
saltkey.put('DATASEC', datasec, struct[i])
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu = saltkey.get('VAREXT', struct[i])
struct[vhdu].data = struct[vhdu].data[dy1:dy2, dx1:dx2]
datasec = '[1:' + str(dx2 - dx1) + ',1:' + str(dy2 -
dy1) + ']'
saltkey.put('DATASEC', datasec, struct[vhdu])
#update the BPM frame
if saltkey.found('BPMEXT', struct[i]):
bhdu = saltkey.get('BPMEXT', struct[i])
struct[bhdu].data = struct[bhdu].data[dy1:dy2, dx1:dx2]
datasec = '[1:' + str(dx2 - dx1) + ',1:' + str(dy2 -
dy1) + ']'
saltkey.put('DATASEC', datasec, struct[bhdu])
#subtract the master bias if necessary
if subbias and bstruct:
struct[i].data -= bstruct[i].data
#update the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu = saltkey.get('VAREXT', struct[i])
try:
vdata = struct[vhdu].data
struct[vhdu].data = vdata + bstruct[vhdu].data
except Exception, e:
msg = 'Cannot update the variance frame in %s[%i] because %s' % (
infile, vhdu, e)
raise SaltError(msg)
def xtalk(struct, xcoeff, namps=2, log=None, verbose=False):
"""xtalk cross-talk corrects the amplifies. It takes
"""
infile = saltkey.getimagename(struct[0])
# how many extensions?
nsciext = saltkey.get('NSCIEXT', struct[0])
nextend = saltkey.get('NEXTEND', struct[0])
nccd = saltkey.get('NCCDS', struct[0])
# how many amplifiers?--this is hard wired
amplifiers = namps * nccd
#setup the log
if log:
message = '%28s %23s' % ('HDU', 'Correction')
message += '\n ----------------------------------------------'
log.message(message, with_header=False, with_stdout=verbose)
#Loop over the image extensions and subtract one
#set from the other--still hardwired at 2
for i in range(1, nsciext, 2):
if struct[i].name == 'SCI' and struct[i + 1].name == 'SCI':
#set up the first amplifier
dat1 = struct[i].data.copy()
ext1 = saltkey.get('EXTVER', struct[i])
if xcoeff:
j = (ext1 - 1) % amplifiers
xc1 = float(xcoeff[j])
else:
xc1 = 1.0 / saltkey.get('XTALK', struct[i])
#set up the second amplifier
dat2 = struct[i + 1].data.copy()
ext2 = saltkey.get('EXTVER', struct[i + 1])
if xcoeff:
j = (ext2 - 1) % amplifiers
xc2 = float(xcoeff[j])
else:
xc2 = 1.0 / saltkey.get('XTALK', struct[i + 1])
#subtract one from the other
struct[i].data = struct[i].data - xc2 * dat2[:, ::-1]
struct[i + 1].data = struct[i + 1].data - xc1 * dat1[:, ::-1]
#correct the variance frame
if saltkey.found('VAREXT', struct[i]):
vhdu1 = saltkey.get('VAREXT', struct[i])
vhdu2 = saltkey.get('VAREXT', struct[i + 1])
try:
struct[vhdu1].data += xc2 * struct[vhdu2].data
struct[vhdu2].data += xc1 * struct[vhdu1].data
except Exception, e:
msg = 'Cannot update the variance frame in %s[%i] because %s' % (
infile, vhdu1, e)
raise SaltError(msg)
#print the message
if log:
message = '%25s[%1d] Amp%1d - Amp%1d * %8.6f' % \
(infile, i, ext1, ext2, xc2)
log.message(message, with_header=False, with_stdout=verbose)
message = '%25s[%1d] Amp%1d - Amp%1d * %8.6f' % \
(infile, i+1, ext2, ext1, xc1)
log.message(message, with_header=False, with_stdout=verbose)
def gain(struct,
mult=True,
usedb=False,
dblist=None,
ampccd=2,
log=None,
verbose=True):
"""gain processes a image hduList and gain corrects each amplifier. It can
either use gain settings in the header or those supplied in a config file
which would be suppleid in the dblist (see helpfile for structure of the
config file). If variance frames exist, it will update those for changes
in the header value as well.
In the end, it will update the gain with a value of one signfing the data
has been transformed into e- from ADU
The program will look for the non-linear gain settings which are given by:
e = GAIN*(1 + GAIN1*E-6*ADU)*ADU
mult--if true, multiple the gains
usedb--use the values in the dblist, if false use the header values
dblist--values for the gain and readnoise from the
ampccd--number of amplifiers per ccd
dblist should have the following lists: speed, rate, gain, noise, bias, amp
"""
#get the infile name
infile = saltkey.getimagename(struct[0])
#how many science extensions
nsciext = saltkey.get('NSCIEXT', struct[0])
#how many data extensions
nextend = saltkey.get('NSCIEXT', struct[0])
# how many amplifiers?
amplifiers = ampccd * saltkey.get('NCCDS', struct[0])
#read the gain and rospeed for the image
gainset = saltkey.get('GAINSET', struct[0])
rospeed = saltkey.get('ROSPEED', struct[0])
#loop through each amplifier and gain correct it
if log:
message = '%28s %6s %5s %3s %5s %5s' \
% ('HDU','GAIN','SPEED','AMP','GAIN','NOISE')
log.message('\n ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
for i in range(nsciext):
hdu = i + 1
amp = i % amplifiers + 1
#get the gain and rdnoise values for the array
if usedb:
gain, rdnoise = get_values(dblist, gainset, rospeed, amp)
gain1 = 0
else:
gain = saltkey.get('GAIN', struct[hdu])
rdnoise = saltkey.get('RDNOISE', struct[hdu])
try:
gain1 = saltkey.get('GAIN1', struct[hdu])
except:
gain1 = 0
if mult:
#correct the gain
gainmult = 1
try:
data = struct[hdu].data
struct[hdu].data = gain * data + gain1 * data**2
except Exception, e:
msg = 'Cannot gain correct %s[%i] because %s' % (infile, hdu,
e)
raise SaltError(msg)
#correct the variance frame
if saltkey.found('VAREXT', struct[hdu]):
vhdu = saltkey.get('VAREXT', struct[hdu])
try:
vdata = struct[vhdu].data
struct[vhdu].data = vdata * gain * (
1 + 2 * gain1 * 1e-6 * data)
except Exception, e:
msg = 'Cannot update the variance frame in %s[%i] because %s' % (
infile, vhdu, e)
raise SaltError(msg)
def crclean(struct, crtype='fast', thresh=5, mbox=5, bbox=11, bthresh=3, flux_ratio=0.2, \
gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=1, update=True, log=None, verbose=True):
"""CRCLEAN cleans SALT-like data of cosmic rays. The user has
three different choices for the type of cosmic ray cleaning being
fast, median, and edge.
crytpe--type of cosmic ray cleaning. Either fast, median, or
thresh--threshold for detecting cosmic rays
mbox--box for median cleaning
bbox--background box for median measurement
bthresh--threshold for iterating on background calculation
flux_ratio--ratio of fluxes for 'fast' method
gain--gain of images--set to None to read in from header
rdnoise--read noise of images--set to None to read in from header
bfactor--block replication factor for 'edge' method
fthresh--threshold for excluding compact sources (edge only)
gbox--Window size to grow sources. gbox=0 for no growth of cosmic rays
maxiter--maximum number of iterations
return struct
"""
#setup the image name
infile = saltkey.getimagename(struct[0])
#count the CR
totcr = 0
#print out the header for the log
if log:
message = '%28s %11s' % ('HDU', 'COSMICRAYS')
log.message('\n ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
#cosmic ray clean each extension
for i in range(len(struct)):
#only clean the cosmic rays if it is a SCI extension or a single extension
if struct[i].name == 'SCI' or len(struct) == 1:
#for the edge method, get the gain and rdnoise from the fits header
#if they are not set
if crtype == 'edge':
if gain == None: gain = saltkey.get('GAIN', struct[i])
if rdnoise == None: rdnoise = saltkey.get('RDNOISE', struct[i])
#get all the cosmic rays from an array
crarr=cleancosmicrays(struct[i].data, crtype, thresh, mbox, bbox, bthresh, flux_ratio, \
gain, rdnoise, bfactor, fthresh, gbox, maxiter)
#update the frame for the various values
mask = (crarr > 0)
if update: struct[i].data[mask] = crarr[mask]
#track the number of cosmic rays
ncr = mask.sum()
totcr += ncr
#if verbose print out information
if log:
message = '%25s[%1d] %i' % (infile, i, ncr)
log.message(message, with_header=False, with_stdout=verbose)
#correct the BPM frame
if saltkey.found('BPMEXT', struct[i]):
b_i = saltkey.get('BPMEXT', struct[i])
try:
struct[b_i].data[mask] = 1
except Exception, e:
msg = 'Cannot update the BPM frame in %s[%i] because %s' % (
infile, b_i, e)
raise SaltError(msg)
def multicrclean(struct, crtype='fast', thresh=5, mbox=5, bbox=11, bthresh=3, flux_ratio=0.2, \
gain=1, rdnoise=5, bfactor=2, fthresh=5, gbox=0, maxiter=1, update=True, log=None, verbose=True):
"""MULTICRCLEAN cleans SALT-like data of cosmic rays. The user has
three different choices for the type of cosmic ray cleaning being
fast, median, and edge. The process is set up to use multithreading for
quick processing of the data.
crytpe--type of cosmic ray cleaning. Either fast, median, or
thresh--threshold for detecting cosmic rays
mbox--box for median cleaning
bbox--background box for median measurement
bthresh--threshold for iterating on background calculation
flux_ratio--ratio of fluxes for 'fast' method
gain--gain of images--set to None to read in from header
rdnoise--read noise of images--set to None to read in from header
bfactor--block replication factor for 'edge' method
fthresh--threshold for excluding compact sources (edge only)
gbox--Window size to grow sources. gbox=0 for no growth of cosmic rays
maxiter--maximum number of iterations
return struct
"""
#setup the image name
infile = saltkey.getimagename(struct[0])
#count the CR
totcr = 0
#print out the header for the log
if log:
message = '%28s %11s' % ('HDU', 'COSMICRAYS')
log.message('\n ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
task_list = []
nproc = 0
for hdu in struct:
if hdu.name == 'SCI':
task_list.append(
(hdu.data, crtype, thresh, mbox, bbox, bthresh, flux_ratio,
gain, rdnoise, bfactor, fthresh, gbox, maxiter))
nproc += 1
#set up the multi-thread
p = mp.Pool()
results = [p.apply_async(cleancosmicrays, i) for i in task_list]
p.close()
for i, hdu in enumerate(struct):
if hdu.name == 'SCI':
#set up the cosmic ray array
crarr = results[i - 1].get()
#update the frame for the various values
mask = (crarr > 0)
if update: struct[i].data[mask] = crarr[mask]
#track the number of cosmic rays
ncr = mask.sum()
totcr += ncr
#if verbose print out information
if log:
message = '%25s[%1d] %i' % (infile, i, ncr)
log.message(message, with_header=False, with_stdout=verbose)
#correct the BPM frame
if saltkey.found('BPMEXT', struct[i]):
b_i = saltkey.get('BPMEXT', struct[i])
try:
struct[b_i].data[mask] = 1
except Exception, e:
msg = 'Cannot update the BPM frame in %s[%i] because %s' % (
infile, b_i, e)
raise SaltError(msg)
def gain(struct,mult=True,usedb=False, dblist=None, ampccd=2, log=None, verbose=True):
"""gain processes a image hduList and gain corrects each amplifier. It can
either use gain settings in the header or those supplied in a config file
which would be suppleid in the dblist (see helpfile for structure of the
config file). If variance frames exist, it will update those for changes
in the header value as well.
In the end, it will update the gain with a value of one signfing the data
has been transformed into e- from ADU
The program will look for the non-linear gain settings which are given by:
e = GAIN*(1 + GAIN1*E-6*ADU)*ADU
mult--if true, multiple the gains
usedb--use the values in the dblist, if false use the header values
dblist--values for the gain and readnoise from the
ampccd--number of amplifiers per ccd
dblist should have the following lists: speed, rate, gain, noise, bias, amp
"""
#get the infile name
infile=saltkey.getimagename(struct[0])
#how many science extensions
nsciext = saltkey.get('NSCIEXT',struct[0])
#how many data extensions
nextend = saltkey.get('NSCIEXT',struct[0])
# how many amplifiers?
amplifiers = ampccd*saltkey.get('NCCDS',struct[0])
#read the gain and rospeed for the image
gainset = saltkey.get('GAINSET',struct[0])
rospeed = saltkey.get('ROSPEED',struct[0])
#loop through each amplifier and gain correct it
if log:
message = '%28s %6s %5s %3s %5s %5s' \
% ('HDU','GAIN','SPEED','AMP','GAIN','NOISE')
log.message('\n ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
log.message(message, with_header=False, with_stdout=verbose)
log.message(' ---------------------------------------------------', \
with_header=False, with_stdout=verbose)
for i in range(nsciext):
hdu = i + 1
amp = i%amplifiers+1
#get the gain and rdnoise values for the array
if usedb:
gain, rdnoise=get_values(dblist, gainset, rospeed, amp)
gain1=0
else:
gain = saltkey.get('GAIN',struct[hdu])
rdnoise = saltkey.get('RDNOISE',struct[hdu])
try:
gain1=saltkey.get('GAIN1',struct[hdu])
except:
gain1=0
if mult:
#correct the gain
gainmult=1
try:
data=struct[hdu].data
struct[hdu].data=gain*data+gain1*data**2
except Exception as e:
msg='Cannot gain correct %s[%i] because %s' % (infile, hdu, e)
raise SaltError(msg)
#correct the variance frame
if saltkey.found('VAREXT', struct[hdu]):
vhdu=saltkey.get('VAREXT', struct[hdu])
try:
vdata=struct[vhdu].data
struct[vhdu].data=vdata*gain*(1+2*gain1*1e-6*data)
except Exception as e:
msg='Cannot update the variance frame in %s[%i] because %s' % (infile, vhdu, e)
raise SaltError(msg)
else:
gainmult=gain
#update the headers
if usedb:
saltkey.put('GAIN',gain,struct[hdu])
saltkey.put('RDNOISE',rdnoise,struct[hdu])
#add a keyword indicating what action was taken
saltkey.new('GAINMULT',gainmult,'Gain multiplication', struct[hdu])
#if logging is true, then print out the following information
if log:
message = '%25s[%1d] %6s %5s %2s %6.2f %5.2f' \
% (infile,hdu,gainset,rospeed,amp, gain, rdnoise)
log.message(message, with_header=False, with_stdout=verbose)
#just to make it look pretty
if log:
log.message('', with_header=False, with_stdout=verbose)
return struct
