def _medfilter(self,im,r,rad):
"""
Median filtering of each slice in the cube
"""
log = self.log
pa = self.pa
bsize = self.boxcar_size # Boxcar smoothing size (def:5)
mdfw = self.medfilt_width # Medfiltering width (def:11)
# to accelerate the code, we bin the image and the 'r'
# image down to 256x256 and do the radial profile on the binned versions
sz = np.shape(im)
bsy = sz[0]/4
bsx = sz[1]/4
rbin = nt.rebin(r,bsy,bsx)
tmp = np.zeros(self.imsize,dtype=np.float32)
# COPY: We are not modifying input image
tmp[:,:] = im
#NOTE: We are not using reset_num so far
imbin = nt.rebin(tmp, bsy,bsx) # bin down to 256x256
for rr in range(8,np.size(rad)):
# determine the median profile beyond r=8
# (irrelevant inside this because of focal plane mask)
if rr < 50:
ss = tmp[abs(r-rr) < .5]
rad[rr] = np.median(ss[np.where(np.isfinite(ss))])
else:
ss = imbin[abs(rbin-rr) < 3]
rad[rr] = np.median(ss[np.where(np.isfinite(ss))])
# smooth the radial profile beyond r=22
rad[22:] = boxcar(rad,(bsize,))[22:]
rad[40:] = boxcar(rad,(bsize,))[40:] # smooth even more
tmp[:,:] -= gi.ginterpolate(rad,r)
tmp[:,:] -= scipy.signal.medfilt2d(np.nan_to_num(tmp),mdfw)
#t = time.time()
#line = '%.2f' % (t-ti)
#log.info('medfilter['+chan+']: ['+str(i+1)+'/'+str(sz[0])+']' +line)
return tmp
def smooth(image,size): ''' Smooth a 2d image array using a boxcar of size given by size ''' # print 'smooth',numpy.max(image),numpy.min(image),numpy.median(image) z=convolve.boxcar(image,(size,size),mode='constant',cval=0.0) # print 'smooth',numpy.max(z),numpy.min(z),numpy.median(z) return z
def test_boxcar9():
a = np.zeros((10,10))
a[3:6,3:6] = 111
result = convolve.boxcar(a, (3,3)).astype(np.longlong)
test = np.array([[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 12, 24, 37, 24, 12, 0, 0, 0],
[ 0, 0, 24, 49, 74, 49, 24, 0, 0, 0],
[ 0, 0, 37, 74, 111, 74, 37, 0, 0, 0],
[ 0, 0, 24, 49, 74, 49, 24, 0, 0, 0],
[ 0, 0, 12, 24, 37, 24, 12, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], dtype=np.int64)
assert_equal(result.all(),test.all())
def test_boxcar8():
a = np.zeros((10,10))
a[0,0] = 100
a[5,5] = 1000
a[9,9] = 10000
result = convolve.boxcar(a, (3,3), mode="constant").astype(np.longlong)
test = np.array([[ 11, 11, 0, 0, 0, 0, 0, 0, 0, 0],
[ 11, 11, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 111, 111, 111, 0, 0, 0],
[ 0, 0, 0, 0, 111, 111, 111, 0, 0, 0],
[ 0, 0, 0, 0, 111, 111, 111, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 1111, 1111],
[ 0, 0, 0, 0, 0, 0, 0, 0, 1111, 1111]], dtype=np.int64)
assert_equal(result.all(),test.all())
def test_boxcar7():
a = np.zeros((10,10))
a[0,0] = 100
a[5,5] = 1000
a[9,9] = 10000
result = convolve.boxcar(a, (3,3), mode="reflect").astype(np.longlong)
test = np.array([[ 44, 22, 0, 0, 0, 0, 0, 0, 0, 0],
[ 22, 11, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 111, 111, 111, 0, 0, 0],
[ 0, 0, 0, 0, 111, 111, 111, 0, 0, 0],
[ 0, 0, 0, 0, 111, 111, 111, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[ 0, 0, 0, 0, 0, 0, 0, 0, 1111, 2222],
[ 0, 0, 0, 0, 0, 0, 0, 0, 2222, 4444]], dtype=np.int64)
assert_equal(result.all(),test.all())
def test_boxcar1():
"""
boxcar computes a 1D or 2D boxcar filter on every 1D or 2D subarray of data.
'boxshape' is a tuple of integers specifying the dimensions of the filter: e.g. (3,3)
if 'output' is specified, it should be the same shape as 'data' and
None will be returned.
supported 'mode's include:
'nearest' elements beyond boundary come from nearest edge pixel.
'wrap' elements beyond boundary come from the opposite array edge.
'reflect' elements beyond boundary come from reflection on same array edge.
'constant' elements beyond boundary are set to 'cval'
"""
result = convolve.boxcar(np.array([10, 0, 0, 0, 0, 0, 1000]), (3,), mode="nearest").astype(np.longlong)
test = np.array([ 6, 3, 0, 0, 0, 333, 666], dtype=np.int64)
assert_equal(result.all(),test.all())
def __init__(
self,
imageList, # list of input data to be combined.
weightImageList, # list of input data weight images to be combined.
readnoiseList, # list of readnoise values to use for the input images.
exposureTimeList, # list of exposure times to use for the input images.
backgroundValueList, # list of image background values to use for the input images
weightMaskList=None, # list of imput data weight masks to use for pixel rejection.
combine_grow=1, # Radius (pixels) for neighbor rejection
combine_nsigma1=4, # Significance for accepting minimum instead of median
combine_nsigma2=3, # Significance for accepting minimum instead of median
fillval=False # Turn on use of imedian/imean
):
warnings.warn(
"The 'minmed' class is deprecated and may be removed"
" in a future version. Use 'min_med()' instead.",
DeprecationWarning)
# Define input variables
self._imageList = imageList
self._weightImageList = weightImageList
self._weightMaskList = weightMaskList
self._exposureTimeList = exposureTimeList
self._readnoiseList = readnoiseList
self._backgroundValueList = backgroundValueList
self._numberOfImages = len(self._imageList)
self._combine_grow = combine_grow
self._combine_nsigma1 = combine_nsigma1
self._combine_nsigma2 = combine_nsigma2
if fillval:
combtype_mean = 'imean'
combtype_median = 'imedian'
else:
combtype_mean = 'mean'
combtype_median = 'median'
# Create a different median image based upon the number of images in the input list.
median_file = np.zeros(self._imageList[0].shape,
dtype=self._imageList[0].dtype)
if (self._numberOfImages == 2):
tmp = numCombine(self._imageList,
numarrayMaskList=self._weightMaskList,
combinationType=combtype_mean,
nlow=0,
nhigh=0,
nkeep=1,
upper=None,
lower=None)
median_file = tmp.combArrObj
else:
# The value of NHIGH=1 will cause problems when there is only 1 valid
# unmasked input image for that pixel due to a difference in behavior
# between 'numcombine' and 'iraf.imcombine'.
# This value may need to be adjusted on the fly based on the number of
# inputs and the number of masked values/pixel.
#
tmp = numCombine(self._imageList,
numarrayMaskList=self._weightMaskList,
combinationType=combtype_median,
nlow=0,
nhigh=1,
nkeep=1,
upper=None,
lower=None)
median_file = tmp.combArrObj
if self._weightMaskList in [None, []]:
self._weightMaskList = [
np.zeros(self._imageList[0].shape,
dtype=self._imageList[0].dtype)
] * len(self._imageList)
# The following section of code will address the problem caused by having
# a value of nhigh = 1. This will behave in a way similar to the way the
# IRAF task IMCOMBINE behaves. In order to accomplish this, the following
# procedure will be followed:
# 1) The input masks will be summed.
# 2) The science data will be summed.
# 3) In the locations of the summed mask where the sum is 1 less than the
# the total number of images, the value of that location in the summed
# sceince image will be used to replace the existing value in the
# existing median_file.
#
# This procuedure is being used to prevent too much data from being thrown
# out of the image. Take for example the case of 3 input images. In two
# of the images the pixel locations have been masked out. Now, if nhigh
# is applied there will be no value to use for that position. However,
# if this new procedure is used that value in the resulting images will
# be the value that was rejected by the nhigh rejection step.
#
# We need to make certain that "bad" pixels in the sci data are set to 0. That way,
# when the sci images are summed, the value of the sum will only come from the "good"
# pixels.
tmpList = []
for image in range(len(self._imageList)):
tmp = np.where(self._weightMaskList[image] == 1, 0,
self._imageList[image])
tmpList.append(tmp)
# Sum the mask files
maskSum = self._sumImages(self._weightMaskList)
# Sum the science images
sciSum = self._sumImages(tmpList)
del (tmpList)
# Use the summed sci image values in locations where the maskSum indicates
# that there is only 1 good pixel to use. The value will be used in the
# median_file image
median_file = np.where(maskSum == self._numberOfImages - 1, sciSum,
median_file)
if self._weightMaskList in [None, []]:
self._weightMaskList = [
np.zeros(self._imageList[0].shape,
dtype=self._imageList[0].dtype)
] * len(self._imageList)
# Sum the weightMaskList elements
maskSum = self._sumImages(self._weightMaskList)
# Create the minimum image from the stack of input images.
# Find the maximum pixel value for the image stack.
maxValue = -1e+9
for image in self._imageList:
newMax = image.max()
if (newMax > maxValue):
maxValue = newMax
# For each image, set pixels masked as "bad" to the "super-maximum" value.
for image in range(len(self._imageList)):
self._imageList[image] = np.where(self._weightMaskList[image] == 1,
maxValue + 1,
self._imageList[image])
# Call numcombine throwing out the highest N - 1 pixels.
tmp = numCombine(self._imageList,
numarrayMaskList=None,
combinationType=combtype_median,
nlow=0,
nhigh=self._numberOfImages - 1,
nkeep=1,
upper=None,
lower=None)
minimum_file = tmp.combArrObj
# Reset any pixl at maxValue + 1 to 0.
minimum_file = np.where(maskSum == self._numberOfImages, 0,
minimum_file)
# Scale the weight images by the background values and add them to the bk
backgroundFileList = []
for image in range(len(self._weightImageList)):
tmp = self._weightImageList[image] * (
self._backgroundValueList[image] /
(self._exposureTimeList[image]))
backgroundFileList.append(tmp)
# Create an image of the total effective background (in DN) per pixel:
# (which is the sum of all the background-scaled weight files)
#
bkgd_file = self._sumImages(backgroundFileList)
del (backgroundFileList)
#
# Scale the weight mask images by the square of the readnoise values
#
readnoiseFileList = []
for image in range(len(self._weightMaskList)):
tmp = (np.logical_not(self._weightMaskList[image]) *
(self._readnoiseList[image] * self._readnoiseList[image]))
readnoiseFileList.append(tmp)
# Create an image of the total readnoise**2 per pixel:
# (which is the sum of all the input readnoise values)
#
readnoise_file = self._sumImages(readnoiseFileList)
del (readnoiseFileList)
# Create an image of the total effective exposure time per pixel:
# (which is simply the sum of all the drizzle output weight files)
#
weight_file = self._sumImages(self._weightImageList)
# Scale up both the median and minimum arrays by the total effective exposure time
# per pixel.
#
minimum_file_weighted = minimum_file * weight_file
median_file_weighted = median_file * weight_file
del (weight_file)
# Calculate the 1-sigma r.m.s.:
# variance = median_electrons + bkgd_electrons + readnoise**2
# rms = sqrt(variance)
# This image has units of electrons.
#
# make this the abs value so that negative numbers dont throw an exception?
rms_file2 = np.fmax(median_file_weighted + bkgd_file + readnoise_file,
np.zeros_like(median_file_weighted))
rms_file = np.sqrt(rms_file2)
del bkgd_file
del readnoise_file
# For the median array, calculate the n-sigma lower threshold to the array
# and incorporate that into the pixel values.
#
median_rms_file = median_file_weighted - (rms_file *
self._combine_nsigma1)
if self._combine_grow != 0:
#
# Do a more sophisticated rejection: For all cases where the minimum pixel will
# be accepted instead of the median, set a lower threshold for that pixel and the
# ones around it (ie become less conservative in rejecting the median). This is
# because in cases of triple-incidence cosmic rays, quite often the low-lying
# outliers of the CRs can influence the median for the initial relatively high
# value of sigma, so a lower threshold must be used to mnake sure that the minimum
# is selected.
#
# This is done as follows:
# 1) make an image which is zero everywhere except where the minimum will be accepted
# 2) box-car smooth this image, to make these regions grow.
# 3) In the file "median_rms_file_electrons", replace these pixels
# by median - combine_nsigma2 * rms
#
# Then use this image in the final replacement, in the same way as for the
# case where this option is not selected.
minimum_flag_file = np.where(
np.less(minimum_file_weighted, median_rms_file), 1, 0)
# The box size value must be an integer. This is not a problem since __combine_grow should always
# be an integer type. The combine_grow column in the MDRIZTAB should also be an integer type.
boxsize = int(2 * self._combine_grow + 1)
boxshape = (boxsize, boxsize)
minimum_grow_file = np.zeros(self._imageList[0].shape,
dtype=self._imageList[0].dtype)
# If the boxcar convolution has failed it is potentially for two reasons:
# 1) The kernel size for the boxcar is bigger than the actual image.
# 2) The grow parameter was specified with a value < 0. This would result
# in an illegal boxshape kernel. The dimensions of the kernel box *MUST*
# be integer and greater than zero.
#
# If the boxcar convolution has failed, try to give a meaningfull explanation
# as to why based upon the conditionals described above.
if (boxsize <= 0):
errormsg1 = "############################################################\n"
errormsg1 += "# The boxcar convolution in minmed has failed. The 'grow' #\n"
errormsg1 += "# parameter must be greater than or equal to zero. You #\n"
errormsg1 += "# specified an input value for the 'grow' parameter of: #\n"
errormsg1 += " combine_grow: " + str(
self._combine_grow) + '\n'
errormsg1 += "############################################################\n"
raise ValueError(errormsg1)
if (boxsize > self._imageList[0].shape[0]):
errormsg2 = "############################################################\n"
errormsg2 += "# The boxcar convolution in minmed has failed. The 'grow' #\n"
errormsg2 += "# parameter specified has resulted in a boxcar kernel that #\n"
errormsg2 += "# has dimensions larger than the actual image. You #\n"
errormsg2 += "# specified an input value for the 'grow' parameter of: #\n"
errormsg2 += " combine_grow: " + str(
self._combine_grow) + '\n'
errormsg2 += "############################################################\n"
print(self._imageList[0].shape)
raise ValueError(errormsg2)
# Attempt the boxcar convolution using the boxshape based upon the user input value of "grow"
boxcar(minimum_flag_file,
boxshape,
output=minimum_grow_file,
mode='constant',
cval=0)
del (minimum_flag_file)
temp1 = (median_file_weighted - (rms_file * self._combine_nsigma1))
temp2 = (median_file_weighted - (rms_file * self._combine_nsigma2))
median_rms2_file = np.where(np.equal(minimum_grow_file, 0), temp1,
temp2)
del (temp1)
del (temp2)
del (rms_file)
del (minimum_grow_file)
# Finally decide whether to use the minimim or the median (in counts/s),
# based on whether the median is more than 3 sigma above the minimum.
#
self.combArrObj = np.where(
np.less(minimum_file_weighted, median_rms2_file), minimum_file,
median_file)
else:
# Finally decide whether to use the minimim or the median (in counts/s),
# based on whether the median is more than 3 sigma above the minimum.
#
self.combArrObj = np.where(
np.less(minimum_file_weighted, median_rms_file), minimum_file,
median_file)
# Set fill regions to a pixel value of 0.
self.combArrObj = np.where(maskSum == self._numberOfImages, 0,
self.combArrObj)
def min_med(images,
weight_images,
readnoise_list,
exptime_list,
background_values,
weight_masks=None,
combine_grow=1,
combine_nsigma1=4,
combine_nsigma2=3,
fillval=False):
""" Create a median array, rejecting the highest pixel and
computing the lowest valid pixel after mask application.
.. note::
In this version of the mimmed algorithm we assume that the units of
all input data is electons.
Parameters
----------
images : list of numpy.ndarray
List of input data to be combined.
weight_images : list of numpy.ndarray
List of input data weight images to be combined.
readnoise_list : list
List of readnoise values to use for the input images.
exptime_list : list
List of exposure times to use for the input images.
background_values : list
List of image background values to use for the input images.
weight_masks : list of numpy.ndarray, None
List of imput data weight masks to use for pixel rejection.
(Default: `None`)
combine_grow : int
Radius (pixels) for neighbor rejection. (Default: 1)
combine_nsigma1 : float
Significance for accepting minimum instead of median. (Default: 4)
combine_nsigma2 : float
Significance for accepting minimum instead of median. (Default: 3)
fillval : bool
Turn on use of imedian/imean. (Default: `False`)
Returns
-------
combined_array : numpy.ndarray
Combined array.
"""
# In this case we want to calculate two things:
# 1) the median array, rejecting the highest pixel (thus running
# imcombine with nlow=0, nhigh=1, nkeep=1, using the masks)
# 2) the lowest valid pixel after applying the masks (thus running
# imcombine with nlow=0, nhigh=3, nkeep=1, using the masks)
#
# We also calculate the sum of the weight files (to produce the total
# effective exposure time for each pixel).
#
# The total effective background in the final image is calculated as
# follows:
# - convert background for each input image to counts/s
# (divide by exptime)
# - multiply this value by the weight image, to obtain the effective
# background counts (in DN) for each pixel, for each image
# - Add these images together, to obtain the total effective background
# for the combined image.
#
# Once we've made these two files, then calculate the SNR based on the
# median-pixel image, and compare with the minimum.
nimages = len(images)
combtype_median = 'imedian' if fillval else 'median'
images = np.asarray(images)
weight_images = np.asarray(weight_images)
if weight_masks == [] or weight_masks is None:
weight_masks = None
mask_sum = np.zeros(images.shape[1:], dtype=np.int16)
all_bad_idx = np.array([], dtype=np.int)
all_bad_idy = np.array([], dtype=np.int)
else:
weight_masks = np.asarray(weight_masks, dtype=np.bool)
mask_sum = np.sum(weight_masks, axis=0, dtype=np.int16)
all_bad_idx, all_bad_idy = np.where(mask_sum == nimages)
# Create a different median image based upon the number of images in the
# input list.
if nimages == 2:
median_file = num_combine(
images,
masks=weight_masks,
combination_type='imean' if fillval else 'mean',
nlow=0,
nhigh=0,
lower=None,
upper=None)
else:
# The value of NHIGH=1 will cause problems when there is only 1 valid
# unmasked input image for that pixel due to a difference in behavior
# between 'num_combine' and 'iraf.imcombine'.
# This value may need to be adjusted on the fly based on the number of
# inputs and the number of masked values/pixel.
#
median_file = num_combine(images,
masks=weight_masks,
combination_type=combtype_median,
nlow=0,
nhigh=1,
lower=None,
upper=None)
# The following section of code will address the problem caused by
# having a value of nhigh = 1. This will behave in a way similar to
# the way the IRAF task IMCOMBINE behaves. In order to accomplish
# this, the following procedure will be followed:
# 1) The input masks will be summed.
# 2) The science data will be summed.
# 3) In the locations of the summed mask where the sum is 1 less than
# the total number of images, the value of that location in the
# summed science image will be used to replace the existing value
# in the existing median_file.
#
# This procedure is being used to prevent too much data from being
# thrown out of the image. Take for example the case of 3 input images.
# In two of the images the pixel locations have been masked out.
# Now, if nhigh is applied there will be no value to use for that
# position. However, if this new procedure is used that value in
# the resulting images will be the value that was rejected by the
# nhigh rejection step.
# We need to make certain that "bad" pixels in the sci data are set to
# 0. That way, when the sci images are summed, the value of the sum
# will only come from the "good" pixels.
if weight_masks is None:
sci_sum = np.sum(images, axis=0)
if nimages == 1:
median_file = sci_sum
else:
sci_sum = np.sum(images * np.logical_not(weight_masks), axis=0)
# Use the summed sci image values in locations where the mask_sum
# indicates that there is only 1 good pixel to use. The value will
# be used in the median_file image
idx = np.where(mask_sum == (nimages - 1))
median_file[idx] = sci_sum[idx]
# Create the minimum image from the stack of input images.
if weight_masks is not None:
# make a copy of images to avoid side-effect of modifying input
# argument:
images = images.copy()
images[weight_masks] = np.nan
images[:, all_bad_idx, all_bad_idy] = 0
minimum_file = np.nanmin(images, axis=0)
else:
minimum_file = np.amin(images, axis=0)
# Scale the weight images by the background values and add them to the bk
# Create an image of the total effective background (in DN) per pixel:
# (which is the sum of all the background-scaled weight files)
s = np.asarray(
[bv / et for bv, et in zip(background_values, exptime_list)])
bkgd_file = np.sum(weight_images * s[:, None, None], axis=0)
# Scale the weight mask images by the square of the readnoise values.
# Create an image of the total readnoise**2 per pixel
# (which is the sum of all the input readnoise values).
if weight_masks is None:
rdn2 = sum((r**2 for r in readnoise_list))
readnoise_file = rdn2 * np.ones_like(images[0])
else:
readnoise_file = np.sum(np.logical_not(weight_masks) *
(np.asarray(readnoise_list)**2)[:, None, None],
axis=0)
# Create an image of the total effective exposure time per pixel:
# (which is simply the sum of all the drizzle output weight files)
weight_file = np.sum(weight_images, axis=0)
# Scale up both the median and minimum arrays by the total effective
# exposure time per pixel.
minimum_file_weighted = minimum_file * weight_file
median_file_weighted = median_file * weight_file
del weight_file
# Calculate the 1-sigma r.m.s.:
# variance = median_electrons + bkgd_electrons + readnoise**2
# rms = sqrt(variance)
# This image has units of electrons.
#
# make this the abs value so that negative numbers dont throw an exception?
rms_file2 = np.fmax(median_file_weighted + bkgd_file + readnoise_file,
np.zeros_like(median_file_weighted))
rms_file = np.sqrt(rms_file2)
del bkgd_file, readnoise_file
# For the median array, calculate the n-sigma lower threshold to the array
# and incorporate that into the pixel values.
median_rms_file = median_file_weighted - rms_file * combine_nsigma1
if combine_grow != 0:
# Do a more sophisticated rejection: For all cases where the minimum
# pixel will be accepted instead of the median, set a lower threshold
# for that pixel and the ones around it (ie become less conservative
# in rejecting the median). This is because in cases of
# triple-incidence cosmic rays, quite often the low-lying outliers
# of the CRs can influence the median for the initial relatively high
# value of sigma, so a lower threshold must be used to mnake sure that
# the minimum is selected.
#
# This is done as follows:
# 1) make an image which is zero everywhere except where the minimum
# will be accepted
# 2) box-car smooth this image, to make these regions grow.
# 3) In the file "median_rms_file_electrons", replace these pixels
# by median - combine_nsigma2 * rms
#
# Then use this image in the final replacement, in the same way as for
# the case where this option is not selected.
minimum_flag_file = np.less(minimum_file_weighted,
median_rms_file).astype(np.float64)
# The box size value must be an integer. This is not a problem since
# __combine_grow should always be an integer type. The combine_grow
# column in the MDRIZTAB should also be an integer type.
boxsize = int(2 * combine_grow + 1)
boxshape = (boxsize, boxsize)
minimum_grow_file = np.zeros_like(images[0])
# If the boxcar convolution has failed it is potentially for
# two reasons:
# 1) The kernel size for the boxcar is bigger than the actual image.
# 2) The grow parameter was specified with a value < 0. This would
# result in an illegal boxshape kernel. The dimensions of the
# kernel box *MUST* be integer and greater than zero.
#
# If the boxcar convolution has failed, try to give a meaningfull
# explanation as to why based upon the conditionals described above.
if boxsize <= 0:
errormsg1 = "############################################################\n"
errormsg1 += "# The boxcar convolution in minmed has failed. The 'grow' #\n"
errormsg1 += "# parameter must be greater than or equal to zero. You #\n"
errormsg1 += "# specified an input value for the 'grow' parameter of: #\n"
errormsg1 += " combine_grow: " + str(combine_grow) + '\n'
errormsg1 += "############################################################\n"
raise ValueError(errormsg1)
if boxsize > images.shape[1]:
errormsg2 = "############################################################\n"
errormsg2 += "# The boxcar convolution in minmed has failed. The 'grow' #\n"
errormsg2 += "# parameter specified has resulted in a boxcar kernel that #\n"
errormsg2 += "# has dimensions larger than the actual image. You #\n"
errormsg2 += "# specified an input value for the 'grow' parameter of: #\n"
errormsg2 += " combine_grow: " + str(combine_grow) + '\n'
errormsg2 += "############################################################\n"
print(images.shape[1:])
raise ValueError(errormsg2)
# Attempt the boxcar convolution using the boxshape based upon the user
# input value of "grow"
boxcar(minimum_flag_file,
boxshape,
output=minimum_grow_file,
mode='constant',
cval=0)
del minimum_flag_file
median_rms_file = np.where(
np.equal(minimum_grow_file,
0), median_file_weighted - rms_file * combine_nsigma1,
median_file_weighted - rms_file * combine_nsigma2)
del rms_file, minimum_grow_file
# Finally decide whether to use the minimim or the median (in counts/s),
# based on whether the median is more than 3 sigma above the minimum.
combined_array = np.where(np.less(minimum_file_weighted, median_rms_file),
minimum_file, median_file)
# Set fill regions to a pixel value of 0.
combined_array[all_bad_idx, all_bad_idy] = 0
return combined_array
import os, sys, pdb, scipy, glob
from pylab import *
from astropy.io import fits
from stsci.convolve import boxcar
verbose = True
if __name__ == '__main__':
filename = sys.argv[1] ## a bias-subtracted, combined flat_field is best
hdu = fits.open(filename)
data = hdu[0].data
if verbose: print('Replacing data region with zeros and ones...')
tmp = data > 25000.
data = tmp.astype(int)
if verbose: print('Boxcar smoothing...')
data = boxcar(data.astype(float), (25, 25))
if verbose: print('Flipping array... ')
tmp = data == 0.
data = tmp.astype(int)
if verbose: print('Boxcar smoothing again...')
data = boxcar(data.astype(float), (51, 51))
if verbose: print('Replacing positive values to create mask...')
tmp = data > 0.
data = tmp.astype(int)
if verbose: print('Writing mask image flagimg.fits...')
out = fits.PrimaryHDU(data)
out.scale('int16')
out.writeto('flagimg.fits')
if verbose: print('Done.')
def __init__(self,
imageList, # list of input data to be combined.
weightImageList, # list of input data weight images to be combined.
readnoiseList, # list of readnoise values to use for the input images.
exposureTimeList, # list of exposure times to use for the input images.
backgroundValueList, # list of image background values to use for the input images
weightMaskList= None, # list of imput data weight masks to use for pixel rejection.
combine_grow = 1, # Radius (pixels) for neighbor rejection
combine_nsigma1 = 4, # Significance for accepting minimum instead of median
combine_nsigma2 = 3, # Significance for accepting minimum instead of median
fillval = False # Turn on use of imedian/imean
):
warnings.warn("The 'minmed' class is deprecated and may be removed"
" in a future version. Use 'min_med()' instead.",
DeprecationWarning)
# Define input variables
self._imageList = imageList
self._weightImageList = weightImageList
self._weightMaskList = weightMaskList
self._exposureTimeList = exposureTimeList
self._readnoiseList = readnoiseList
self._backgroundValueList = backgroundValueList
self._numberOfImages = len( self._imageList)
self._combine_grow = combine_grow
self._combine_nsigma1 = combine_nsigma1
self._combine_nsigma2 = combine_nsigma2
if fillval:
combtype_mean = 'imean'
combtype_median = 'imedian'
else:
combtype_mean = 'mean'
combtype_median = 'median'
# Create a different median image based upon the number of images in the input list.
median_file = np.zeros(self._imageList[0].shape,dtype=self._imageList[0].dtype)
if (self._numberOfImages == 2):
tmp = numCombine(self._imageList,numarrayMaskList=self._weightMaskList,
combinationType=combtype_mean,nlow=0,nhigh=0,
nkeep=1,upper=None,lower=None)
median_file = tmp.combArrObj
else:
# The value of NHIGH=1 will cause problems when there is only 1 valid
# unmasked input image for that pixel due to a difference in behavior
# between 'numcombine' and 'iraf.imcombine'.
# This value may need to be adjusted on the fly based on the number of
# inputs and the number of masked values/pixel.
#
tmp = numCombine(self._imageList,numarrayMaskList=self._weightMaskList,
combinationType=combtype_median,nlow=0,nhigh=1,
nkeep=1,upper=None,lower=None)
median_file = tmp.combArrObj
if self._weightMaskList in [None,[]]:
self._weightMaskList = [np.zeros(self._imageList[0].shape,dtype=self._imageList[0].dtype)]*len(self._imageList)
# The following section of code will address the problem caused by having
# a value of nhigh = 1. This will behave in a way similar to the way the
# IRAF task IMCOMBINE behaves. In order to accomplish this, the following
# procedure will be followed:
# 1) The input masks will be summed.
# 2) The science data will be summed.
# 3) In the locations of the summed mask where the sum is 1 less than the
# the total number of images, the value of that location in the summed
# sceince image will be used to replace the existing value in the
# existing median_file.
#
# This procuedure is being used to prevent too much data from being thrown
# out of the image. Take for example the case of 3 input images. In two
# of the images the pixel locations have been masked out. Now, if nhigh
# is applied there will be no value to use for that position. However,
# if this new procedure is used that value in the resulting images will
# be the value that was rejected by the nhigh rejection step.
#
# We need to make certain that "bad" pixels in the sci data are set to 0. That way,
# when the sci images are summed, the value of the sum will only come from the "good"
# pixels.
tmpList = []
for image in range(len(self._imageList)):
tmp =np.where(self._weightMaskList[image] == 1, 0, self._imageList[image])
tmpList.append(tmp)
# Sum the mask files
maskSum = self._sumImages(self._weightMaskList)
# Sum the science images
sciSum = self._sumImages(tmpList)
del(tmpList)
# Use the summed sci image values in locations where the maskSum indicates
# that there is only 1 good pixel to use. The value will be used in the
# median_file image
median_file = np.where(maskSum == self._numberOfImages-1,sciSum,median_file)
if self._weightMaskList in [None,[]]:
self._weightMaskList = [np.zeros(self._imageList[0].shape,dtype=self._imageList[0].dtype)]*len(self._imageList)
# Sum the weightMaskList elements
maskSum = self._sumImages(self._weightMaskList)
# Create the minimum image from the stack of input images.
# Find the maximum pixel value for the image stack.
maxValue = -1e+9
for image in self._imageList:
newMax = image.max()
if (newMax > maxValue):
maxValue = newMax
# For each image, set pixels masked as "bad" to the "super-maximum" value.
for image in range(len(self._imageList)):
self._imageList[image] = np.where(self._weightMaskList[image] == 1,maxValue+1,self._imageList[image])
# Call numcombine throwing out the highest N - 1 pixels.
tmp = numCombine(self._imageList,numarrayMaskList=None,
combinationType=combtype_median,nlow=0,nhigh=self._numberOfImages-1,
nkeep=1,upper=None,lower=None)
minimum_file = tmp.combArrObj
# Reset any pixl at maxValue + 1 to 0.
minimum_file = np.where(maskSum == self._numberOfImages, 0, minimum_file)
# Scale the weight images by the background values and add them to the bk
backgroundFileList = []
for image in range(len(self._weightImageList)):
tmp = self._weightImageList[image] * (self._backgroundValueList[image]/(self._exposureTimeList[image]))
backgroundFileList.append(tmp)
# Create an image of the total effective background (in DN) per pixel:
# (which is the sum of all the background-scaled weight files)
#
bkgd_file = self._sumImages(backgroundFileList)
del(backgroundFileList)
#
# Scale the weight mask images by the square of the readnoise values
#
readnoiseFileList = []
for image in range(len(self._weightMaskList)):
tmp = (np.logical_not(self._weightMaskList[image]) *
(self._readnoiseList[image] * self._readnoiseList[image]))
readnoiseFileList.append(tmp)
# Create an image of the total readnoise**2 per pixel:
# (which is the sum of all the input readnoise values)
#
readnoise_file = self._sumImages(readnoiseFileList)
del(readnoiseFileList)
# Create an image of the total effective exposure time per pixel:
# (which is simply the sum of all the drizzle output weight files)
#
weight_file = self._sumImages(self._weightImageList)
# Scale up both the median and minimum arrays by the total effective exposure time
# per pixel.
#
minimum_file_weighted = minimum_file * weight_file
median_file_weighted = median_file * weight_file
del(weight_file)
# Calculate the 1-sigma r.m.s.:
# variance = median_electrons + bkgd_electrons + readnoise**2
# rms = sqrt(variance)
# This image has units of electrons.
#
# make this the abs value so that negative numbers dont throw an exception?
rms_file2 = np.fmax(
median_file_weighted + bkgd_file + readnoise_file,
np.zeros_like(median_file_weighted)
)
rms_file = np.sqrt(rms_file2)
del bkgd_file
del readnoise_file
# For the median array, calculate the n-sigma lower threshold to the array
# and incorporate that into the pixel values.
#
median_rms_file = median_file_weighted - (rms_file * self._combine_nsigma1)
if self._combine_grow != 0:
#
# Do a more sophisticated rejection: For all cases where the minimum pixel will
# be accepted instead of the median, set a lower threshold for that pixel and the
# ones around it (ie become less conservative in rejecting the median). This is
# because in cases of triple-incidence cosmic rays, quite often the low-lying
# outliers of the CRs can influence the median for the initial relatively high
# value of sigma, so a lower threshold must be used to mnake sure that the minimum
# is selected.
#
# This is done as follows:
# 1) make an image which is zero everywhere except where the minimum will be accepted
# 2) box-car smooth this image, to make these regions grow.
# 3) In the file "median_rms_file_electrons", replace these pixels
# by median - combine_nsigma2 * rms
#
# Then use this image in the final replacement, in the same way as for the
# case where this option is not selected.
minimum_flag_file = np.where(np.less(minimum_file_weighted,median_rms_file), 1, 0)
# The box size value must be an integer. This is not a problem since __combine_grow should always
# be an integer type. The combine_grow column in the MDRIZTAB should also be an integer type.
boxsize = int(2 * self._combine_grow + 1)
boxshape = (boxsize, boxsize)
minimum_grow_file = np.zeros(self._imageList[0].shape,dtype=self._imageList[0].dtype)
# If the boxcar convolution has failed it is potentially for two reasons:
# 1) The kernel size for the boxcar is bigger than the actual image.
# 2) The grow parameter was specified with a value < 0. This would result
# in an illegal boxshape kernel. The dimensions of the kernel box *MUST*
# be integer and greater than zero.
#
# If the boxcar convolution has failed, try to give a meaningfull explanation
# as to why based upon the conditionals described above.
if (boxsize <= 0):
errormsg1 = "############################################################\n"
errormsg1 += "# The boxcar convolution in minmed has failed. The 'grow' #\n"
errormsg1 += "# parameter must be greater than or equal to zero. You #\n"
errormsg1 += "# specified an input value for the 'grow' parameter of: #\n"
errormsg1 += " combine_grow: " + str(self._combine_grow)+'\n'
errormsg1 += "############################################################\n"
raise ValueError(errormsg1)
if (boxsize > self._imageList[0].shape[0]):
errormsg2 = "############################################################\n"
errormsg2 += "# The boxcar convolution in minmed has failed. The 'grow' #\n"
errormsg2 += "# parameter specified has resulted in a boxcar kernel that #\n"
errormsg2 += "# has dimensions larger than the actual image. You #\n"
errormsg2 += "# specified an input value for the 'grow' parameter of: #\n"
errormsg2 += " combine_grow: " +str(self._combine_grow)+'\n'
errormsg2 += "############################################################\n"
print(self._imageList[0].shape)
raise ValueError(errormsg2)
# Attempt the boxcar convolution using the boxshape based upon the user input value of "grow"
boxcar(minimum_flag_file, boxshape, output=minimum_grow_file,
mode='constant', cval=0)
del(minimum_flag_file)
temp1 = (median_file_weighted - (rms_file * self._combine_nsigma1))
temp2 = (median_file_weighted - (rms_file * self._combine_nsigma2))
median_rms2_file = np.where(np.equal(minimum_grow_file, 0), temp1, temp2)
del(temp1)
del(temp2)
del(rms_file)
del(minimum_grow_file)
# Finally decide whether to use the minimim or the median (in counts/s),
# based on whether the median is more than 3 sigma above the minimum.
#
self.combArrObj = np.where(
np.less(minimum_file_weighted, median_rms2_file),
minimum_file,
median_file
)
else:
# Finally decide whether to use the minimim or the median (in counts/s),
# based on whether the median is more than 3 sigma above the minimum.
#
self.combArrObj = np.where(
np.less(minimum_file_weighted, median_rms_file),
minimum_file,
median_file
)
# Set fill regions to a pixel value of 0.
self.combArrObj = np.where(maskSum == self._numberOfImages, 0, self.combArrObj)
def min_med(images, weight_images, readnoise_list, exptime_list,
background_values, weight_masks=None, combine_grow=1,
combine_nsigma1=4, combine_nsigma2=3, fillval=False):
""" Create a median array, rejecting the highest pixel and
computing the lowest valid pixel after mask application.
.. note::
In this version of the mimmed algorithm we assume that the units of
all input data is electons.
Parameters
----------
images : list of numpy.ndarray
List of input data to be combined.
weight_images : list of numpy.ndarray
List of input data weight images to be combined.
readnoise_list : list
List of readnoise values to use for the input images.
exptime_list : list
List of exposure times to use for the input images.
background_values : list
List of image background values to use for the input images.
weight_masks : list of numpy.ndarray, None
List of imput data weight masks to use for pixel rejection.
(Default: `None`)
combine_grow : int
Radius (pixels) for neighbor rejection. (Default: 1)
combine_nsigma1 : float
Significance for accepting minimum instead of median. (Default: 4)
combine_nsigma2 : float
Significance for accepting minimum instead of median. (Default: 3)
fillval : bool
Turn on use of imedian/imean. (Default: `False`)
Returns
-------
combined_array : numpy.ndarray
Combined array.
"""
# In this case we want to calculate two things:
# 1) the median array, rejecting the highest pixel (thus running
# imcombine with nlow=0, nhigh=1, nkeep=1, using the masks)
# 2) the lowest valid pixel after applying the masks (thus running
# imcombine with nlow=0, nhigh=3, nkeep=1, using the masks)
#
# We also calculate the sum of the weight files (to produce the total
# effective exposure time for each pixel).
#
# The total effective background in the final image is calculated as
# follows:
# - convert background for each input image to counts/s
# (divide by exptime)
# - multiply this value by the weight image, to obtain the effective
# background counts (in DN) for each pixel, for each image
# - Add these images together, to obtain the total effective background
# for the combined image.
#
# Once we've made these two files, then calculate the SNR based on the
# median-pixel image, and compare with the minimum.
nimages = len(images)
combtype_median = 'imedian' if fillval else 'median'
images = np.asarray(images)
weight_images = np.asarray(weight_images)
if weight_masks == [] or weight_masks is None:
weight_masks = None
mask_sum = np.zeros(images.shape[1:], dtype=np.int16)
all_bad_idx = np.array([], dtype=np.int)
all_bad_idy = np.array([], dtype=np.int)
else:
weight_masks = np.asarray(weight_masks, dtype=np.bool)
mask_sum = np.sum(weight_masks, axis=0, dtype=np.int16)
all_bad_idx, all_bad_idy = np.where(mask_sum == nimages)
# Create a different median image based upon the number of images in the
# input list.
if nimages == 2:
median_file = num_combine(
images,
masks=weight_masks,
combination_type='imean' if fillval else 'mean',
nlow=0, nhigh=0, lower=None, upper=None
)
else:
# The value of NHIGH=1 will cause problems when there is only 1 valid
# unmasked input image for that pixel due to a difference in behavior
# between 'num_combine' and 'iraf.imcombine'.
# This value may need to be adjusted on the fly based on the number of
# inputs and the number of masked values/pixel.
#
median_file = num_combine(
images,
masks=weight_masks,
combination_type=combtype_median,
nlow=0, nhigh=1, lower=None, upper=None
)
# The following section of code will address the problem caused by
# having a value of nhigh = 1. This will behave in a way similar to
# the way the IRAF task IMCOMBINE behaves. In order to accomplish
# this, the following procedure will be followed:
# 1) The input masks will be summed.
# 2) The science data will be summed.
# 3) In the locations of the summed mask where the sum is 1 less than
# the total number of images, the value of that location in the
# summed science image will be used to replace the existing value
# in the existing median_file.
#
# This procedure is being used to prevent too much data from being
# thrown out of the image. Take for example the case of 3 input images.
# In two of the images the pixel locations have been masked out.
# Now, if nhigh is applied there will be no value to use for that
# position. However, if this new procedure is used that value in
# the resulting images will be the value that was rejected by the
# nhigh rejection step.
# We need to make certain that "bad" pixels in the sci data are set to
# 0. That way, when the sci images are summed, the value of the sum
# will only come from the "good" pixels.
if weight_masks is None:
sci_sum = np.sum(images, axis=0)
if nimages == 1:
median_file = sci_sum
else:
sci_sum = np.sum(images * np.logical_not(weight_masks), axis=0)
# Use the summed sci image values in locations where the mask_sum
# indicates that there is only 1 good pixel to use. The value will
# be used in the median_file image
idx = np.where(mask_sum == (nimages - 1))
median_file[idx] = sci_sum[idx]
# Create the minimum image from the stack of input images.
if weight_masks is not None:
# make a copy of images to avoid side-effect of modifying input
# argument:
images = images.copy()
images[weight_masks] = np.nan
images[:, all_bad_idx, all_bad_idy] = 0
minimum_file = np.nanmin(images, axis=0)
else:
minimum_file = np.amin(images, axis=0)
# Scale the weight images by the background values and add them to the bk
# Create an image of the total effective background (in DN) per pixel:
# (which is the sum of all the background-scaled weight files)
s = np.asarray([bv / et for bv, et in
zip(background_values, exptime_list)])
bkgd_file = np.sum(weight_images * s[:, None, None], axis=0)
# Scale the weight mask images by the square of the readnoise values.
# Create an image of the total readnoise**2 per pixel
# (which is the sum of all the input readnoise values).
if weight_masks is None:
rdn2 = sum((r**2 for r in readnoise_list))
readnoise_file = rdn2 * np.ones_like(images[0])
else:
readnoise_file = np.sum(
np.logical_not(weight_masks) *
(np.asarray(readnoise_list)**2)[:, None, None],
axis=0
)
# Create an image of the total effective exposure time per pixel:
# (which is simply the sum of all the drizzle output weight files)
weight_file = np.sum(weight_images, axis=0)
# Scale up both the median and minimum arrays by the total effective
# exposure time per pixel.
minimum_file_weighted = minimum_file * weight_file
median_file_weighted = median_file * weight_file
del weight_file
# Calculate the 1-sigma r.m.s.:
# variance = median_electrons + bkgd_electrons + readnoise**2
# rms = sqrt(variance)
# This image has units of electrons.
#
# make this the abs value so that negative numbers dont throw an exception?
rms_file2 = np.fmax(
median_file_weighted + bkgd_file + readnoise_file,
np.zeros_like(median_file_weighted)
)
rms_file = np.sqrt(rms_file2)
del bkgd_file, readnoise_file
# For the median array, calculate the n-sigma lower threshold to the array
# and incorporate that into the pixel values.
median_rms_file = median_file_weighted - rms_file * combine_nsigma1
if combine_grow != 0:
# Do a more sophisticated rejection: For all cases where the minimum
# pixel will be accepted instead of the median, set a lower threshold
# for that pixel and the ones around it (ie become less conservative
# in rejecting the median). This is because in cases of
# triple-incidence cosmic rays, quite often the low-lying outliers
# of the CRs can influence the median for the initial relatively high
# value of sigma, so a lower threshold must be used to mnake sure that
# the minimum is selected.
#
# This is done as follows:
# 1) make an image which is zero everywhere except where the minimum
# will be accepted
# 2) box-car smooth this image, to make these regions grow.
# 3) In the file "median_rms_file_electrons", replace these pixels
# by median - combine_nsigma2 * rms
#
# Then use this image in the final replacement, in the same way as for
# the case where this option is not selected.
minimum_flag_file = np.less(minimum_file_weighted,
median_rms_file).astype(np.float64)
# The box size value must be an integer. This is not a problem since
# __combine_grow should always be an integer type. The combine_grow
# column in the MDRIZTAB should also be an integer type.
boxsize = int(2 * combine_grow + 1)
boxshape = (boxsize, boxsize)
minimum_grow_file = np.zeros_like(images[0])
# If the boxcar convolution has failed it is potentially for
# two reasons:
# 1) The kernel size for the boxcar is bigger than the actual image.
# 2) The grow parameter was specified with a value < 0. This would
# result in an illegal boxshape kernel. The dimensions of the
# kernel box *MUST* be integer and greater than zero.
#
# If the boxcar convolution has failed, try to give a meaningfull
# explanation as to why based upon the conditionals described above.
if boxsize <= 0:
errormsg1 = "############################################################\n"
errormsg1 += "# The boxcar convolution in minmed has failed. The 'grow' #\n"
errormsg1 += "# parameter must be greater than or equal to zero. You #\n"
errormsg1 += "# specified an input value for the 'grow' parameter of: #\n"
errormsg1 += " combine_grow: " + str(combine_grow)+'\n'
errormsg1 += "############################################################\n"
raise ValueError(errormsg1)
if boxsize > images.shape[1]:
errormsg2 = "############################################################\n"
errormsg2 += "# The boxcar convolution in minmed has failed. The 'grow' #\n"
errormsg2 += "# parameter specified has resulted in a boxcar kernel that #\n"
errormsg2 += "# has dimensions larger than the actual image. You #\n"
errormsg2 += "# specified an input value for the 'grow' parameter of: #\n"
errormsg2 += " combine_grow: " + str(combine_grow) + '\n'
errormsg2 += "############################################################\n"
print(images.shape[1:])
raise ValueError(errormsg2)
# Attempt the boxcar convolution using the boxshape based upon the user
# input value of "grow"
boxcar(minimum_flag_file, boxshape, output=minimum_grow_file,
mode='constant', cval=0)
del minimum_flag_file
median_rms_file = np.where(
np.equal(minimum_grow_file, 0),
median_file_weighted - rms_file * combine_nsigma1,
median_file_weighted - rms_file * combine_nsigma2
)
del rms_file, minimum_grow_file
# Finally decide whether to use the minimim or the median (in counts/s),
# based on whether the median is more than 3 sigma above the minimum.
combined_array = np.where(
np.less(minimum_file_weighted, median_rms_file),
minimum_file,
median_file
)
# Set fill regions to a pixel value of 0.
combined_array[all_bad_idx, all_bad_idy] = 0
return combined_array
def test_boxcar4():
result = convolve.boxcar(np.array([10, 0, 0, 0, 0, 0, 1000]), (3,), mode="constant").astype(np.longlong)
test = np.array([ 3, 3, 0, 0, 0, 333, 333], dtype=np.int64)
assert_equal(result.all(),test.all())
def test_boxcar3():
result = convolve.boxcar(np.array([10, 0, 0, 0, 0, 0, 1000]), (3,), mode="reflect").astype(np.longlong)
test = np.array([ 6, 3, 0, 0, 0, 333, 666], dtype=np.int64)
assert_equal(result.all(),test.all())
def make_plots(fuv, ccd1, ccd2, ccd3, ccd4, reject_cr_flag, norm_spec_flag, ax1_ylim, ax2_ylim, title, base_dir, bin_flag = None):
check_existance_of_files(ccd1, ccd2, ccd3, ccd4, fuv)
if norm_spec_flag == True:
tbdata5 = pyfits.getdata(fuv, 1)
wl5 = tbdata5['wavelength'].ravel()
net5 = tbdata5['flux'].ravel()
#dq = tbdata5['dq'].ravel()
#wl5 = wl5[dq&512 != 512]
#net5 = net5[dq&512 != 512]
#cr_reject
if reject_cr_flag:
#Open data files
tbdata1 = pyfits.getdata(ccd1, 1)
tbdata2 = pyfits.getdata(ccd2, 1)
tbdata3 = pyfits.getdata(ccd3, 1)
tbdata4 = pyfits.getdata(ccd4, 1)
wl1 = tbdata1['wavelength'].ravel()
net1 = tbdata1['net'].ravel()
dq = tbdata1['dq'].ravel()
wl1 = wl1[dq&512 != 512]
net1 = net1[dq&512 != 512]
wl2 = tbdata2['wavelength'].ravel()
net2 = tbdata2['net'].ravel()
dq = tbdata2['dq'].ravel()
wl2 = wl2[dq&512 != 512]
net2 = net2[dq&512 != 512]
wl3 = tbdata3['wavelength'].ravel()
net3 = tbdata3['net'].ravel()
dq = tbdata3['dq'].ravel()
wl3 = wl3[dq&512 != 512]
net3 = net3[dq&512 != 512]
wl4 = tbdata4['wavelength'].ravel()
net4 = tbdata4['net'].ravel()
dq = tbdata4['dq'].ravel()
wl4 = wl4[dq&512 != 512]
net4 = net4[dq&512 != 512]
else:
tbdata1 = pyfits.getdata(ccd1.replace('.fits', '_crj.fits'), 1)
wl1 = tbdata1['wavelength'].ravel()
net1 = tbdata1['net'].ravel()
tbdata2 = pyfits.getdata(ccd2.replace('.fits', '_crj.fits'), 1)
wl2 = tbdata2['wavelength'].ravel()
net2 = tbdata2['net'].ravel()
tbdata3 = pyfits.getdata(ccd3.replace('.fits', '_crj.fits'), 1)
wl3 = tbdata3['wavelength'].ravel()
net3 = tbdata3['net'].ravel()
tbdata4 = pyfits.getdata(ccd4.replace('.fits', '_crj.fits'), 1)
wl4 = tbdata4['wavelength'].ravel()
net4 = tbdata4['net'].ravel()
net1 = normalize_spec.manually_remove_CR(wl1, net1, flt = os.path.join(base_dir, 'combined_dithered_images/%s_%i_combined_img.fits' %(pyfits.getval(ccd1, 'targname', 0)[5:], pyfits.getval(ccd1, 'cenwave', 0))))
write_fits_file(ccd1.replace('.fits', '_crj.fits'), wl1, net1, pyfits.getheader(ccd1, 0), spec_name = 'net')
norm_spec2 = normalize_spec.manually_remove_CR(wl2, net2, flt = os.path.join(base_dir, 'combined_dithered_images/%s_%i_combined_img.fits' %(pyfits.getval(ccd2, 'targname', 0)[5:], pyfits.getval(ccd2, 'cenwave', 0))))
write_fits_file(ccd2.replace('.fits', '_crj.fits'), wl2, net2, pyfits.getheader(ccd2, 0), spec_name = 'net')
norm_spec3 = normalize_spec.manually_remove_CR(wl3, net3, flt = os.path.join(base_dir, 'combined_dithered_images/%s_%i_combined_img.fits' %(pyfits.getval(ccd3, 'targname', 0)[5:], pyfits.getval(ccd3, 'cenwave', 0))))
write_fits_file(ccd3.replace('.fits', '_crj.fits'), wl3, net3, pyfits.getheader(ccd3, 0), spec_name = 'net')
norm_spec4 = normalize_spec.manually_remove_CR(wl4, net4, flt = os.path.join(base_dir, 'combined_dithered_images/%s_%i_combined_img.fits' %(pyfits.getval(ccd4, 'targname', 0)[5:], pyfits.getval(ccd4, 'cenwave', 0))))
write_fits_file(ccd4.replace('.fits', '_crj.fits'), wl4, net4, pyfits.getheader(ccd4, 0), spec_name = 'net')
if bin_flag:
wl1, net1 = bin_data(wl1, net1, bin_size = 3)
wl2, net2 = bin_data(wl2, net2, bin_size = 3)
wl3, net3 = bin_data(wl3, net3, bin_size = 3)
wl4, net4 = bin_data(wl4, net4, bin_size = 3)
#normalize spectra
wl1, norm_spec1, cont1 = normalize_spec.normalize_spectrum(wl1, net1)
write_fits_file(ccd1.replace('.fits', '_norm.fits'), wl1, norm_spec1, pyfits.getheader(ccd1, 0), spec_name = 'net')
wl2, norm_spec2, cont2 = normalize_spec.normalize_spectrum(wl2, net2)
write_fits_file(ccd2.replace('.fits', '_norm.fits'), wl2, norm_spec2, pyfits.getheader(ccd2, 0), spec_name = 'net')
wl3, norm_spec3, cont3 = normalize_spec.normalize_spectrum(wl3, net3)
write_fits_file(ccd3.replace('.fits', '_norm.fits'), wl3, norm_spec3, pyfits.getheader(ccd3, 0), spec_name = 'net')
wl4, norm_spec4, cont4 = normalize_spec.normalize_spectrum(wl4, net4)
write_fits_file(ccd4.replace('.fits', '_norm.fits'), wl4, norm_spec4, pyfits.getheader(ccd4, 0), spec_name = 'net')
wl5, norm_spec5, cont5 = normalize_spec.normalize_spectrum(wl5, net5, kernel_size = 401)
write_fits_file(fuv .replace('.fits', '_norm.fits'), wl5, norm_spec5, pyfits.getheader(fuv, 0))
else:
tbdata1 = pyfits.getdata(ccd1.replace('.fits', '_norm.fits'), 1)
wl1 = tbdata1['wavelength']
norm_spec1 = tbdata1['net']
tbdata2 = pyfits.getdata(ccd2.replace('.fits', '_norm.fits'), 1)
wl2 = tbdata2['wavelength']
norm_spec2 = tbdata2['net']
tbdata3 = pyfits.getdata(ccd3.replace('.fits', '_norm.fits'), 1)
wl3 = tbdata3['wavelength']
norm_spec3 = tbdata3['net']
tbdata4 = pyfits.getdata(ccd4.replace('.fits', '_norm.fits'), 1)
wl4 = tbdata4['wavelength']
norm_spec4 = tbdata4['net']
tbdata5 = pyfits.getdata(fuv.replace('.fits', '_norm.fits'), 1)
wl5 = tbdata5['wavelength']
norm_spec5 = tbdata5['flux']
#create plotting objects
fig = pyplot.figure(6, figsize = [26, 15])
fig.clf()
ax1 = fig.add_subplot(3, 1, 1)
ax2 = fig.add_subplot(3, 1, 2)
ax3 = fig.add_subplot(3, 1, 3)
ax2.plot(wl1, norm_spec1, 'k')
ax2.plot(wl2, norm_spec2, 'b')
ax2.plot(wl3, norm_spec3, 'k')
ax2.plot(wl4, norm_spec4, 'b')
ax3.plot(wl1, boxcar(norm_spec1, (3,)), 'k')
ax3.plot(wl2, boxcar(norm_spec2, (3,)), 'b')
ax3.plot(wl3, boxcar(norm_spec3, (3,)), 'k')
ax3.plot(wl4, boxcar(norm_spec4, (3,)), 'b')
ax1.plot(wl5, norm_spec5, 'k')
ax2.set_xlim(3900, 4750)
ax3.set_xlim(3900, 4750)
ax1.set_xlim(1150, 1700)
ax1.set_ylim(ax1_ylim[0], ax1_ylim[1])
ax2.set_ylim(ax2_ylim[0], ax2_ylim[1])
ax3.set_ylim(ax2_ylim[0], ax2_ylim[1])
ax1_xticks = np.arange(1150, 1750, 50)
ax2_xticks = np.arange(3900, 4750, 50)
ax1.set_xticks(ax1_xticks)
ax2.set_xticks(ax2_xticks)
ax3.set_xticks(ax2_xticks)
ax1.grid(b = 'on', color = 'gray', ls = ':')
ax2.grid(b = 'on', color = 'gray', ls = ':')
ax3.grid(b = 'on', color = 'gray', ls = ':')
#Label Plots
ax2.set_title('Optical Spectrum %s' %(title))
ax1.set_title('FUV Spectrum %s' %(title))
ax3.set_title('Optical Spectrum %s R~4000' %(title))
ax3.set_xlabel('Wavelength')
ax2.set_ylabel('Relative Intensity')
pyplot.draw()
return ax1, ax2, ax3, fig
im.writeto(image[0:-5] + "_gauss" + str(kernel) + ".fits", clobber=True)
im.close()
# gaussian kernels (size in pixels)
kernelsbox = [1,2,3,4,5,6,7,8,9,10,15,20]
# image selection
image = "cig/cig96_def_crop.fits"
# boxcar smoothing
im = fits.open(image)
for kernel in kernelsbox:
boxshape = [kernel,kernel]
im[0].data = convolve.boxcar(im[0].data, boxshape, output=None, mode='wrap', cval=0.0)
# smoothed image
im.writeto(image[0:-5] + "_boxcar" + str(kernel) + ".fits", clobber=True)
im.close()
# In[28]:
# VPeris images gaussian filter
# gaussian kernels (size in pixels)
kernels = [1,2,3,4,5,10,15,20,25,30]
# image selection
image = "v_peris/cig96_suma_peris.fit"
def make_plots(fuv,
ccd1,
ccd2,
ccd3,
ccd4,
reject_cr_flag,
norm_spec_flag,
ax1_ylim,
ax2_ylim,
title,
base_dir,
bin_flag=None):
check_existance_of_files(ccd1, ccd2, ccd3, ccd4, fuv)
if norm_spec_flag == True:
tbdata5 = pyfits.getdata(fuv, 1)
wl5 = tbdata5['wavelength'].ravel()
net5 = tbdata5['flux'].ravel()
#dq = tbdata5['dq'].ravel()
#wl5 = wl5[dq&512 != 512]
#net5 = net5[dq&512 != 512]
#cr_reject
if reject_cr_flag:
#Open data files
tbdata1 = pyfits.getdata(ccd1, 1)
tbdata2 = pyfits.getdata(ccd2, 1)
tbdata3 = pyfits.getdata(ccd3, 1)
tbdata4 = pyfits.getdata(ccd4, 1)
wl1 = tbdata1['wavelength'].ravel()
net1 = tbdata1['net'].ravel()
dq = tbdata1['dq'].ravel()
wl1 = wl1[dq & 512 != 512]
net1 = net1[dq & 512 != 512]
wl2 = tbdata2['wavelength'].ravel()
net2 = tbdata2['net'].ravel()
dq = tbdata2['dq'].ravel()
wl2 = wl2[dq & 512 != 512]
net2 = net2[dq & 512 != 512]
wl3 = tbdata3['wavelength'].ravel()
net3 = tbdata3['net'].ravel()
dq = tbdata3['dq'].ravel()
wl3 = wl3[dq & 512 != 512]
net3 = net3[dq & 512 != 512]
wl4 = tbdata4['wavelength'].ravel()
net4 = tbdata4['net'].ravel()
dq = tbdata4['dq'].ravel()
wl4 = wl4[dq & 512 != 512]
net4 = net4[dq & 512 != 512]
else:
tbdata1 = pyfits.getdata(ccd1.replace('.fits', '_crj.fits'), 1)
wl1 = tbdata1['wavelength'].ravel()
net1 = tbdata1['net'].ravel()
tbdata2 = pyfits.getdata(ccd2.replace('.fits', '_crj.fits'), 1)
wl2 = tbdata2['wavelength'].ravel()
net2 = tbdata2['net'].ravel()
tbdata3 = pyfits.getdata(ccd3.replace('.fits', '_crj.fits'), 1)
wl3 = tbdata3['wavelength'].ravel()
net3 = tbdata3['net'].ravel()
tbdata4 = pyfits.getdata(ccd4.replace('.fits', '_crj.fits'), 1)
wl4 = tbdata4['wavelength'].ravel()
net4 = tbdata4['net'].ravel()
net1 = normalize_spec.manually_remove_CR(
wl1,
net1,
flt=os.path.join(
base_dir, 'combined_dithered_images/%s_%i_combined_img.fits' %
(pyfits.getval(ccd1, 'targname',
0)[5:], pyfits.getval(ccd1, 'cenwave', 0))))
write_fits_file(ccd1.replace('.fits', '_crj.fits'),
wl1,
net1,
pyfits.getheader(ccd1, 0),
spec_name='net')
norm_spec2 = normalize_spec.manually_remove_CR(
wl2,
net2,
flt=os.path.join(
base_dir, 'combined_dithered_images/%s_%i_combined_img.fits' %
(pyfits.getval(ccd2, 'targname',
0)[5:], pyfits.getval(ccd2, 'cenwave', 0))))
write_fits_file(ccd2.replace('.fits', '_crj.fits'),
wl2,
net2,
pyfits.getheader(ccd2, 0),
spec_name='net')
norm_spec3 = normalize_spec.manually_remove_CR(
wl3,
net3,
flt=os.path.join(
base_dir, 'combined_dithered_images/%s_%i_combined_img.fits' %
(pyfits.getval(ccd3, 'targname',
0)[5:], pyfits.getval(ccd3, 'cenwave', 0))))
write_fits_file(ccd3.replace('.fits', '_crj.fits'),
wl3,
net3,
pyfits.getheader(ccd3, 0),
spec_name='net')
norm_spec4 = normalize_spec.manually_remove_CR(
wl4,
net4,
flt=os.path.join(
base_dir, 'combined_dithered_images/%s_%i_combined_img.fits' %
(pyfits.getval(ccd4, 'targname',
0)[5:], pyfits.getval(ccd4, 'cenwave', 0))))
write_fits_file(ccd4.replace('.fits', '_crj.fits'),
wl4,
net4,
pyfits.getheader(ccd4, 0),
spec_name='net')
if bin_flag:
wl1, net1 = bin_data(wl1, net1, bin_size=3)
wl2, net2 = bin_data(wl2, net2, bin_size=3)
wl3, net3 = bin_data(wl3, net3, bin_size=3)
wl4, net4 = bin_data(wl4, net4, bin_size=3)
#normalize spectra
wl1, norm_spec1, cont1 = normalize_spec.normalize_spectrum(wl1, net1)
write_fits_file(ccd1.replace('.fits', '_norm.fits'),
wl1,
norm_spec1,
pyfits.getheader(ccd1, 0),
spec_name='net')
wl2, norm_spec2, cont2 = normalize_spec.normalize_spectrum(wl2, net2)
write_fits_file(ccd2.replace('.fits', '_norm.fits'),
wl2,
norm_spec2,
pyfits.getheader(ccd2, 0),
spec_name='net')
wl3, norm_spec3, cont3 = normalize_spec.normalize_spectrum(wl3, net3)
write_fits_file(ccd3.replace('.fits', '_norm.fits'),
wl3,
norm_spec3,
pyfits.getheader(ccd3, 0),
spec_name='net')
wl4, norm_spec4, cont4 = normalize_spec.normalize_spectrum(wl4, net4)
write_fits_file(ccd4.replace('.fits', '_norm.fits'),
wl4,
norm_spec4,
pyfits.getheader(ccd4, 0),
spec_name='net')
wl5, norm_spec5, cont5 = normalize_spec.normalize_spectrum(
wl5, net5, kernel_size=401)
write_fits_file(fuv.replace('.fits', '_norm.fits'), wl5, norm_spec5,
pyfits.getheader(fuv, 0))
else:
tbdata1 = pyfits.getdata(ccd1.replace('.fits', '_norm.fits'), 1)
wl1 = tbdata1['wavelength']
norm_spec1 = tbdata1['net']
tbdata2 = pyfits.getdata(ccd2.replace('.fits', '_norm.fits'), 1)
wl2 = tbdata2['wavelength']
norm_spec2 = tbdata2['net']
tbdata3 = pyfits.getdata(ccd3.replace('.fits', '_norm.fits'), 1)
wl3 = tbdata3['wavelength']
norm_spec3 = tbdata3['net']
tbdata4 = pyfits.getdata(ccd4.replace('.fits', '_norm.fits'), 1)
wl4 = tbdata4['wavelength']
norm_spec4 = tbdata4['net']
tbdata5 = pyfits.getdata(fuv.replace('.fits', '_norm.fits'), 1)
wl5 = tbdata5['wavelength']
norm_spec5 = tbdata5['flux']
#create plotting objects
fig = pyplot.figure(6, figsize=[26, 15])
fig.clf()
ax1 = fig.add_subplot(3, 1, 1)
ax2 = fig.add_subplot(3, 1, 2)
ax3 = fig.add_subplot(3, 1, 3)
ax2.plot(wl1, norm_spec1, 'k')
ax2.plot(wl2, norm_spec2, 'b')
ax2.plot(wl3, norm_spec3, 'k')
ax2.plot(wl4, norm_spec4, 'b')
ax3.plot(wl1, boxcar(norm_spec1, (3, )), 'k')
ax3.plot(wl2, boxcar(norm_spec2, (3, )), 'b')
ax3.plot(wl3, boxcar(norm_spec3, (3, )), 'k')
ax3.plot(wl4, boxcar(norm_spec4, (3, )), 'b')
ax1.plot(wl5, norm_spec5, 'k')
ax2.set_xlim(3900, 4750)
ax3.set_xlim(3900, 4750)
ax1.set_xlim(1150, 1700)
ax1.set_ylim(ax1_ylim[0], ax1_ylim[1])
ax2.set_ylim(ax2_ylim[0], ax2_ylim[1])
ax3.set_ylim(ax2_ylim[0], ax2_ylim[1])
ax1_xticks = np.arange(1150, 1750, 50)
ax2_xticks = np.arange(3900, 4750, 50)
ax1.set_xticks(ax1_xticks)
ax2.set_xticks(ax2_xticks)
ax3.set_xticks(ax2_xticks)
ax1.grid(b='on', color='gray', ls=':')
ax2.grid(b='on', color='gray', ls=':')
ax3.grid(b='on', color='gray', ls=':')
#Label Plots
ax2.set_title('Optical Spectrum %s' % (title))
ax1.set_title('FUV Spectrum %s' % (title))
ax3.set_title('Optical Spectrum %s R~4000' % (title))
ax3.set_xlabel('Wavelength')
ax2.set_ylabel('Relative Intensity')
pyplot.draw()
return ax1, ax2, ax3, fig
