def test_extract_array_even_shape_rounding():
"""
Test overlap_slices (via extract_array) for rounding with an
even-shaped extraction.
"""
data = np.arange(10)
shape = (2,)
positions_expected = [(1.49, (1, 2)), (1.5, (1, 2)), (1.501, (1, 2)),
(1.99, (1, 2)), (2.0, (1, 2)), (2.01, (2, 3)),
(2.49, (2, 3)), (2.5, (2, 3)), (2.501, (2, 3)),
(2.99, (2, 3)), (3.0, (2, 3)), (3.01, (3, 4))]
for pos, exp in positions_expected:
out = extract_array(data, shape, (pos, ), mode='partial')
assert_array_equal(out, exp)
# test negative positions
positions = (-0.99, -0.51, -0.5, -0.49, -0.01, 0)
exp1 = (-99, 0)
exp2 = (0, 1)
expected = [exp1, ] * 6 + [exp2, ]
for pos, exp in zip(positions, expected):
out = extract_array(data, shape, (pos, ), mode='partial',
fill_value=-99)
assert_array_equal(out, exp)
def test_extract_array_odd_shape_rounding():
"""
Test overlap_slices (via extract_array) for rounding with an
even-shaped extraction.
"""
data = np.arange(10)
shape = (3, )
positions_expected = [(1.49, (0, 1, 2)), (1.5, (0, 1, 2)),
(1.501, (1, 2, 3)), (1.99, (1, 2, 3)),
(2.0, (1, 2, 3)), (2.01, (1, 2, 3)),
(2.49, (1, 2, 3)), (2.5, (1, 2, 3)),
(2.501, (2, 3, 4)), (2.99, (2, 3, 4)),
(3.0, (2, 3, 4)), (3.01, (2, 3, 4))]
for pos, exp in positions_expected:
out = extract_array(data, shape, (pos, ), mode='partial')
assert_array_equal(out, exp)
# test negative positions
positions = (-0.99, -0.51, -0.5, -0.49, -0.01, 0)
exp1 = (-99, -99, 0)
exp2 = (-99, 0, 1)
expected = [
exp1,
] * 3 + [
exp2,
] * 4
for pos, exp in zip(positions, expected):
out = extract_array(data,
shape, (pos, ),
mode='partial',
fill_value=-99)
assert_array_equal(out, exp)
def test_extract_array_return_pos():
'''Check that the return position is calculated correctly.
The result will differ by mode. All test here are done in 1d because it's
easier to construct correct test cases.
'''
large_test_array = np.arange(5, dtype=float)
for i in np.arange(-1, 6):
extracted, new_pos = extract_array(large_test_array,
3,
i,
mode='partial',
return_position=True)
assert new_pos == (1, )
# Now check an array with an even number
for i, expected in zip([1.49, 1.51, 3], [0.49, 0.51, 1]):
extracted, new_pos = extract_array(large_test_array, (2, ), (i, ),
mode='strict',
return_position=True)
assert new_pos == (expected, )
# For mode='trim' the answer actually depends
for i, expected in zip(np.arange(-1, 6), (-1, 0, 1, 1, 1, 1, 1)):
extracted, new_pos = extract_array(large_test_array, (3, ), (i, ),
mode='trim',
return_position=True)
assert new_pos == (expected, )
def subtract_psf(data, psf, positions, fluxes, mask=None):
"""
Removes PSF/PRF at the given positions.
To calculate residual images the PSF/PRF model is subtracted from the data
at the given positions.
Parameters
----------
data : ndarray
Image data.
psf : `photutils.psf.DiscretePRF` or `photutils.psf.GaussianPSF`
PSF/PRF model to be substracted from the data.
positions : ndarray
List of center positions where PSF/PRF is removed.
fluxes : ndarray
List of fluxes of the sources, for correct
normalization.
"""
# Set up indices
indices = np.indices(data.shape)
data_ = data.copy()
# Loop over position
for i, position in enumerate(positions):
x_0, y_0 = position
y = extract_array(indices[0], psf.shape, (y_0, x_0))
x = extract_array(indices[1], psf.shape, (y_0, x_0))
psf.amplitude.value = fluxes[i]
psf.x_0.value, psf.y_0.value = x_0, y_0
psf_image = psf(x, y)
data_ = add_array(data_, -psf_image, (y_0, x_0))
return data_
def subtract_psf(data, psf, positions, fluxes, mask=None):
"""
Removes PSF/PRF at the given positions.
To calculate residual images the PSF/PRF model is subtracted from the data
at the given positions.
Parameters
----------
data : ndarray
Image data.
psf : `photutils.psf.DiscretePRF` or `photutils.psf.GaussianPSF`
PSF/PRF model to be substracted from the data.
positions : ndarray
List of center positions where PSF/PRF is removed.
fluxes : ndarray
List of fluxes of the sources, for correct
normalization.
"""
# Set up indices
indices = np.indices(data.shape)
data_ = data.copy()
# Loop over position
for i, position in enumerate(positions):
x_0, y_0 = position
y = extract_array(indices[0], psf.shape, (y_0, x_0))
x = extract_array(indices[1], psf.shape, (y_0, x_0))
psf.amplitude.value = fluxes[i]
psf.x_0.value, psf.y_0.value = x_0, y_0
psf_image = psf(x, y)
data_ = add_array(data_, -psf_image, (y_0, x_0))
return data_
def test_extract_array_1d_odd():
'''Extract 1 d arrays.
All dimensions are treated the same, so we can test in 1 dim.
The first few lines test the most error-prone part: Extraction of an
array on the boundaries.
Additional tests (e.g. dtype of return array) are done for the last
case only.
'''
assert np.all(
extract_array(np.arange(4), (3, ), (
-1, ), fill_value=-99) == np.array([-99, -99, 0]))
assert np.all(
extract_array(np.arange(4), (3, ), (
0, ), fill_value=-99) == np.array([-99, 0, 1]))
for i in [1, 2]:
assert np.all(
extract_array(np.arange(4), (3, ), (
i, )) == np.array([i - 1, i, i + 1]))
assert np.all(
extract_array(np.arange(4), (3, ), (
3, ), fill_value=-99) == np.array([2, 3, -99]))
arrayin = np.arange(4.)
extracted = extract_array(arrayin, (3, ), (4, ))
assert extracted[0] == 3
assert np.isnan(extracted[1]) # since I cannot use `==` to test for nan
assert extracted.dtype == arrayin.dtype
def test_extract_array_1d():
"""In 1d, shape can be int instead of tuple"""
assert np.all(
extract_array(np.arange(4), 3, (
-1, ), fill_value=-99) == np.array([-99, -99, 0]))
assert np.all(
extract_array(np.arange(4), 3, -1, fill_value=-99) == np.array(
[-99, -99, 0]))
def test_extract_array_1d_even():
'''Extract 1 d arrays.
All dimensions are treated the same, so we can test in 1 dim.
'''
assert np.all(extract_array(np.arange(4), (2, ), (0, ),
fill_value=-99) == np.array([-99, 0]))
for i in [1, 2, 3]:
assert np.all(extract_array(np.arange(4), (2, ), (i, )) ==
np.array([i - 1, i]))
assert np.all(extract_array(np.arange(4.), (2, ), (4, ),
fill_value=np.inf) == np.array([3, np.inf]))
def test_extract_array_1d_trim():
'''Extract 1 d arrays.
All dimensions are treated the same, so we can test in 1 dim.
'''
assert np.all(extract_array(np.arange(4), (2, ), (0, ),
mode='trim') == np.array([0]))
for i in [1, 2, 3]:
assert np.all(extract_array(np.arange(4), (2, ), (i, ),
mode='trim') == np.array([i - 1, i]))
assert np.all(extract_array(np.arange(4.), (2, ), (4, ),
mode='trim') == np.array([3]))
def cut_to_bounding_box(self, bkg_nsig=3):
'''
Reduce the array down to the minimum size based on the mask.
'''
if self._empty_mask_flag:
if not self.ignore_warnings:
warn("The mask is empty.")
# Set the center_coords to the default then
self._center_coords = (self._orig_shape[0] / 2,
self._orig_shape[1] / 2)
return
# Not mask, since the mask is for holes.
yslice, xslice = nd.find_objects(~self.mask)[0]
yextent = int(yslice.stop - yslice.start)
xextent = int(xslice.stop - xslice.start)
self._center_coords = (int(yslice.start) + (yextent / 2),
int(xslice.start) + (xextent / 2))
cut_shape = (yextent + 2 * self.pad_size, xextent + 2 * self.pad_size)
cut_arr = extract_array(self.array,
cut_shape,
self.center_coords,
mode='partial',
fill_value=np.NaN)
# Fill the NaNs with samples from the noise distribution
# This does take the correlation of the beam out... this is fine for
# the time being, but adding a quick convolution w/ the beam will make
# this "proper".
all_noise = self.array <= bkg_nsig * self.sigma
nans = np.isnan(cut_arr)
samps = np.random.random_integers(0,
all_noise.sum() - 1,
size=nans.sum())
cut_arr[nans] = self.array[all_noise][samps]
cut_mask = extract_array(self.mask,
cut_shape,
self.center_coords,
mode='partial',
fill_value=True)
self.array = cut_arr
self.mask = cut_mask
def get_img_flags(dqmask, x, y, shape, edge_val=1):
"""
Parameters
----------
dqmask: array-like
Data quality mask for the image
x: array-like
x coordinates of points
y: array-like
y coordinates of points
shape: tuple
Shape of the patch to extract from the image
edge_val: integer
Value to use for pixels outside the edge of the image
Returns
-------
img_flags: `~numpy.ndarray`
Flags selected from the bad pixel mask
"""
if hasattr(x, 'shape'):
rows = x.shape[0]
else:
rows = len(x)
img_flags = []
for n in range(rows):
patch = extract_array(dqmask, shape, (y[n], x[n]), fill_value=edge_val)
val = 0
for n in np.unique(patch):
val = val | n
img_flags.append(val)
img_flags = np.array(img_flags)
return img_flags
def guess_center_nested(image, halfwidth=50):
'''Guess the position of the central object as two-step process
First, this function calculates the center of mass of an image.
This works well if the central object is the only bright source, however
even a moderately bright source that is far away can shift the center of
mass of an image by a few pixels. To improve the first guess the function
selects a subimage with the halfwidth ``halfwidth`` in a second step
and calculates the center of mass of that subimage.
Parameters
----------
image : 2d np.array
input image
halfwidth : int
half width of the subimage selected in the second step.
Returns
-------
xm, ym : float
x and y coordinates estimated position of the central object
'''
xm, ym = ndimage.center_of_mass(np.ma.masked_invalid(image))
n = 2 * halfwidth + 1
subimage, xmymsmall = extract_array(image, (n, n), (xm, ym),
return_position=True)
x1, y1 = ndimage.center_of_mass(np.ma.masked_invalid(subimage))
# xmymsmall is the xm, ym position in the coordinates of subimage
# So, correct the initial (xm, ym) by delta(xmsmall, x1)
return xm + (x1 - xmymsmall[0]), ym + (y1 - xmymsmall[1])
def fit(self, data, indices):
"""
Fit PSF/PRF to data.
Fits the PSF/PRF to the data and returns the best fitting flux.
If the data contains NaN values or if the source is not completely
contained in the image data the fitting is omitted and a flux of 0
is returned.
For reasons of performance, indices for the data have to be created
outside and passed to the function.
The fit is performed on a slice of the data with the same size as
the PRF.
Parameters
----------
data : ndarray
Array containig image data.
indices : ndarray
Array with indices of the data. As
returned by np.indices(data.shape)
Returns
-------
flux : float
Best fit flux value. Returns flux = 0 if PSF is not completely
contained in the image or if NaN values are present.
"""
# Set position
position = (self.y_0.value, self.x_0.value)
# Extract sub array with data of interest
sub_array_data = extract_array(data, self.shape, position)
# Fit only if PSF is completely contained in the image and no NaN
# values are present
if (sub_array_data.shape == self.shape
and not np.isnan(sub_array_data).any()):
y = extract_array(indices[0], self.shape, position)
x = extract_array(indices[1], self.shape, position)
m = self.fitter(self, x, y, sub_array_data)
return m.amplitude.value
else:
return 0
def fit(self, data, indices):
"""
Fit PSF/PRF to data.
Fits the PSF/PRF to the data and returns the best fitting flux.
If the data contains NaN values or if the source is not completely
contained in the image data the fitting is omitted and a flux of 0
is returned.
For reasons of performance, indices for the data have to be created
outside and passed to the function.
The fit is performed on a slice of the data with the same size as
the PRF.
Parameters
----------
data : ndarray
Array containig image data.
indices : ndarray
Array with indices of the data. As
returned by np.indices(data.shape)
Returns
-------
flux : float
Best fit flux value. Returns flux = 0 if PSF is not completely
contained in the image or if NaN values are present.
"""
# Set position
position = (self.y_0.value, self.x_0.value)
# Extract sub array with data of interest
sub_array_data = extract_array(data, self.shape, position)
# Fit only if PSF is completely contained in the image and no NaN
# values are present
if (sub_array_data.shape == self.shape and
not np.isnan(sub_array_data).any()):
y = extract_array(indices[0], self.shape, position)
x = extract_array(indices[1], self.shape, position)
m = self.fitter(self, x, y, sub_array_data)
return m.amplitude.value
else:
return 0
def fit(self, data, indices):
"""
Fit PSF/PRF to data.
Fits the PSF/PRF to the data and returns the best fitting flux.
If the data contains NaN values or if the source is not completely
contained in the image data the fitting is omitted and a flux of 0
is returned.
For reasons of performance, indices for the data have to be created
outside and passed to the function.
The fit is performed on a slice of the data with the same size as
the PRF.
Parameters
----------
data : ndarray
Array containig image data.
indices : ndarray
Array with indices of the data. As
returned by np.indices(data.shape)
"""
# Extract sub array of the data of the size of the PRF grid
sub_array_data = extract_array(data, self.shape,
(self.y_0.value, self.x_0.value))
# Fit only if PSF is completely contained in the image and no NaN
# values are present
if (sub_array_data.shape == self.shape and
not np.isnan(sub_array_data).any()):
y = extract_array(indices[0], self.shape,
(self.y_0.value, self.x_0.value))
x = extract_array(indices[1], self.shape,
(self.y_0.value, self.x_0.value))
# TODO: It should be discussed whether this is the right
# place to fix the warning. Maybe it should be handled better
# in astropy.modeling.fitting
with warnings.catch_warnings():
warnings.simplefilter("ignore", AstropyUserWarning)
m = self.fitter(self, x, y, sub_array_data)
return m.amplitude.value
else:
return 0
def fit(self, data, indices):
"""
Fit PSF/PRF to data.
Fits the PSF/PRF to the data and returns the best fitting flux.
If the data contains NaN values or if the source is not completely
contained in the image data the fitting is omitted and a flux of 0
is returned.
For reasons of performance, indices for the data have to be created
outside and passed to the function.
The fit is performed on a slice of the data with the same size as
the PRF.
Parameters
----------
data : ndarray
Array containig image data.
indices : ndarray
Array with indices of the data. As
returned by np.indices(data.shape)
"""
# Extract sub array of the data of the size of the PRF grid
sub_array_data = extract_array(data, self.shape,
(self.y_0.value, self.x_0.value))
# Fit only if PSF is completely contained in the image and no NaN
# values are present
if (sub_array_data.shape == self.shape
and not np.isnan(sub_array_data).any()):
y = extract_array(indices[0], self.shape,
(self.y_0.value, self.x_0.value))
x = extract_array(indices[1], self.shape,
(self.y_0.value, self.x_0.value))
# TODO: It should be discussed whether this is the right
# place to fix the warning. Maybe it should be handled better
# in astropy.modeling.fitting
with warnings.catch_warnings():
warnings.simplefilter("ignore", AstropyUserWarning)
m = self.fitter(self, x, y, sub_array_data)
return m.amplitude.value
else:
return 0
def test_extract_array_easy(mode):
"""
Test extract_array utility function.
Test by extracting an array of ones out of an array of zeros.
"""
large_test_array = np.zeros((11, 11))
small_test_array = np.ones((5, 5))
large_test_array[3:8, 3:8] = small_test_array
extracted_array = extract_array(large_test_array, (5, 5), (5, 5),
mode=mode)
assert np.all(extracted_array == small_test_array)
def extract_array(*args, **kwargs):
"""
Wrapper for astropy.nddata.utils.extract_array that reproduces v1.1 behavior
even if an older Astropy is installed.
"""
from astropy.nddata.utils import extract_array
# fill_value keyword is not in v1.0.x
if 'fill_value' in inspect.getargspec(extract_array)[0]:
return extract_array(*args, **kwargs)
else:
return _extract_array_astropy1p1(*args, **kwargs)
def test_extract_array_easy(mode):
"""
Test extract_array utility function.
Test by extracting an array of ones out of an array of zeros.
"""
large_test_array = np.zeros((11, 11))
small_test_array = np.ones((5, 5))
large_test_array[3:8, 3:8] = small_test_array
extracted_array = extract_array(large_test_array, (5, 5), (5, 5),
mode=mode)
assert np.all(extracted_array == small_test_array)
def test_extract_array_return_pos():
'''Check that the return position is calculated correctly.
The result will differ by mode. All test here are done in 1d because it's
easier to construct correct test cases.
'''
large_test_array = np.arange(5)
for i in np.arange(-1, 6):
extracted, new_pos = extract_array(large_test_array, 3, i,
mode='partial',
return_position=True)
assert new_pos == (1, )
# Now check an array with an even number
for i, expected in zip([1.49, 1.51, 3], [0.49, 0.51, 1]):
extracted, new_pos = extract_array(large_test_array, (2,), (i,),
mode='strict', return_position=True)
assert new_pos == (expected, )
# For mode='trim' the answer actually depends
for i, expected in zip(np.arange(-1, 6), (-1, 0, 1, 1, 1, 1, 1)):
extracted, new_pos = extract_array(large_test_array, (3,), (i,),
mode='trim', return_position=True)
assert new_pos == (expected, )
def extract_array(*args, **kwargs):
"""
Wrapper for astropy.nddata.utils.extract_array that reproduces v1.1
behavior even if an older Astropy is installed.
"""
from astropy.nddata.utils import extract_array
# fill_value keyword is not in v1.0.x
if 'fill_value' in inspect.getargspec(extract_array)[0]:
return extract_array(*args, **kwargs)
else:
return _extract_array_astropy1p1(*args, **kwargs)
def test_extract_array_1d_odd():
'''Extract 1 d arrays.
All dimensions are treated the same, so we can test in 1 dim.
The first few lines test the most error-prone part: Extraction of an
array on the boundaries.
Additional tests (e.g. dtype of return array) are done for the last
case only.
'''
assert np.all(extract_array(np.arange(4), (3,), (-1, ),
fill_value=-99) == np.array([-99, -99, 0]))
assert np.all(extract_array(np.arange(4), (3,), (0, ),
fill_value=-99) == np.array([-99, 0, 1]))
for i in [1, 2]:
assert np.all(extract_array(np.arange(4), (3,), (i, )) ==
np.array([i-1, i, i+1]))
assert np.all(extract_array(np.arange(4), (3,), (3, ),
fill_value=-99) == np.array([2, 3, -99]))
arrayin = np.arange(4.)
extracted = extract_array(arrayin, (3,), (4, ))
assert extracted[0] == 3
assert np.isnan(extracted[1]) # since I cannot use `==` to test for nan
assert extracted.dtype == arrayin.dtype
def add_islands5533(evt, image):
'''Extract 5x5 and 3x3 event islands from image at FRAME, X, Y pos in ``evt``
Parameters
----------
evt : `astropy.table.Table`
Event table
image : np.array of shape (frame, x, y)
3d background subtracted image
'''
evt['5X5'] = [extract_array(image[i, :, :], (5, 5), (j, k))
for i, j, k in zip(evt['FRAME'], evt['X'] - 1 - (evt.meta['ROIX0'][0] - 1),
evt['Y'] - 1 - (evt.meta['ROIY0'][0] - 1))]
evt['3X3'] = evt['5X5'].data[:, 1:-1, 1:-1]
def extract(self, cutout_region):
data = self.data
data_shape = data.shape
position, shape = self._get_position_shape(data_shape, cutout_region)
logger.debug('Position {} and Shape {}'.format(position, shape))
# No pixels specified, so return the entire HDU
if (not position and not shape) or shape == data_shape:
logger.debug('Returning entire HDU data for {}'.format(
cutout_region.get_extension()))
cutout_data = data
else:
logger.debug('Cutting out {} at {} for extension {} from \
{}.'.format(shape, position, cutout_region.get_extension(),
data.shape))
cutout_data, position = extract_array(data,
shape,
position,
mode='partial',
return_position=True)
if self.wcs:
cutout_shape = cutout_data.shape
output_wcs = deepcopy(self.wcs)
wcs_crpix = output_wcs.wcs.crpix
l_wcs_crpix = len(wcs_crpix)
ranges = cutout_region.get_ranges()
logger.debug('Adjusting WCS.')
for idx, _ in enumerate(ranges):
if idx < l_wcs_crpix:
wcs_crpix[idx] -= (ranges[idx][0] - 1.0)
output_wcs._naxis = list(cutout_shape)
if self.wcs.sip:
curr_sip = self.wcs.sip
output_wcs.sip = Sip(curr_sip.a, curr_sip.b, curr_sip.ap,
curr_sip.bp, wcs_crpix[0:2])
logger.debug('WCS adjusted.')
else:
logger.debug('No WCS present.')
output_wcs = None
return CutoutResult(data=cutout_data,
wcs=output_wcs,
wcs_crpix=wcs_crpix)
def _daofind_cutout_conv(self):
"""
3D array containing 2D cutouts centered on each source from the
DAOFind convolved data.
The cutout size always matches the size of the DAOFind kernel,
which has odd dimensions.
"""
cutout = []
for xpeak, ypeak in zip(self._xpeak, self._ypeak):
cutout.append(
extract_array(self._daofind_convolved_data,
self._daofind_kernel.data.shape, (ypeak, xpeak),
fill_value=0.0))
return np.array(cutout) # all cutouts are the same size
def _daofind_cutout_conv(self):
"""
3D array containing 2D cutouts centered on each source from the
DAOFind convolved data.
The cutout size always matches the size of the DAOFind kernel,
which has odd dimensions.
"""
cutout = []
for xcen, ycen in zip(*np.transpose(self._xypos_finite)):
try:
cutout_ = extract_array(self._daofind_convolved_data,
self._daofind_kernel.shape,
(ycen, xcen),
fill_value=0.0)
except NoOverlapError:
cutout_ = np.zeros(self._daofind_kernel.shape)
cutout.append(cutout_)
return np.array(cutout) # all cutouts are the same size
def extract(self, cutout_region):
data = self.data
data_shape = data.shape
position, shape = self._get_position_shape(data_shape, cutout_region)
self.logger.debug('Position {} and Shape {}'.format(position, shape))
# No pixels specified, so return the entire HDU
if (not position and not shape) or shape == data_shape:
self.logger.debug('Returning entire HDU data for {}'.format(
cutout_region.get_extension()))
cutout_data = data
else:
self.logger.debug('Cutting out {} at {} for extension {} from {}.'.format(
shape, position, cutout_region.get_extension(), data.shape))
cutout_data, position = extract_array(data, shape, position, mode='partial', return_position=True)
if self.wcs is not None:
cutout_shape = cutout_data.shape
output_wcs = deepcopy(self.wcs)
wcs_crpix = output_wcs.wcs.crpix
ranges = cutout_region.get_ranges()
l_ranges = len(ranges)
while len(wcs_crpix) < l_ranges:
wcs_crpix = np.append(wcs_crpix, 1.0)
for idx, _ in enumerate(ranges):
wcs_crpix[idx] -= (ranges[idx][0] - 1)
output_wcs._naxis = list(cutout_shape)
if self.wcs.sip is not None:
curr_sip = self.wcs.sip
output_wcs.sip = Sip(curr_sip.a, curr_sip.b,
curr_sip.ap, curr_sip.bp,
wcs_crpix[0:2])
else:
output_wcs = None
return CutoutResult(data=cutout_data, wcs=output_wcs, wcs_crpix=wcs_crpix)
def subtract_psf(data, psf, posflux, subshape=None):
"""
Subtract PSF/PRFs from an image.
Parameters
----------
data : `~astropy.nddata.NDData` or array (must be 2D)
Image data.
psf : `astropy.modeling.Fittable2DModel` instance
PSF/PRF model to be substracted from the data.
posflux : Array-like of shape (3, N) or `~astropy.table.Table`
Positions and fluxes for the objects to subtract. If an array,
it is interpreted as ``(x, y, flux)`` If a table, the columns
'x_fit', 'y_fit', and 'flux_fit' must be present.
subshape : length-2 or None
The shape of the region around the center of the location to
subtract the PSF from. If None, subtract from the whole image.
Returns
-------
subdata : same shape and type as ``data``
The image with the PSF subtracted
"""
if data.ndim != 2:
raise ValueError('{0}-d array not supported. Only 2-d arrays can be '
'passed to subtract_psf.'.format(data.ndim))
# translate array input into table
if hasattr(posflux, 'colnames'):
if 'x_fit' not in posflux.colnames:
raise ValueError('Input table does not have x_fit')
if 'y_fit' not in posflux.colnames:
raise ValueError('Input table does not have y_fit')
if 'flux_fit' not in posflux.colnames:
raise ValueError('Input table does not have flux_fit')
else:
posflux = Table(names=['x_fit', 'y_fit', 'flux_fit'], data=posflux)
# Set up contstants across the loop
psf = psf.copy()
xname, yname, fluxname = _extract_psf_fitting_names(psf)
indices = np.indices(data.shape)
subbeddata = data.copy()
if subshape is None:
indicies_reversed = indices[::-1]
for row in posflux:
getattr(psf, xname).value = row['x_fit']
getattr(psf, yname).value = row['y_fit']
getattr(psf, fluxname).value = row['flux_fit']
subbeddata -= psf(*indicies_reversed)
else:
for row in posflux:
x_0, y_0 = row['x_fit'], row['y_fit']
y = extract_array(indices[0], subshape, (y_0, x_0))
x = extract_array(indices[1], subshape, (y_0, x_0))
getattr(psf, xname).value = x_0
getattr(psf, yname).value = y_0
getattr(psf, fluxname).value = row['flux_fit']
subbeddata = add_array(subbeddata, -psf(x, y), (y_0, x_0))
return subbeddata
def test_extract_array_wrong_mode():
'''Call extract_array with non-existing mode.'''
with pytest.raises(ValueError) as e:
extract_array(np.arange(4), (2, ), (0, ), mode='full')
assert "Valid modes are 'partial', 'trim', and 'strict'." == str(e.value)
def test_extract_array_1d():
"""In 1d, shape can be int instead of tuple"""
assert np.all(extract_array(np.arange(4), 3, (-1, ),
fill_value=-99) == np.array([-99, -99, 0]))
assert np.all(extract_array(np.arange(4), 3, -1,
fill_value=-99) == np.array([-99, -99, 0]))
def get_subpixel_patch(img_data, src_pos=None, src_shape=None,
max_offset=3, subsampling=5, normalize=False, xmin=None,
ymin=None, xmax=None, ymax=None, order=5,
window_sampling=100, window_radius=1, smoothing=0,
show_plots=False):
"""
Interpolate an image by subdividing each pixel into ``subpixels``
pixels. It also (optionally) centers the patch on the pixel with
the maximum flux.
Parameters
----------
img_data: array-like
The image data to use for extraction
src_pos: tuple, optional
Position (y,x) of the center of the patch. Either obj_pos must
be given or one of the following: xmin,ymin and obj_shape;
xmax,ymax and obj_shape; or xmin,xmax,ymin,ymax.
src_shape: tuple, optional
Shape (y size, x size) of the patch in the *original* image.
Each of these must be an odd number
max_offset: float, optional
Size of the border (in pixels from the original image) to extract
from the patch of the original image. This allows the function
to re-center the patch on the new maximum, which may be slightly
different. If ``offset_buffer=0`` or the change in position is
greater than the ``offset_buffer``, then re-centering is cancelled.
This is necessary because in crowded fields multiple objects
may be overlapping and this prevents fainter sources from being
re-centered on their brighter neighbors. For this reason it is
good to set ``offset_buffer`` small so that only minor corrections
will be made.
*Default=2*
subpixels: integer, optional
Number of pixels to subdivide the original image. This
must be an odd number. *Default=5*
normalize: bool, optional
Whether or not to normalize the data so that the maximum
value is 1. *Default=True*
xmin,ymin: tuple, optional
Location of the top left corner of the patch to extract. If these
arguments are used then either xmax,ymax or obj_shape must also
be set.
xmax, ymax: tuple, optional
Location of the top left corner of the patch to extract. If these
arguments are used then either xmin,ymin or obj_shape must also
be set.
order: tuple, optional
Order of the polynomial to fit to the data. *Default=5*
window_radius: int, optional
Radius of the window (in image pixels) to use to center the patch.
*Default=1*
window_sampling: int, optional
How much to subdivide the window used to recenter the
source. *Default=100*
show_plots: bool, optional
Whether or not to show a plot of the array centered on the
sources position. *Default=False*
Returns
-------
obj_data: `numpy.ndarray`
The subdivided patch of the image centered on the maximum value.
X,Y: `~numpy.ndarray`
x and y coordinates of the patch
"""
from astropy.nddata.utils import extract_array
try:
from scipy import interpolate
except ImportError:
raise Exception(
"You must have scipy installed to use the subpixel method")
err_msg = "To extract a patch one of the following combinations " \
"must be given: obj_pos and obj_shape; xmin,ymin and obj_shape;" \
"xmax,ymax and obj_shape; or xmin,xmax,ymin,ymax,"
# Allow the user to choose the patch based on its center position
# and shape, bounds, or a single x,y bound and shape
if src_pos is None:
if xmin is not None and ymin is not None:
if xmax is not None and ymax is not None:
src_pos = (.5*(ymin+ymax),.5*(xmin+xmax))
src_shape = (ymax-ymin, xmax-xmin)
elif src_shape is not None:
src_pos = (ymin+.5*src_shape[0], xmin+.5*src_shape[1])
else:
raise Exception(err_msg)
elif xmax is not None and ymax is not None:
if src_shape is not None:
src_pos = (ymax-.5*src_shape[0], xmax-.5*src_shape[1])
else:
raise Exception(err_msg)
elif src_shape is None:
raise Exception(err_msg)
obj_pos = np.array(src_pos)
obj_shape = np.array(src_shape)
# use a pixelated position (since this corresponds to the indices of the
# array) to extract that data, which will be used for interpolation
pix_pos = np.round(obj_pos)
# Extract object data from the image with a buffer in case the maximum is
# not in the center and create an interpolation function
data = extract_array(img_data, tuple(obj_shape+2*max_offset), pix_pos)
x_radius = .5*(data.shape[1]-1)
y_radius = .5*(data.shape[0]-1)
X = np.linspace(pix_pos[1]-x_radius, pix_pos[1]+x_radius,
data.shape[1])
Y = np.linspace(pix_pos[0]-y_radius, pix_pos[0]+y_radius,
data.shape[0])
data_func = interpolate.RectBivariateSpline(Y, X, data,
kx=order, ky=order, s=smoothing)
# If the extracted array contains NaN values return a masked array
if not np.all(np.isfinite(data)) or max_offset==0:
x_radius = .5*(obj_shape[1]-1)
y_radius = .5*(obj_shape[0]-1)
X = np.linspace(obj_pos[1]-x_radius, obj_pos[1]+x_radius,
obj_shape[1]*subsampling)
Y = np.linspace(obj_pos[0]-y_radius, obj_pos[0]+y_radius,
obj_shape[0]*subsampling)
new_pos = src_pos
if max_offset==0:
# Get the interpolated information centered on the source position
obj_data = data_func(Y, X)
else:
w = "The patch for the source at {0} ".format(obj_pos)
w += "contains NaN values, cannot interpolate"
warnings.warn(w)
obj_data = None
else:
dx = max_offset*subsampling
X = np.linspace(obj_pos[1]-dx, obj_pos[1]+dx,
(2*dx+1))
Y = np.linspace(obj_pos[0]-dx, obj_pos[0]+dx,
(2*dx+1))
Z = data_func(Y, X)
# Calculate the number of subpixels to move the center
peak_idx = np.array(np.unravel_index(np.argmax(Z), Z.shape))
center = .5*(np.array(Z.shape)-1)
dpeak = np.abs(center-peak_idx)
# Get the the interpolated image centered on the maximum pixel value
center_pos = obj_pos-dpeak/subsampling
X = np.linspace(center_pos[1]-max_offset,
center_pos[1]+max_offset, window_sampling)
Y = np.linspace(center_pos[0]-max_offset,
center_pos[0]+max_offset, window_sampling)
Z = data_func(Y,X)
if show_plots:
import matplotlib.pyplot as plt
plt.imshow(Z, interpolation='none')
plt.title("before centering")
plt.show()
yidx,xidx = np.unravel_index(np.argmax(Z), Z.shape)
new_pos = (Y[yidx], X[xidx])
# Extract the array centered on the new position
x_radius = .5*(obj_shape[1]-1)
y_radius = .5*(obj_shape[0]-1)
X = np.linspace(new_pos[1]-x_radius, new_pos[1]+x_radius,
obj_shape[1]*subsampling)
Y = np.linspace(new_pos[0]-y_radius, new_pos[0]+y_radius,
obj_shape[0]*subsampling)
obj_data = data_func(Y, X)
if show_plots:
import matplotlib.pyplot as plt
plt.imshow(obj_data, interpolation='none')
plt.title("after centering")
plt.show()
peak_idx = np.unravel_index(np.argmax(obj_data), obj_data.shape)
center = (int((obj_data.shape[0]-1)/2), int((obj_data.shape[1]-1)/2.))
dpeak = ((center[0]-peak_idx[0])/subsampling,
(center[1]-peak_idx[1])/subsampling)
# Normalize the data so that each source will be on the same scale
if normalize and obj_data is not None:
obj_data = obj_data/np.max(obj_data)
return obj_data, X, Y, new_pos
def main(imgs_dir, ref_path, new_path, details):
if not os.path.isdir(imgs_dir):
os.makedirs(imgs_dir)
ref_dest = os.path.join(imgs_dir, 'ref.fits')
new_dest = os.path.join(imgs_dir, 'new.fits')
os.rename(ref_path, ref_dest)
os.rename(new_path, new_dest)
ref = si.SingleImage(ref_dest, borders=False) #, crop=((150,150), (150, 150)))
new = si.SingleImage(new_dest, borders=False) #, crop=((150,150), (150, 150)))
#~ ## Adding stars
foo = new.cov_matrix
srcs = new.best_sources
rows = []
for i in range(15):
j = np.random.choice(len(srcs), 1, replace=False)
flux = srcs['flux'][j][0]
xs = srcs['x'][j][0]
ys = srcs['y'][j][0]
position = (ys, xs)
sx, sy = new.stamp_shape
sx += 3
sy += 3
star = extract_array(new.pixeldata.data,
(sx, sy), position,
mode='partial',
fill_value=new._bkg.globalrms)
#print flux
#~ star = flux*new.db.load(j)[0]
x = np.random.choice(new.pixeldata.shape[0]-3*sx, 1)[0] + sx#* np.random.random())
y = np.random.choice(new.pixeldata.shape[1]-3*sy, 1)[0] + sy
# np.int((new.pixeldata.shape[1]-star.shape[1]) * np.random.random())
#~ if new.pixeldata.data[x:x+sx, y:y+sy].shape != star.shape:
#~ import ipdb; ipdb.set_trace()
new.pixeldata.data[x:x+sx, y:y+sy] = star
#~ except:
#~ continue
xc = x+sx/2.
yc = y+sy/2.
app_mag = -2.5*np.log10(flux)+25.
rows.append([yc, xc, app_mag, flux])
newcat = Table(rows=rows, names=['x', 'y', 'app_mag', 'flux'])
fits.writeto(filename=new_dest, header=fits.getheader(new_dest),
data=new.pixeldata.data, overwrite=True)
new._clean()
new = si.SingleImage(new_dest, borders=False) #, crop=((150,150), (150, 150)))
newcat.write(os.path.join(imgs_dir, 'transient.list'),
format='ascii.fast_no_header',
overwrite=True)
fits.writeto(filename=os.path.join(imgs_dir, 'interp_ref.fits'), data=ref.interped, overwrite=True)
fits.writeto(filename=os.path.join(imgs_dir, 'interp_new.fits'), data=new.interped, overwrite=True)
try:
print 'Images to be subtracted: {} {}'.format(ref_dest, new_dest)
import time
t0 = time.time()
D, P, S, mask = ps.diff(ref, new, align=False,
iterative=False, shift=False, beta=True)
dt_z = time.time() - t0
new._clean()
ref._clean()
mea, med, std = sigma_clipped_stats(D.real)
D = np.ma.MaskedArray(D.real, mask).filled(mea)
fits.writeto(os.path.join(imgs_dir,'diff.fits'), D, overwrite=True)
#~ utils.encapsule_R(D, path=os.path.join(imgs_dir, 'diff.fits'))
utils.encapsule_R(P, path=os.path.join(imgs_dir, 'psf_d.fits'))
utils.encapsule_R(S, path=os.path.join(imgs_dir, 's_diff.fits'))
scorrdetected = utils.find_S_local_maxima(S, threshold=3.5)
print 'S_corr found thath {} transients were above 3.5 sigmas'.format(len(scorrdetected))
ascii.write(table=np.asarray(scorrdetected),
output=os.path.join(imgs_dir, 's_corr_detected.csv'),
names=['X_IMAGE', 'Y_IMAGE', 'SIGNIFICANCE'],
format='csv')
S = np.ascontiguousarray(S)
#~ s_bkg = sep.Background(S)
mean, median, std = sigma_clipped_stats(S)
sdetected = sep.extract(S-median, 3.5*std,
filter_kernel=None)
print 'S_corr with sep found thath {} transients were above 3.5 sigmas'.format(len(sdetected))
ascii.write(table=sdetected,
output=os.path.join(imgs_dir, 'sdetected.csv'),
format='csv')
## With OIS
t0 = time.time()
ois_d = ois.optimal_system(fits.getdata(new_dest), fits.getdata(ref_dest))[0]
dt_o = time.time() - t0
utils.encapsule_R(ois_d, path=os.path.join(imgs_dir, 'diff_ois.fits'))
## With HOTPANTS
t0 = time.time()
os.system('hotpants -v 0 -inim {} -tmplim {} -outim {} -r 15 -tu 40000 -tl -100 -il -100 -iu 40000'.format(new_dest, ref_dest,
os.path.join(imgs_dir, 'diff_hot.fits')))
dt_h = time.time() - t0
return [newcat.to_pandas(), [dt_z, dt_o, dt_h]]
except:
raise
def create_from_image(cls,
imdata,
positions,
size,
fluxes=None,
mask=None,
mode='mean',
subsampling=1,
fix_nan=False):
"""
Create a discrete point response function (PRF) from image data.
Given a list of positions and size this function estimates an
image of the PRF by extracting and combining the individual PRFs
from the given positions.
NaN values are either ignored by passing a mask or can be
replaced by the mirrored value with respect to the center of the
PRF.
Note that if fluxes are *not* specified explicitly, it will be
flux estimated from an aperture of the same size as the PRF
image. This does *not* account for aperture corrections so often
will *not* be what you want for anything other than quick-look
needs.
Parameters
----------
imdata : array
Data array with the image to extract the PRF from
positions : List or array or `~astropy.table.Table`
List of pixel coordinate source positions to use in creating
the PRF. If this is a `~astropy.table.Table` it must have
columns called ``x_0`` and ``y_0``.
size : odd int
Size of the quadratic PRF image in pixels.
mask : bool array, optional
Boolean array to mask out bad values.
fluxes : array, optional
Object fluxes to normalize extracted PRFs. If not given (or
None), the flux is estimated from an aperture of the same
size as the PRF image.
mode : {'mean', 'median'}
One of the following modes to combine the extracted PRFs:
* 'mean': Take the pixelwise mean of the extracted PRFs.
* 'median': Take the pixelwise median of the extracted PRFs.
subsampling : int
Factor of subsampling of the PRF (default = 1).
fix_nan : bool
Fix NaN values in the data by replacing it with the
mirrored value. Assuming that the PRF is symmetrical.
Returns
-------
prf : `photutils.psf.sandbox.DiscretePRF`
Discrete PRF model estimated from data.
"""
# Check input array type and dimension.
if np.iscomplexobj(imdata):
raise TypeError('Complex type not supported')
if imdata.ndim != 2:
raise ValueError(f'{imdata.ndim}-d array not supported. '
'Only 2-d arrays supported.')
if size % 2 == 0:
raise TypeError("Size must be odd.")
if fluxes is not None and len(fluxes) != len(positions):
raise TypeError('Position and flux arrays must be of equal '
'length.')
if mask is None:
mask = np.isnan(imdata)
if isinstance(positions, (list, tuple)):
positions = np.array(positions)
if isinstance(positions, Table) or \
(isinstance(positions, np.ndarray) and
positions.dtype.names is not None):
# One can do clever things like
# positions['x_0', 'y_0'].as_array().view((positions['x_0'].dtype,
# 2))
# but that requires positions['x_0'].dtype is
# positions['y_0'].dtype.
# Better do something simple to allow type promotion if required.
pos = np.empty((len(positions), 2))
pos[:, 0] = positions['x_0']
pos[:, 1] = positions['y_0']
positions = pos
if isinstance(fluxes, (list, tuple)):
fluxes = np.array(fluxes)
if mode == 'mean':
combine = np.ma.mean
elif mode == 'median':
combine = np.ma.median
else:
raise Exception('Invalid mode to combine prfs.')
data_internal = np.ma.array(data=imdata, mask=mask)
prf_model = np.ndarray(shape=(subsampling, subsampling, size, size))
positions_subpixel_indices = \
np.array([subpixel_indices(_, subsampling) for _ in positions],
dtype=int)
for i in range(subsampling):
for j in range(subsampling):
extracted_sub_prfs = []
sub_prf_indices = np.all(positions_subpixel_indices == [j, i],
axis=1)
if not sub_prf_indices.any():
raise ValueError('The source coordinates do not sample '
'all sub-pixel positions. Reduce the '
'value of the subsampling parameter.')
positions_sub_prfs = positions[sub_prf_indices]
for k, position in enumerate(positions_sub_prfs):
x, y = position
extracted_prf = extract_array(data_internal, (size, size),
(y, x))
# Check shape to exclude incomplete PRFs at the boundaries
# of the image
if (extracted_prf.shape == (size, size)
and np.ma.sum(extracted_prf) != 0):
# Replace NaN values by mirrored value, with respect
# to the prf's center
if fix_nan:
prf_nan = extracted_prf.mask
if prf_nan.any():
if (prf_nan.sum() > 3
or prf_nan[size // 2, size // 2]):
continue
else:
extracted_prf = mask_to_mirrored_value(
extracted_prf, prf_nan,
(size // 2, size // 2))
# Normalize and add extracted PRF to data cube
if fluxes is None:
extracted_prf_norm = (np.ma.copy(extracted_prf) /
np.ma.sum(extracted_prf))
else:
fluxes_sub_prfs = fluxes[sub_prf_indices]
extracted_prf_norm = (np.ma.copy(extracted_prf) /
fluxes_sub_prfs[k])
extracted_sub_prfs.append(extracted_prf_norm)
else:
continue
prf_model[i, j] = np.ma.getdata(
combine(np.ma.dstack(extracted_sub_prfs), axis=2))
return cls(prf_model, subsampling=subsampling)
def _ts_value(position, counts, exposure, background, C_0_map, kernel, flux,
method, optimizer, threshold):
"""
Compute TS value at a given pixel position i, j using the approach described
in Stewart (2009).
Parameters
----------
position : tuple (i, j)
Pixel position.
counts : `~numpy.ndarray`
Count map.
background : `~numpy.ndarray`
Background map.
exposure : `~numpy.ndarray`
Exposure map.
kernel : `astropy.convolution.Kernel2D`
Source model kernel.
flux : `~numpy.ndarray`
Flux map. The flux value at the given pixel position is used as
starting value for the minimization.
Returns
-------
TS : float
TS value at the given pixel position.
"""
# Get data slices
counts_ = extract_array(counts, kernel.shape, position).astype(float)
background_ = extract_array(background, kernel.shape,
position).astype(float)
exposure_ = extract_array(exposure, kernel.shape, position).astype(float)
C_0_ = extract_array(C_0_map, kernel.shape, position)
model = (exposure_ * kernel._array).astype(float)
C_0 = C_0_.sum()
if threshold is not None:
with np.errstate(invalid='ignore', divide='ignore'):
C_1 = f_cash(flux[position], counts_, background_, model)
# Don't fit if pixel is low significant
if C_0 - C_1 < threshold:
return C_0 - C_1, flux[position] * FLUX_FACTOR, 0
if method == 'fit minuit':
amplitude, niter = _fit_amplitude_minuit(counts_, background_, model,
flux[position])
elif method == 'fit scipy':
amplitude, niter = _fit_amplitude_scipy(counts_, background_, model)
elif method == 'root newton':
amplitude, niter = _root_amplitude(counts_, background_, model,
flux[position])
elif method == 'root brentq':
amplitude, niter = _root_amplitude_brentq(counts_, background_, model)
else:
raise ValueError('Invalid fitting method.')
if niter > MAX_NITER:
logging.warn('Exceeded maximum number of function evaluations!')
return np.nan, amplitude * FLUX_FACTOR, niter
with np.errstate(invalid='ignore', divide='ignore'):
C_1 = f_cash(amplitude, counts_, background_, model)
# Compute and return TS value
return (C_0 - C_1) * np.sign(amplitude), amplitude * FLUX_FACTOR, niter
def test_extract_Array_float():
"""integer is at bin center"""
for a in np.arange(2.51, 3.49, 0.1):
assert np.all(extract_array(np.arange(5), 3, a) ==
np.array([2, 3, 4]))
def create_prf(data,
positions,
size,
fluxes=None,
mask=None,
mode='mean',
subsampling=1,
fix_nan=False):
"""
Estimate point response function (PRF) from image data.
Given a list of positions and size this function estimates an image of
the PRF by extracting and combining the individual PRFs from the given
positions. Different modes of combining are available.
NaN values are either ignored by passing a mask or can be replaced by
the mirrored value with respect to the center of the PRF.
Furthermore it is possible to specify fluxes to have a correct
normalization of the individual PRFs. Otherwise the flux is estimated from
a quadratic aperture of the same size as the PRF image.
Parameters
----------
data : array
Data array
positions : List or array
List of pixel coordinate source positions to use in creating the PRF.
size : odd int
Size of the quadratic PRF image in pixels.
mask : bool array, optional
Boolean array to mask out bad values.
fluxes : array, optional
Object fluxes to normalize extracted PRFs.
mode : {'mean', 'median'}
One of the following modes to combine the extracted PRFs:
* 'mean'
Take the pixelwise mean of the extracted PRFs.
* 'median'
Take the pixelwise median of the extracted PRFs.
subsampling : int
Factor of subsampling of the PRF (default = 1).
fix_nan : bool
Fix NaN values in the data by replacing it with the
mirrored value. Assuming that the PRF is symmetrical.
Returns
-------
prf : `photutils.psf.DiscretePRF`
Discrete PRF model estimated from data.
Notes
-----
In Astronomy different definitions of Point Spread Function (PSF) and
Point Response Function (PRF) are used. Here we assume that the PRF is
an image of a point source after discretization e.g. with a CCD. This
definition is equivalent to the `Spitzer definiton of the PRF
<http://irsa.ipac.caltech.edu/data/SPITZER/docs/dataanalysistools/tools/mopex/mopexusersguide/89/>`_.
References
----------
`Spitzer PSF vs. PRF
<http://irsa.ipac.caltech.edu/data/SPITZER/docs/files/spitzer/PRF_vs_PSF.pdf>`_
`Kepler PSF calibration
<http://keplerscience.arc.nasa.gov/CalibrationPSF.shtml>`_
`The Kepler Pixel Response Function
<http://adsabs.harvard.edu/abs/2010ApJ...713L..97B>`_
"""
# Check input array type and dimension.
if np.iscomplexobj(data):
raise TypeError('Complex type not supported')
if data.ndim != 2:
raise ValueError('{0}-d array not supported. '
'Only 2-d arrays supported.'.format(data.ndim))
if size % 2 == 0:
raise TypeError("Size must be odd.")
if fluxes is not None and len(fluxes) != len(positions):
raise TypeError("Position and flux arrays must be of equal length.")
if mask is None:
mask = np.isnan(data)
if isinstance(positions, (list, tuple)):
positions = np.array(positions)
if isinstance(fluxes, (list, tuple)):
fluxes = np.array(fluxes)
if mode == 'mean':
combine = np.ma.mean
elif mode == 'median':
combine = np.ma.median
else:
raise Exception('Invalid mode to combine prfs.')
data_internal = np.ma.array(data=data, mask=mask)
prf_model = np.ndarray(shape=(subsampling, subsampling, size, size))
positions_subpixel_indices = np.array(
[subpixel_indices(_, subsampling) for _ in positions], dtype=np.int)
for i in range(subsampling):
for j in range(subsampling):
extracted_sub_prfs = []
sub_prf_indices = np.all(positions_subpixel_indices == [j, i],
axis=1)
positions_sub_prfs = positions[sub_prf_indices]
for k, position in enumerate(positions_sub_prfs):
x, y = position
extracted_prf = extract_array(data_internal, (size, size),
(y, x))
# Check shape to exclude incomplete PRFs at the boundaries
# of the image
if (extracted_prf.shape == (size, size)
and np.ma.sum(extracted_prf) != 0):
# Replace NaN values by mirrored value, with respect
# to the prf's center
if fix_nan:
prf_nan = extracted_prf.mask
if prf_nan.any():
if (prf_nan.sum() > 3
or prf_nan[size // 2, size // 2]):
continue
else:
extracted_prf = mask_to_mirrored_num(
extracted_prf, prf_nan,
(size // 2, size // 2))
# Normalize and add extracted PRF to data cube
if fluxes is None:
extracted_prf_norm = (np.ma.copy(extracted_prf) /
np.ma.sum(extracted_prf))
else:
fluxes_sub_prfs = fluxes[sub_prf_indices]
extracted_prf_norm = (np.ma.copy(extracted_prf) /
fluxes_sub_prfs[k])
extracted_sub_prfs.append(extracted_prf_norm)
else:
continue
prf_model[i, j] = np.ma.getdata(
combine(np.ma.dstack(extracted_sub_prfs), axis=2))
return DiscretePRF(prf_model, subsampling=subsampling)
def extr_array(NN,
fitsim,
ra,
dec,
nn_ra=None,
nn_dec=None,
nn_dist=None,
maj=None,
s=(3 / 60.),
head=None,
hdulist=None):
"""
Produces a smaller image from the entire fitsim, with dimension s x s.
around coordinates ra,dec. If head != None, then provide the head and hdulist
of the image that the source is in, else provide fitsim.
(A big part of this function is a re-run of the postage function,
since we need to make a new postage basically, but this time as an array.)
Arguments:
NN - Boolean indicating if it's an NN source or a MG source.
fitsim -- Image (.fits) that the source is located in.
Faster if there is a small cutout file.
ra,dec -- Right ascension and declination of the source. Will be center of image
nn_ra, nn_dec, nn_dist -- RA, DEC and distance of nearest neighbour source.
maj -- major axis of the source. Used for classifying the error.
s -- Dimension of the cutout image. Default 3 arcminutes.
head -- If theres no fitsim file, provide the head and hdulist of the large file.
Returns:
data_array -- Numpy array containing the extracted cutout image.
"""
if not head:
head = pf.getheader(fitsim)
hdulist = pf.open(fitsim)
# for NN take the projected middle as the RA and DEC
if NN:
ra, dec = (ra + nn_ra) / 2., (dec + nn_dec) / 2.
# Parse the WCS keywords in the primary HDU
wcs = pw.WCS(hdulist[0].header)
# Some pixel coordinates of interest.
skycrd = np.array([ra, dec])
skycrd = np.array([[ra, dec, 0, 0]], np.float_)
# Convert pixel coordinates to world coordinates
# The second argument is "origin" -- in this case we're declaring we
# have 1-based (Fortran-like) coordinates.
pixel = wcs.wcs_sky2pix(skycrd, 1)
# Some pixel coordinates of interest.
x = pixel[0][0]
y = pixel[0][1]
pixsize = abs(wcs.wcs.cdelt[0])
if pl.isnan(s):
s = 25.
N = (s / pixsize)
# print 'x=%.5f, y=%.5f, N=%i' %(x,y,N)
ximgsize = head.get('NAXIS1')
yimgsize = head.get('NAXIS2')
if x == 0:
x = ximgsize / 2
if y == 0:
y = yimgsize / 2
offcentre = False
# subimage limits: check if runs over edges
xlim1 = x - (N / 2)
if (xlim1 < 1):
xlim1 = 1
offcentre = True
xlim2 = x + (N / 2)
if (xlim2 > ximgsize):
xlim2 = ximgsize
offcentre = True
ylim1 = y - (N / 2)
if (ylim1 < 1):
ylim1 = 1
offcentre = True
ylim2 = y + (N / 2)
if (ylim2 > yimgsize):
offcentre = True
ylim2 = yimgsize
xl = int(xlim1)
yl = int(ylim1)
xu = int(xlim2)
yu = int(ylim2)
# extract the data array instead of making a postage stamp
data_array = extract_array(hdulist[0].data, (yu - yl, xu - xl), (y, x))
return data_array
def test_extract_array_nan_fillvalue():
if Version(np.__version__) >= Version('1.20'):
msg = 'fill_value cannot be set to np.nan if the input array has'
with pytest.raises(ValueError, match=msg):
extract_array(np.ones((10, 10), dtype=int), (5, 5), (1, 1),
fill_value=np.nan)
def test_extract_array_wrong_mode():
'''Call extract_array with non-existing mode.'''
with pytest.raises(ValueError) as e:
extract_array(np.arange(4), (2, ), (0, ), mode='full')
assert "Valid modes are 'partial', 'trim', and 'strict'." == str(e.value)
def create_prf(data, positions, size, fluxes=None, mask=None, mode='mean',
subsampling=1, fix_nan=False):
"""
Estimate point response function (PRF) from image data.
Given a list of positions and size this function estimates an image of
the PRF by extracting and combining the individual PRFs from the given
positions. Different modes of combining are available.
NaN values are either ignored by passing a mask or can be replaced by
the mirrored value with respect to the center of the PRF.
Furthermore it is possible to specify fluxes to have a correct
normalization of the individual PRFs. Otherwise the flux is estimated from
a quadratic aperture of the same size as the PRF image.
Parameters
----------
data : array
Data array
positions : List or array
List of pixel coordinate source positions to use in creating the PRF.
size : odd int
Size of the quadratic PRF image in pixels.
mask : bool array, optional
Boolean array to mask out bad values.
fluxes : array, optional
Object fluxes to normalize extracted PRFs.
mode : {'mean', 'median'}
One of the following modes to combine the extracted PRFs:
* 'mean'
Take the pixelwise mean of the extracted PRFs.
* 'median'
Take the pixelwise median of the extracted PRFs.
subsampling : int
Factor of subsampling of the PRF (default = 1).
fix_nan : bool
Fix NaN values in the data by replacing it with the
mirrored value. Assuming that the PRF is symmetrical.
Returns
-------
prf : `photutils.psf.DiscretePRF`
Discrete PRF model estimated from data.
Notes
-----
In Astronomy different definitions of Point Spread Function (PSF) and
Point Response Function (PRF) are used. Here we assume that the PRF is
an image of a point source after discretization e.g. with a CCD. This
definition is equivalent to the `Spitzer definiton of the PRF
<http://irsa.ipac.caltech.edu/data/SPITZER/docs/dataanalysistools/tools/mopex/mopexusersguide/89/>`_.
References
----------
`Spitzer PSF vs. PRF
<http://irsa.ipac.caltech.edu/data/SPITZER/docs/files/spitzer/PRF_vs_PSF.pdf>`_
`Kepler PSF calibration
<http://keplerscience.arc.nasa.gov/CalibrationPSF.shtml>`_
`The Kepler Pixel Response Function
<http://adsabs.harvard.edu/abs/2010ApJ...713L..97B>`_
"""
# Check input array type and dimension.
if np.iscomplexobj(data):
raise TypeError('Complex type not supported')
if data.ndim != 2:
raise ValueError('{0}-d array not supported. '
'Only 2-d arrays supported.'.format(data.ndim))
if size % 2 == 0:
raise TypeError("Size must be odd.")
if fluxes is not None and len(fluxes) != len(positions):
raise TypeError("Position and flux arrays must be of equal length.")
if mask is None:
mask = np.isnan(data)
if isinstance(positions, (list, tuple)):
positions = np.array(positions)
if isinstance(fluxes, (list, tuple)):
fluxes = np.array(fluxes)
if mode == 'mean':
combine = np.ma.mean
elif mode == 'median':
combine = np.ma.median
else:
raise Exception('Invalid mode to combine prfs.')
data_internal = np.ma.array(data=data, mask=mask)
prf_model = np.ndarray(shape=(subsampling, subsampling, size, size))
positions_subpixel_indices = np.array([subpixel_indices(_, subsampling)
for _ in positions], dtype=np.int)
for i in range(subsampling):
for j in range(subsampling):
extracted_sub_prfs = []
sub_prf_indices = np.all(positions_subpixel_indices == [j, i],
axis=1)
positions_sub_prfs = positions[sub_prf_indices]
for k, position in enumerate(positions_sub_prfs):
x, y = position
extracted_prf = extract_array(data_internal, (size, size),
(y, x))
# Check shape to exclude incomplete PRFs at the boundaries
# of the image
if (extracted_prf.shape == (size, size) and
np.ma.sum(extracted_prf) != 0):
# Replace NaN values by mirrored value, with respect
# to the prf's center
if fix_nan:
prf_nan = extracted_prf.mask
if prf_nan.any():
if (prf_nan.sum() > 3 or
prf_nan[size // 2, size // 2]):
continue
else:
extracted_prf = mask_to_mirrored_num(
extracted_prf, prf_nan,
(size // 2, size // 2))
# Normalize and add extracted PRF to data cube
if fluxes is None:
extracted_prf_norm = (np.ma.copy(extracted_prf) /
np.ma.sum(extracted_prf))
else:
fluxes_sub_prfs = fluxes[sub_prf_indices]
extracted_prf_norm = (np.ma.copy(extracted_prf) /
fluxes_sub_prfs[k])
extracted_sub_prfs.append(extracted_prf_norm)
else:
continue
prf_model[i, j] = np.ma.getdata(
combine(np.ma.dstack(extracted_sub_prfs), axis=2))
return DiscretePRF(prf_model, subsampling=subsampling)
def find_orientation(i,
fitsim,
ra,
dec,
Maj,
s=(3 / 60.),
plot=False,
head=None,
hdulist=None):
'''
Finds the orientation of multiple gaussian single sources
To run this function for all sources, use setup_find_orientation_multiple_gaussians() instead.
A big part of this function is a re-run of the postage function,
since we need to make a new postage basically, but this time as an array.
Arguments:
i : the new_Index of the source
fitsim: the postage created earlier
ra, dec: the ra and dec of the source
Maj: the major axis of the source
s: the width of the image, default 3 arcmin, because it has to be a tiny bit
lower than the postage created earlier (width 4 arcmin) or NaNs will appear in the image
head and hdulist, the header and hdulist if a postage hasn't been created before.
(This is so it doesn't open every time in the loop but is opened before the loop.)
Output:
i, max_angle, len(err_indices)
Which are: the new_Index of the source, the best angle of the orientation, and
the amount of orientations that have avg flux value > 80% of the peak avg flux value along the line
If plot=True, produces the plots of the best orientation and the Flux vs Orientation as well
'''
################## BEGIN Postage Function #############
if not head:
head = pf.getheader(fitsim)
hdulist = pf.open(fitsim)
# Parse the WCS keywords in the primary HDU
wcs = pw.WCS(hdulist[0].header)
# Some pixel coordinates of interest.
skycrd = np.array([ra, dec])
skycrd = np.array([[ra, dec, 0, 0]], np.float_)
# Convert pixel coordinates to world coordinates
# The second argument is "origin" -- in this case we're declaring we
# have 1-based (Fortran-like) coordinates.
pixel = wcs.wcs_sky2pix(skycrd, 1)
# Some pixel coordinates of interest.
x = pixel[0][0]
y = pixel[0][1]
pixsize = abs(wcs.wcs.cdelt[0])
if pl.isnan(s):
s = 25.
N = (s / pixsize)
# print 'x=%.5f, y=%.5f, N=%i' %(x,y,N)
ximgsize = head.get('NAXIS1')
yimgsize = head.get('NAXIS2')
if x == 0:
x = ximgsize / 2
if y == 0:
y = yimgsize / 2
offcentre = False
# subimage limits: check if runs over edges
xlim1 = x - (N / 2)
if (xlim1 < 1):
xlim1 = 1
offcentre = True
xlim2 = x + (N / 2)
if (xlim2 > ximgsize):
xlim2 = ximgsize
offcentre = True
ylim1 = y - (N / 2)
if (ylim1 < 1):
ylim1 = 1
offcentre = True
ylim2 = y + (N / 2)
if (ylim2 > yimgsize):
offcentre = True
ylim2 = yimgsize
xl = int(xlim1)
yl = int(ylim1)
xu = int(xlim2)
yu = int(ylim2)
################## END Postage Function #############
# extract the data array instead of making a postage stamp
from astropy.nddata.utils import extract_array
data_array = extract_array(hdulist[0].data, (yu - yl, xu - xl), (y, x))
# use a radius for the line that is the major axis,
# but with 2 pixels more added to the radius
# to make sure we do capture the whole source
radius = Maj / 60 * 40 #arcsec -- > arcmin --> image units
radius = int(radius) + 2 # is chosen arbitrarily
# in the P173+55 1 arcmin = 40 image units, should check if this is true everywhere
if True in (np.isnan(data_array)):
if s < (2 / 60.):
print "No hope left for this source "
return i, 0.0, 100.5
elif s == (2 / 60.):
print "Nan in the cutout image AGAIN ", head['OBJECT'], ' i = ', i
try:
return find_orientation(i,
fitsim,
ra,
dec,
Maj,
s=(Maj * 2 / 60 / 60),
plot=plot,
head=head,
hdulist=hdulist)
except RuntimeError:
print "No hope left for this source, "
return i, 0.0, 100.5
else:
print "NaN in the cutout image: ", head['OBJECT'], ' i = ', i
try:
return find_orientation(i,
fitsim,
ra,
dec,
Maj,
s=(2 / 60.),
plot=plot,
head=head,
hdulist=hdulist)
except RuntimeError:
print "No hope left for this source, "
return i, 0.0, 100.5
from scipy.ndimage import interpolation
from scipy.ndimage import map_coordinates
#the center of the image is at the halfway point -1 for using array-index
xcenter = np.shape(data_array)[0] / 2 - 1 # pixel coordinates
ycenter = np.shape(data_array)[0] / 2 - 1 # pixel coordinates
# make a line with 'num' points and radius = radius
x0, y0 = xcenter - radius, ycenter
x1, y1 = xcenter + radius, ycenter
num = 1000
x, y = np.linspace(x0, x1, num), np.linspace(y0, y1, num)
# make a second and third line to have a linewidth of 3
# x0_2, y0_2 = xcenter-radius,ycenter-1
# x1_2, y1_2 = xcenter+radius,ycenter-1
# x0_3, y0_3 = xcenter-radius,ycenter+1
# x1_3, y1_3 = xcenter+radius,ycenter+1
# x2, y2 = np.linspace(x0_2,x1_2,num), np.linspace(y0_2,y1_2,num)
# x3, y3 = np.linspace(x0_3,x1_3,num), np.linspace(y0_3,y1_3,num)
# the final orientation will be max_angle
max_angle = 0
max_value = 0
# flux values for 0 to 179 degrees of rotation (which is also convenietly their index)
all_values = []
for angle in range(0, 180):
# using spline order 3 interpolation to rotate the data by 0-180 deg
data_array2 = interpolation.rotate(data_array,
angle * 1.,
reshape=False)
# extract the values along the line,
zi = map_coordinates(data_array2, np.vstack((y, x)), prefilter=False)
# zi2 = map_coordinates(data_array2, np.vstack((y2,x2)),prefilter=False)
# zi3 = map_coordinates(data_array2, np.vstack((y3,x3)),prefilter=False)
# calc the mean flux
# zi = zi+zi2+zi3
meanie = np.sum(zi)
if meanie > max_value:
max_value = meanie
max_angle = angle
all_values.append(meanie)
# calculate all orientiations for which the average flux lies within
# 80 per cent of the peak average flux
err_orientations = np.where(all_values > (0.8 * max_value))[0]
# print 'winner at : '
# print max_angle, ' degrees, average flux (random units): ' , max_value
# print 'amount of orientations within 80 per cent : ', len(err_orientations)
if len(err_orientations) > 15:
classification = 'Large err'
else:
classification = 'Small err'
if plot:
data_array2 = interpolation.rotate(data_array,
max_angle,
reshape=False)
plt.imshow(data_array2, origin='lower')
plt.plot([x0, x1], [y0, y1], 'r-', alpha=0.3)
plt.plot([x0_2, x1_2], [y0_2, y1_2], 'r-', alpha=0.3)
plt.plot([x0_3, x1_3], [y0_3, y1_3], 'r-', alpha=0.3)
plt.title('Field: ' + head['OBJECT'] + ' | Source ' + str(i) +
'\n Best orientation = ' + str(max_angle) +
' degrees | classification: ' + classification)
# plt.savefig('/data1/osinga/figures/test2_src'+str(i)+'.png')
plt.savefig(
'/data1/osinga/figures/cutouts/all_multiple_gaussians2/elongated/'
+ head['OBJECT'] + 'src_' + str(i) +
'.png') # // EDITED TO INCLUDE '2'
plt.clf()
plt.close()
plt.plot(all_values, label='all orientations')
plt.scatter(err_orientations,
np.array(all_values)[err_orientations],
color='y',
label='0.8 fraction')
plt.axvline(x=max_angle,
ymin=0,
ymax=1,
color='r',
label='best orientation')
plt.title('Best orientation for Source ' + str(i) +
'\nClassification: ' + classification + ' | Error: ' +
str(len(err_orientations)))
plt.ylabel('Average flux (arbitrary units)')
plt.xlabel('orientation (degrees)')
plt.legend()
plt.xlim(0, 180)
# plt.savefig('/data1/osinga/figures/test2_src'+str(i)+'_orientation.png')
plt.savefig(
'/data1/osinga/figures/cutouts/all_multiple_gaussians2/elongated/'
+ head['OBJECT'] + 'src_' + str(i) +
'_orientation.png') # // EDITED TO INCLUDE '2'
plt.clf()
plt.close()
return i, max_angle, len(err_orientations)
def _ts_value(position, counts, exposure, background, C_0_map, kernel, flux,
method, optimizer, threshold):
"""
Compute TS value at a given pixel position i, j using the approach described
in Stewart (2009).
Parameters
----------
position : tuple (i, j)
Pixel position.
counts : `~numpy.ndarray`
Count map.
background : `~numpy.ndarray`
Background map.
exposure : `~numpy.ndarray`
Exposure map.
kernel : `astropy.convolution.Kernel2D`
Source model kernel.
flux : `~numpy.ndarray`
Flux map. The flux value at the given pixel position is used as
starting value for the minimization.
Returns
-------
TS : float
TS value at the given pixel position.
"""
# Get data slices
counts_ = extract_array(counts, kernel.shape, position).astype(float)
background_ = extract_array(background, kernel.shape, position).astype(float)
exposure_ = extract_array(exposure, kernel.shape, position).astype(float)
C_0_ = extract_array(C_0_map, kernel.shape, position)
model = (exposure_ * kernel._array).astype(float)
C_0 = C_0_.sum()
if threshold is not None:
with np.errstate(invalid='ignore', divide='ignore'):
C_1 = f_cash(flux[position], counts_, background_, model)
# Don't fit if pixel is low significant
if C_0 - C_1 < threshold:
return C_0 - C_1, flux[position] * FLUX_FACTOR, 0
if method == 'fit minuit':
amplitude, niter = _fit_amplitude_minuit(counts_, background_, model,
flux[position])
elif method == 'fit scipy':
amplitude, niter = _fit_amplitude_scipy(counts_, background_, model)
elif method == 'root newton':
amplitude, niter = _root_amplitude(counts_, background_, model,
flux[position])
elif method == 'root brentq':
amplitude, niter = _root_amplitude_brentq(counts_, background_, model)
else:
raise ValueError('Invalid fitting method.')
if niter > MAX_NITER:
log.warning('Exceeded maximum number of function evaluations!')
return np.nan, amplitude * FLUX_FACTOR, niter
with np.errstate(invalid='ignore', divide='ignore'):
C_1 = f_cash(amplitude, counts_, background_, model)
# Compute and return TS value
return (C_0 - C_1) * np.sign(amplitude), amplitude * FLUX_FACTOR, niter
def create_from_image(cls, imdata, positions, size, fluxes=None,
mask=None, mode='mean', subsampling=1,
fix_nan=False):
"""
Create a discrete point response function (PRF) from image data.
Given a list of positions and size this function estimates an
image of the PRF by extracting and combining the individual PRFs
from the given positions.
NaN values are either ignored by passing a mask or can be
replaced by the mirrored value with respect to the center of the
PRF.
Note that if fluxes are *not* specified explicitly, it will be
flux estimated from an aperture of the same size as the PRF
image. This does *not* account for aperture corrections so often
will *not* be what you want for anything other than quick-look
needs.
Parameters
----------
imdata : array
Data array with the image to extract the PRF from
positions : List or array or `~astropy.table.Table`
List of pixel coordinate source positions to use in creating
the PRF. If this is a `~astropy.table.Table` it must have
columns called ``x_0`` and ``y_0``.
size : odd int
Size of the quadratic PRF image in pixels.
mask : bool array, optional
Boolean array to mask out bad values.
fluxes : array, optional
Object fluxes to normalize extracted PRFs. If not given (or
None), the flux is estimated from an aperture of the same
size as the PRF image.
mode : {'mean', 'median'}
One of the following modes to combine the extracted PRFs:
* 'mean': Take the pixelwise mean of the extracted PRFs.
* 'median': Take the pixelwise median of the extracted PRFs.
subsampling : int
Factor of subsampling of the PRF (default = 1).
fix_nan : bool
Fix NaN values in the data by replacing it with the
mirrored value. Assuming that the PRF is symmetrical.
Returns
-------
prf : `photutils.psf.sandbox.DiscretePRF`
Discrete PRF model estimated from data.
"""
# Check input array type and dimension.
if np.iscomplexobj(imdata):
raise TypeError('Complex type not supported')
if imdata.ndim != 2:
raise ValueError('{0}-d array not supported. '
'Only 2-d arrays supported.'.format(imdata.ndim))
if size % 2 == 0:
raise TypeError("Size must be odd.")
if fluxes is not None and len(fluxes) != len(positions):
raise TypeError('Position and flux arrays must be of equal '
'length.')
if mask is None:
mask = np.isnan(imdata)
if isinstance(positions, (list, tuple)):
positions = np.array(positions)
if isinstance(positions, Table) or \
(isinstance(positions, np.ndarray) and
positions.dtype.names is not None):
# One can do clever things like
# positions['x_0', 'y_0'].as_array().view((positions['x_0'].dtype,
# 2))
# but that requires positions['x_0'].dtype is
# positions['y_0'].dtype.
# Better do something simple to allow type promotion if required.
pos = np.empty((len(positions), 2))
pos[:, 0] = positions['x_0']
pos[:, 1] = positions['y_0']
positions = pos
if isinstance(fluxes, (list, tuple)):
fluxes = np.array(fluxes)
if mode == 'mean':
combine = np.ma.mean
elif mode == 'median':
combine = np.ma.median
else:
raise Exception('Invalid mode to combine prfs.')
data_internal = np.ma.array(data=imdata, mask=mask)
prf_model = np.ndarray(shape=(subsampling, subsampling, size, size))
positions_subpixel_indices = \
np.array([subpixel_indices(_, subsampling) for _ in positions],
dtype=np.int)
for i in range(subsampling):
for j in range(subsampling):
extracted_sub_prfs = []
sub_prf_indices = np.all(positions_subpixel_indices == [j, i],
axis=1)
if not sub_prf_indices.any():
raise ValueError('The source coordinates do not sample '
'all sub-pixel positions. Reduce the '
'value of the subsampling parameter.')
positions_sub_prfs = positions[sub_prf_indices]
for k, position in enumerate(positions_sub_prfs):
x, y = position
extracted_prf = extract_array(data_internal, (size, size),
(y, x))
# Check shape to exclude incomplete PRFs at the boundaries
# of the image
if (extracted_prf.shape == (size, size) and
np.ma.sum(extracted_prf) != 0):
# Replace NaN values by mirrored value, with respect
# to the prf's center
if fix_nan:
prf_nan = extracted_prf.mask
if prf_nan.any():
if (prf_nan.sum() > 3 or
prf_nan[size // 2, size // 2]):
continue
else:
extracted_prf = mask_to_mirrored_num(
extracted_prf, prf_nan,
(size // 2, size // 2))
# Normalize and add extracted PRF to data cube
if fluxes is None:
extracted_prf_norm = (np.ma.copy(extracted_prf) /
np.ma.sum(extracted_prf))
else:
fluxes_sub_prfs = fluxes[sub_prf_indices]
extracted_prf_norm = (np.ma.copy(extracted_prf) /
fluxes_sub_prfs[k])
extracted_sub_prfs.append(extracted_prf_norm)
else:
continue
prf_model[i, j] = np.ma.getdata(
combine(np.ma.dstack(extracted_sub_prfs), axis=2))
return cls(prf_model, subsampling=subsampling)
def find_orientation(i,
fitsim,
ra,
dec,
Maj,
Min,
s=(3 / 60.),
plot=False,
head=None,
hdulist=None):
'''
Finds the orientation of multiple gaussian single sources
To run this function for all sources, use setup_find_orientation_multiple_gaussians() instead.
A big part of this function is a re-run of the postage function,
since we need to make a new postage basically, but this time as an array.
Arguments:
i : the new_Index of the source
fitsim: the postage created earlier
ra, dec: the ra and dec of the source
Maj: the major axis of the source
s: the width of the image, default 3 arcmin, because it has to be a tiny bit
lower than the postage created earlier (width 4 arcmin) or NaNs will appear in the image
head and hdulist, the header and hdulist if a postage hasn't been created before.
(This is so it doesn't open every time in the loop but is opened before the loop.)
Output:
i, max_angle, len(err_indices)
Which are: the new_Index of the source, the best angle of the orientation, and
the amount of orientations that have avg flux value > 80% of the peak avg flux value along the line
If plot=True, produces the plots of the best orientation and the Flux vs Orientation as well
'''
################## BEGIN Postage Function #############
if not head:
head = pf.getheader(fitsim)
hdulist = pf.open(fitsim)
# Parse the WCS keywords in the primary HDU
wcs = pw.WCS(hdulist[0].header)
# Some pixel coordinates of interest.
skycrd = np.array([ra, dec])
skycrd = np.array([[ra, dec, 0, 0]], np.float_)
# Convert pixel coordinates to world coordinates
# The second argument is "origin" -- in this case we're declaring we
# have 1-based (Fortran-like) coordinates.
pixel = wcs.wcs_sky2pix(skycrd, 1)
# Some pixel coordinates of interest.
x = pixel[0][0]
y = pixel[0][1]
pixsize = abs(wcs.wcs.cdelt[0])
if pl.isnan(s):
s = 25.
N = (s / pixsize)
# print 'x=%.5f, y=%.5f, N=%i' %(x,y,N)
ximgsize = head.get('NAXIS1')
yimgsize = head.get('NAXIS2')
if x == 0:
x = ximgsize / 2
if y == 0:
y = yimgsize / 2
offcentre = False
# subimage limits: check if runs over edges
xlim1 = x - (N / 2)
if (xlim1 < 1):
xlim1 = 1
offcentre = True
xlim2 = x + (N / 2)
if (xlim2 > ximgsize):
xlim2 = ximgsize
offcentre = True
ylim1 = y - (N / 2)
if (ylim1 < 1):
ylim1 = 1
offcentre = True
ylim2 = y + (N / 2)
if (ylim2 > yimgsize):
offcentre = True
ylim2 = yimgsize
xl = int(xlim1)
yl = int(ylim1)
xu = int(xlim2)
yu = int(ylim2)
################## END Postage Function #############
# extract the data array instead of making a postage stamp
from astropy.nddata.utils import extract_array
data_array = extract_array(hdulist[0].data, (yu - yl, xu - xl), (y, x))
# use a radius for the line that is the major axis,
# but with 2 pixels more added to the radius
# to make sure we do capture the whole source
radius = Maj / 60 * 40 #arcsec -- > arcmin --> image units
radius = int(radius) + 2 # is chosen arbitrarily
# in the P173+55 1 arcmin = 40 image units, should check if this is true everywhere, for FieldNames it is.
# check if there are no NaNs in the cutout image, if there are then make smaller cutout
if True in (np.isnan(data_array)):
if s < (2 / 60.):
print "No hope left for this source "
return i, 0.0, 100.5
elif s == (2 / 60.):
print "Nan in the cutout image AGAIN ", head['OBJECT'], ' i = ', i
try:
return find_orientation(i,
fitsim,
ra,
dec,
Maj,
Min,
s=(Maj * 2. / 60. / 60.),
plot=plot,
head=head,
hdulist=hdulist)
except RuntimeError:
print "No hope left for this source, "
return i, 0.0, 100.5
else:
print "NaN in the cutout image: ", head['OBJECT'], ' i = ', i
try:
return find_orientation(i,
fitsim,
ra,
dec,
Maj,
Min,
s=(2 / 60.),
plot=plot,
head=head,
hdulist=hdulist)
except RuntimeError:
print "No hope left for this source, "
return i, 0.0, 100.5
from scipy.ndimage import interpolation
from scipy.ndimage import map_coordinates
#the center of the image is at the halfway point -1 for using array-index
xcenter = np.shape(data_array)[0] / 2 - 1 # pixel coordinates
ycenter = np.shape(data_array)[1] / 2 - 1 # pixel coordinates
# make a line with 'num' points and radius = radius
x0, y0 = xcenter - radius, ycenter
x1, y1 = xcenter + radius, ycenter
num = 1000
x, y = np.linspace(x0, x1, num), np.linspace(y0, y1, num)
# the final orientation will be max_angle
max_angle = 0
max_value = 0
# flux values for 0 to 179 degrees of rotation (which is also convenietly their index)
all_values = []
for angle in range(0, 180):
# using spline order 3 interpolation to rotate the data by 0-180 deg
data_array2 = interpolation.rotate(data_array,
angle * 1.,
reshape=False)
# extract the values along the line,
zi = map_coordinates(data_array2, np.vstack((y, x)), prefilter=False)
# calc the mean flux
meanie = np.sum(zi)
if meanie > max_value:
max_value = meanie
max_angle = angle
all_values.append(meanie)
# calculate all orientiations for which the average flux lies within
# 80 per cent of the peak average flux
err_orientations = np.where(all_values > (0.8 * max_value))[0]
# find the cutoff value, dependent on the distance
cutoff = 2 * np.arctan(Min / Maj) * 180 / np.pi # convert rad to deg
if len(err_orientations) > cutoff:
classification = 'Large err'
else:
classification = 'Small err'
# then find the amount of maxima and the lobe_ratio
from scipy.signal import argrelextrema
data_array2 = interpolation.rotate(data_array, max_angle, reshape=False)
zi = map_coordinates(data_array2, np.vstack((y, x)), prefilter=False)
# find the local maxima in the flux along the line
indx_extrema = argrelextrema(zi, np.greater)
amount_of_maxima = len(indx_extrema[0])
# calculate the flux ratio of the lobes
lobe_ratio_bool = False
lobe_ratio = 0.0 # in case there is only 1 maximum
if amount_of_maxima > 1:
if amount_of_maxima == 2:
lobe_ratio = (zi[indx_extrema][0] / zi[indx_extrema][1])
else:
# more maxima, so the lobe_ratio is defined as the ratio between the brightest lobes
indx_maximum_extrema = np.flip(
np.argsort(zi[indx_extrema]),
0)[:2] #argsort sorts ascending so flip makes it descending
indx_maximum_extrema = indx_extrema[0][indx_maximum_extrema]
lobe_ratio = (zi[indx_maximum_extrema[0]] /
zi[indx_maximum_extrema][1])
if plot:
# the plot of the rotated source and the flux along the line
fig, axes = plt.subplots(nrows=2, figsize=(12, 12))
axes[0].imshow(data_array2, origin='lower')
axes[0].plot([x0, x1], [y0, y1], 'r-', alpha=0.3)
axes[1].plot(zi)
Fratio = 10.
if amount_of_maxima > 1:
if ((1. / Fratio) < lobe_ratio < (Fratio)):
lobe_ratio_bool = True
plt.suptitle('Field: ' + head['OBJECT'] + ' | Source ' + str(i) +
'\n Best orientation = ' + str(max_angle) +
' degrees | classification: ' + classification +
'\nlobe ratio ' + str(lobe_ratio_bool) + ' | extrema: ' +
str(indx_extrema) + '\n lobe ratio: ' + str(lobe_ratio))
# saving the figures to seperate directories
if amount_of_maxima == 2:
if classification == 'Small err':
plt.savefig(
'/data1/osinga/figures/cutouts/P173+55/angle_distance/2_maxima/small_err/'
+ head['OBJECT'] + 'src_' + str(i) +
'.png') # // EDITED TO P173+55
else:
plt.savefig(
'/data1/osinga/figures/cutouts/P173+55/angle_distance/2_maxima/large_err/'
+ head['OBJECT'] + 'src_' + str(i) +
'.png') # // EDITED TO P173+55
elif amount_of_maxima > 2:
if classification == 'Small err':
plt.savefig(
'/data1/osinga/figures/cutouts/P173+55/angle_distance/more_maxima/small_err/'
+ head['OBJECT'] + 'src_' + str(i) +
'.png') # // EDITED TO P173+55
else:
plt.savefig(
'/data1/osinga/figures/cutouts/P173+55/angle_distance/more_maxima/large_err/'
+ head['OBJECT'] + 'src_' + str(i) +
'.png') # // EDITED TO P173+55
else:
plt.savefig(
'/data1/osinga/figures/cutouts/P173+55/angle_distance/1_maximum/'
+ head['OBJECT'] + 'src_' + str(i) +
'.png') # // EDITED TO P173+55
plt.clf()
plt.close()
# the plot of the flux vs orientation
plt.plot(all_values, label='all orientations')
plt.scatter(err_orientations,
np.array(all_values)[err_orientations],
color='y',
label='0.8 fraction')
plt.axvline(x=max_angle,
ymin=0,
ymax=1,
color='r',
label='best orientation')
plt.title('Best orientation for Source ' + str(i) +
'\nClassification: ' + classification + ' | Cutoff: ' +
str(cutoff) + ' | Error: ' + str(len(err_orientations)))
plt.ylabel('Average flux (arbitrary units)')
plt.xlabel('orientation (degrees)')
plt.legend()
plt.xlim(0, 180)
# saving the figures to seperate directories
if amount_of_maxima == 2:
if classification == 'Small err':
plt.savefig(
'/data1/osinga/figures/cutouts/P173+55/angle_distance/2_maxima/small_err/'
+ head['OBJECT'] + 'src_' + str(i) +
'_orientation.png') # // EDITED TO P173+55
else:
plt.savefig(
'/data1/osinga/figures/cutouts/P173+55/angle_distance/2_maxima/large_err/'
+ head['OBJECT'] + 'src_' + str(i) +
'_orientation.png') # // EDITED TO P173+55
elif amount_of_maxima > 2:
if classification == 'Small err':
plt.savefig(
'/data1/osinga/figures/cutouts/P173+55/angle_distance/more_maxima/small_err/'
+ head['OBJECT'] + 'src_' + str(i) +
'_orientation.png') # // EDITED TO P173+55
else:
plt.savefig(
'/data1/osinga/figures/cutouts/P173+55/angle_distance/more_maxima/large_err/'
+ head['OBJECT'] + 'src_' + str(i) +
'_orientation.png') # // EDITED TO P173+55
else:
plt.savefig(
'/data1/osinga/figures/cutouts/P173+55/angle_distance/1_maximum/'
+ head['OBJECT'] + 'src_' + str(i) +
'_orientation.png') # // EDITED TO P173+55
plt.clf()
plt.close()
return i, max_angle, len(
err_orientations), amount_of_maxima, classification, lobe_ratio
def _best_srcs(self):
"""Property, a dictionary of best sources detected in the image.
Keys are:
fitshape: tuple, the size of the stamps on each source detected
sources: a table of sources, with the imformation from sep
positions: an array, with the position of each source stamp
n_sources: the total number of sources extracted
"""
if not hasattr(self, '_best_sources'):
try:
srcs = sep.extract(self.bkg_sub_img,
thresh=6 * self.bkg.globalrms,
mask=self.masked.mask)
except Exception:
sep.set_extract_pixstack(700000)
srcs = sep.extract(self.bkg_sub_img,
thresh=8 * self.bkg.globalrms,
mask=self.masked.mask)
except ValueError:
srcs = sep.extract(self.bkg_sub_img.byteswap().newbyteorder(),
thresh=8 * self.bkg.globalrms,
mask=self.masked.mask)
if len(srcs) < 20:
try:
srcs = sep.extract(self.bkg_sub_img,
thresh=5 * self.bkg.globalrms,
mask=self.masked.mask)
except Exception:
sep.set_extract_pixstack(900000)
srcs = sep.extract(self.bkg_sub_img,
thresh=5 * self.bkg.globalrms,
mask=self.masked.mask)
if len(srcs) < 10:
print 'No sources detected'
#~ print 'raw sources = {}'.format(len(srcs))
p_sizes = np.percentile(srcs['npix'], q=[25, 55, 75])
best_big = srcs['npix'] >= p_sizes[0]
best_small = srcs['npix'] <= p_sizes[2]
best_flag = srcs['flag'] <= 1
fluxes_quartiles = np.percentile(srcs['flux'], q=[15, 85])
low_flux = srcs['flux'] > fluxes_quartiles[0]
# hig_flux = srcs['flux'] < fluxes_quartiles[1]
# best_srcs = srcs[best_big & best_flag & best_small & hig_flux & low_flux]
best_srcs = srcs[best_flag & best_small & low_flux & best_big]
if self._shape is not None:
fitshape = self._shape
else:
p_sizes = 3. * np.sqrt(
np.percentile(best_srcs['npix'], q=[35, 65, 95]))
if p_sizes[1] >= 21:
dx = int(p_sizes[1])
if dx % 2 != 1: dx += 1
fitshape = (dx, dx)
else:
fitshape = (21, 21)
if len(best_srcs) > 1800:
jj = np.random.choice(len(best_srcs), 1800, replace=False)
best_srcs = best_srcs[jj]
print 'Sources good to calculate = {}'.format(len(best_srcs))
self._best_sources = {'sources': best_srcs, 'fitshape': fitshape}
self.db = npdb.NumPyDB_cPickle(self.dbname, mode='store')
pos = []
jj = 0
for row in best_srcs:
position = [row['y'], row['x']]
sub_array_data = extract_array(self.bkg_sub_img,
fitshape,
position,
fill_value=self.bkg.globalrms)
sub_array_data = sub_array_data / np.sum(sub_array_data)
# Patch.append(sub_array_data)
self.db.dump(sub_array_data, jj)
pos.append(position)
jj += 1
# self._best_sources['patches'] = np.array(Patch)
self._best_sources['positions'] = np.array(pos)
self._best_sources['n_sources'] = jj
# self._best_sources['detected'] = srcs
# self.db = npdb.NumPyDB_cPickle(self._dbname, mode='store')
#~ print 'returning best sources\n'
return self._best_sources
def get_sources(img_data, dqmask_data=None, wtmap_data=None, exptime=None,
sex_params={}, objects=None, subtract_bkg=False, gain=None,
wcs=None, aper_radius=None, windowed=True, edge_val=1, origin=0,
transform='wcs'):
"""
Load sources from an image and if a data quality mask is provided, included
a bitmap of flags in the output catalog. This using SEP to detect
sources using SExtractor functions (unless an input catalog is provided)
and by default calculates the windowed position (see SExtractor or SEP docs
for more details).
Parameters
----------
img_data: array-like
Image to use for search
dqmask_data: array-like, optional
Data quality mask array. If this is included the output catalog
will include a ``flags`` column that gives a binary flag for bad
pixels in the image.
wtmap: array-like, optional
Not currently implemented
exptime: float, optional
Exposure time. If ``exptime`` is not given, then no magnitudes
will be calculated
sex_params: dict, optional
Dictionary of SExtractor parameters to pass to the
`~sep.extract` function.
objects: `~astropy.table.Table`, optional
If a catalog of sources has already been generated, SExtraction
will be skipped and the input catalog will be used for aperture
photometry. The input catalog must contain a list of sources
either ``ra,dec`` or ``x,y`` as columns.
subtract_bkg: bool, optional
Whether or not to subtract the background
gain: float, optional
Gain of the detector, used to calculate the magnitude error
wcs: `~astropy.wcs.WCS`, optional
World coordinates to calculate RA,DEC from image X,Y. If ``wcs``
is not given then the output catalog will only contain cartesian
coordinates for each source.
aper_radius: int, optional
Radius of the aperture to use for photometry. If no ``aperture_radius``
is specified, only the Kron flux will be included in the final
catalog
windowed: bool, optional
Whether or not to use SExtractors window algorithm to calculate a more
precise position. *Default=True*
edge_val: integer
Value to use for pixels outside the edge of the image
transform: string, optional
Type of transform to use. Either 'wcs','sip', or 'all'.
*Default=wcs*
"""
import sep
# Estimate the background and subtract it (in place) from the image
# Note: we might not need to do background subtraction due to the
# community pipeline
bkg = sep.Background(img_data, dqmask_data)
if subtract_bkg:
bkg.subfrom(img_data)
bkg = sep.Background(img_data, dqmask_data)
# Find the objects
if objects is None:
if 'extract' not in sex_params:
sex_params['extract'] = {'thresh': 1.5*bkg.globalrms}
if 'thresh' not in sex_params['extract']:
if 'thresh' in sex_params:
sex_params['extract']['thresh'] = \
sex_params['thresh']*bkg.globalrms
else:
raise Exception(
"You must provide a threshold parameter"
" for source extraction")
sources = sep.extract(img_data, **sex_params['extract'])
objects = table.Table(sources)
# Remove sources with a>>b (usually cosmic rays)
#objects = objects[objects['a']<objects['b']*5]
# Set WCS or X, Y if necessary
if wcs is not None:
if transform=='wcs':
transform_method = wcs.wcs_pix2world
elif transform=='all':
transform_method = wcs.all_pix2world
elif transform=='sip':
transform_method = wcs.sip_pix2foc
if 'ra' not in objects.columns.keys() and wcs is not None:
objects['ra'], objects['dec'] = transform_method(
objects['x'], objects['y'], 0)
if 'x' not in objects.columns.keys():
if wcs is None:
raise Exception("You must provide a wcs transformation if "
"specifying ra and dec")
objects['x'], objects['y'] = wcs.all_world2pix(
objects['ra'], objects['dec'], 0)
if windowed:
logger.info("using kron to get windowed positions")
objects['xwin'], objects['ywin'] = get_winpos(
img_data, objects['x'], objects['y'], objects['a'])
if wcs is not None:
objects['rawin'], objects['decwin'] = transform_method(
objects['xwin'], objects['ywin'],0)
# Calculate aperture flux
if aper_radius is not None:
objects['aper_radius'] = aper_radius
flux, flux_err, flag = sep.sum_circle(
img_data, objects['x'], objects['y'], aper_radius, gain=gain)
objects['aper_flux'] = flux
objects['aper_flux_err'] = flux_err
objects['aper_flag'] = flag
objects['aper_mag'] = -2.5*np.log10(objects['aper_flux']/exptime)
if gain is not None:
objects['aper_mag_err'] = 1.0857*np.sqrt(
2*np.pi*aper_radius**2*bkg.globalrms**2+
objects['aper_flux']/gain)/objects['aper_flux']
else:
objects['aper_mag_err'] = 1.0857*np.sqrt(
2*np.pi*aper_radius**2*bkg.globalrms**2)/objects['aper_flux']
# Get the pipeline flags for the image
if dqmask_data is not None:
objects['flags'] = get_img_flags(dqmask_data,
objects['x'], objects['y'],
(2*aper_radius+1, 2*aper_radius+1),
edge_val)
# Calculate the positional error
# See SExtractor Documentation for more on
# ERRX2 and ERRY2
# Ignore for now since this is very computationally
# expensive with little to gain
if False:
back_data = bkg.back()
err_x = np.zeros((len(objects,)), dtype=float)
err_y = np.zeros((len(objects,)), dtype=float)
err_xy = np.zeros((len(objects,)), dtype=float)
for n, src in enumerate(objects):
X = np.linspace(src['x']-aper_radius, src['x']+aper_radius,
2*aper_radius+1)
Y = np.linspace(src['y']-aper_radius, src['y']+aper_radius,
2*aper_radius+1)
X,Y = np.meshgrid(X,Y)
flux = extract_array(img_data,
(2*aper_radius+1, 2*aper_radius+1), (src['y'], src['x']))
back = extract_array(back_data,
(2*aper_radius+1, 2*aper_radius+1), (src['y'], src['x']))
if gain is not None and gain > 0:
sigma2 = back**2 + flux/gain
else:
sigma2 = back**2
flux2 = np.sum(flux)**2
err_x[n] = np.sum(sigma2*(X-src['x'])**2)/flux2
err_y[n] = np.sum(sigma2*(Y-src['y'])**2)/flux2
err_xy[n] = np.sum(sigma2*(X-src['x'])*(Y-src['y']))/flux2
objects['ERRX2'] = err_x
objects['ERRY2'] = err_y
objects['ERRXY'] = err_xy
# include an index for each source that might be useful later on in
# post processing
objects['src_idx'] = [n for n in range(len(objects))]
return objects, bkg
def test_extract_Array_float():
"""integer is at bin center"""
for a in np.arange(2.51, 3.49, 0.1):
assert np.all(extract_array(np.arange(5), 3, a) == np.array([2, 3, 4]))
def subtract_psf(data, psf, posflux, subshape=None):
"""
Subtract PSF/PRFs from an image.
Parameters
----------
data : `~astropy.nddata.NDData` or array (must be 2D)
Image data.
psf : `astropy.modeling.Fittable2DModel` instance
PSF/PRF model to be substracted from the data.
posflux : Array-like of shape (3, N) or `~astropy.table.Table`
Positions and fluxes for the objects to subtract. If an array,
it is interpreted as ``(x, y, flux)`` If a table, the columns
'x_fit', 'y_fit', and 'flux_fit' must be present.
subshape : length-2 or None
The shape of the region around the center of the location to
subtract the PSF from. If None, subtract from the whole image.
Returns
-------
subdata : same shape and type as ``data``
The image with the PSF subtracted
"""
if data.ndim != 2:
raise ValueError(f'{data.ndim}-d array not supported. Only 2-d '
'arrays can be passed to subtract_psf.')
# translate array input into table
if hasattr(posflux, 'colnames'):
if 'x_fit' not in posflux.colnames:
raise ValueError('Input table does not have x_fit')
if 'y_fit' not in posflux.colnames:
raise ValueError('Input table does not have y_fit')
if 'flux_fit' not in posflux.colnames:
raise ValueError('Input table does not have flux_fit')
else:
posflux = Table(names=['x_fit', 'y_fit', 'flux_fit'], data=posflux)
# Set up contstants across the loop
psf = psf.copy()
xname, yname, fluxname = _extract_psf_fitting_names(psf)
indices = np.indices(data.shape)
subbeddata = data.copy()
if subshape is None:
indicies_reversed = indices[::-1]
for row in posflux:
getattr(psf, xname).value = row['x_fit']
getattr(psf, yname).value = row['y_fit']
getattr(psf, fluxname).value = row['flux_fit']
subbeddata -= psf(*indicies_reversed)
else:
for row in posflux:
x_0, y_0 = row['x_fit'], row['y_fit']
# float dtype needed for fill_value=np.nan
y = extract_array(indices[0].astype(float), subshape, (y_0, x_0))
x = extract_array(indices[1].astype(float), subshape, (y_0, x_0))
getattr(psf, xname).value = x_0
getattr(psf, yname).value = y_0
getattr(psf, fluxname).value = row['flux_fit']
subbeddata = add_array(subbeddata, -psf(x, y), (y_0, x_0))
return subbeddata
def extract_data(data, indices, box_size, position):
x, y = position
d = extract_array(data, (box_size, box_size), (y, x))
xi = extract_array(indices[1], (box_size, box_size), (y, x))
yi = extract_array(indices[0], (box_size, box_size), (y, x))
return d, xi, yi
