def _pyse(self,
detectionthresholdmap,
analysisthresholdmap,
deblend_nthresh,
force_beam,
labelled_data=None,
labels=[]):
"""
Run Python-based source extraction on this image.
Args:
detectionthresholdmap (numpy.ndarray):
analysisthresholdmap (numpy.ndarray):
deblend_nthresh (int): number of subthresholds for deblending. 0
disables.
force_beam (bool): force all extractions to have major/minor axes
equal to the restoring beam
labelled_data (numpy.ndarray): labelled island map (output of
numpy.ndimage.label()). Will be calculated automatically if not
provided.
labels (tuple): list of labels in the island map to use for
fitting.
Returns:
(..utility.containers.ExtractionResults):
This is described in detail in the "Source Extraction System" document
by John Swinbank, available from TKP svn.
"""
# Map our chunks onto a list of islands.
island_list = []
if labelled_data is None:
labels, labelled_data = self.label_islands(detectionthresholdmap,
analysisthresholdmap)
# Get a bounding box for each island:
# NB Slices ordered by label value (1...N,)
# 'None' returned for missing label indices.
slices = ndimage.find_objects(labelled_data)
for label in labels:
chunk = slices[label - 1]
analysis_threshold = (analysisthresholdmap[chunk] /
self.rmsmap[chunk]).max()
# In selected_data only the pixels with the "correct"
# (see above) labels are retained. Other pixel values are
# set to -(bignum).
# In this way, disconnected pixels within (rectangular)
# slices around islands (particularly the large ones) do
# not affect the source measurements.
selected_data = numpy.ma.where(
labelled_data[chunk] == label, self.data_bgsubbed[chunk].data,
-extract.BIGNUM).filled(fill_value=-extract.BIGNUM)
island_list.append(
extract.Island(selected_data, self.rmsmap[chunk], chunk,
analysis_threshold,
detectionthresholdmap[chunk], self.beam,
deblend_nthresh, DEBLEND_MINCONT,
STRUCTURING_ELEMENT))
# If required, we can save the 'left overs' from the deblending and
# fitting processes for later analysis. This needs setting up here:
if self.residuals:
self.residuals_from_gauss_fitting = numpy.zeros(self.data.shape)
self.residuals_from_deblending = numpy.zeros(self.data.shape)
for island in island_list:
self.residuals_from_deblending[island.chunk] += (
island.data.filled(fill_value=0.))
# Deblend each of the islands to its consituent parts, if necessary
if deblend_nthresh:
deblended_list = [x.deblend() for x in island_list]
# deblended_list = [x.deblend() for x in island_list]
island_list = list(utils.flatten(deblended_list))
# Set up the fixed fit parameters if 'force beam' is on:
if force_beam:
fixed = {
'semimajor': self.beam[0],
'semiminor': self.beam[1],
'theta': self.beam[2]
}
else:
fixed = None
# Iterate over the list of islands and measure the source in each,
# appending it to the results list.
results = containers.ExtractionResults()
for island in island_list:
fit_results = island.fit(fixed=fixed)
if fit_results:
measurement, residual = fit_results
else:
# Failed to fit; drop this island and go to the next.
continue
try:
det = extract.Detection(measurement, self, chunk=island.chunk)
if (det.ra.error == float('inf')
or det.dec.error == float('inf')):
logger.warn(
'Bad fit from blind extraction at pixel coords:'
'%f %f - measurement discarded'
'(increase fitting margin?)', det.x, det.y)
else:
results.append(det)
except RuntimeError as e:
logger.error("Island not processed; unphysical?")
if self.residuals:
self.residuals_from_deblending[island.chunk] -= (
island.data.filled(fill_value=0.))
self.residuals_from_gauss_fitting[island.chunk] += residual
def is_usable(det):
# Check that both ends of each axis are usable; that is, that they
# fall within an unmasked part of the image.
# The axis will not likely fall exactly on a pixel number, so
# check all the surroundings.
def check_point(x, y):
x = (int(x), int(numpy.ceil(x)))
y = (int(y), int(numpy.ceil(y)))
for position in itertools.product(x, y):
try:
if self.data.mask[position[0], position[1]]:
# Point falls in mask
return False
except IndexError:
# Point falls completely outside image
return False
# Point is ok
return True
for point in ((det.start_smaj_x, det.start_smaj_y),
(det.start_smin_x, det.start_smin_y),
(det.end_smaj_x, det.end_smaj_y), (det.end_smin_x,
det.end_smin_y)):
if not check_point(*point):
logger.debug("Unphysical source at pixel %f, %f" %
(det.x.value, det.y.value))
return False
return True
# Filter will return a list; ensure we return an ExtractionResults.
return containers.ExtractionResults(list(filter(is_usable, results)))
def deblend(self, niter=0):
"""Return a decomposed numpy array of all the subislands.
Iterate up through subthresholds, looking for our island
splitting into two. If it does, start again, with two or more
separate islands.
"""
logger.debug("Deblending source")
for level in self.subthrrange[niter:]:
# The idea is to retain the parent island when no significant
# subislands are found and jump to the next subthreshold
# using niter.
# Deblending is started at a level higher than the lowest
# pixel value in the island.
# Deblending at the level of the lowest pixel value will
# likely not yield anything, because the island was formed at
# threshold just below that.
# So that is why we use niter+1 (>=1) instead of niter (>=0).
if level > self.data.max():
# level is above the highest pixel value...
# Return the current island.
break
clipped_data = numpy.where(
self.data.filled(fill_value=0) >= level, 1, 0)
labels, number = ndimage.label((clipped_data),
self.structuring_element)
# If we have more than one island, then we need to make subislands.
if number > 1:
subislands = []
label = 0
for chunk in ndimage.find_objects(labels):
label += 1
newdata = numpy.where(labels == label,
self.data.filled(fill_value=-BIGNUM),
-BIGNUM)
# NB: In class Island(object), rms * analysis_threshold
# is taken as the threshold for the bottom of the island.
# Everything below that level is masked.
# For subislands, this product should be equal to level
# and flat, i.e., horizontal.
# We can achieve this by setting rms=level*ones and
# analysis_threshold=1.
island = Island(
newdata[chunk],
(numpy.ones(self.data[chunk].shape) * level),
(slice(self.chunk[0].start + chunk[0].start,
self.chunk[0].start + chunk[0].stop),
slice(self.chunk[1].start + chunk[1].start,
self.chunk[1].start + chunk[1].stop)), 1,
self.detection_map[chunk], self.beam,
self.deblend_nthresh, self.deblend_mincont,
self.structuring_element,
self.rms_orig[chunk[0].start:chunk[0].stop,
chunk[1].start:chunk[1].stop],
self.flux_orig, self.subthrrange)
subislands.append(island)
# This line should filter out any subisland with insufficient
# flux, in about the same way as SExtractor.
# Sufficient means: the flux of the branch above the
# subthreshold (=level) must exceed some user given fraction
# of the composite object, i.e., the original island.
subislands = [
isl for isl in subislands
if (isl.data - numpy.ma.array(
numpy.ones(isl.data.shape) * level, mask=isl.data.mask)
).sum() > self.deblend_mincont * self.flux_orig
]
# Discard subislands below detection threshold
subislands = [
isl for isl in subislands
if (isl.data - isl.detection_map).max() >= 0
]
numbersignifsub = len(subislands)
# Proceed with the previous island, but make sure the next
# subthreshold is higher than the present one.
# Or we would end up in an infinite loop...
if numbersignifsub > 1:
if niter + 1 < self.deblend_nthresh:
# Apparently, the map command always results in
# nested lists.
return list(
utils.flatten([
island.deblend(niter=niter + 1)
for island in subislands
]))
else:
return subislands
elif numbersignifsub == 1 and niter + 1 < self.deblend_nthresh:
return Island.deblend(self, niter=niter + 1)
else:
# In this case we have numbersignifsub == 0 or
# (1 and reached the highest subthreshold level).
# Pull out of deblending loop, return current island.
break
# We've not found any subislands: just return this island.
return self
def _interpolate(self, grid, roundup=False):
"""
Interpolate a grid to produce a map of the dimensions of the image.
Args:
grid (numpy.ma.MaskedArray)
Kwargs:
roundup (bool)
Returns:
(numpy.ma.MaskedArray)
Used to transform the RMS, background or FDR grids produced by
L{_grids()} to a map we can compare with the image data.
If roundup is true, values of the resultant map which are lower than
the input grid are trimmed.
"""
# there's no point in working with the whole of the data array if it's
# masked.
useful_chunk = ndimage.find_objects(numpy.where(self.data.mask, 0, 1))
assert (len(useful_chunk) == 1)
my_xdim, my_ydim = self.data[useful_chunk[0]].shape
if MEDIAN_FILTER:
f_grid = ndimage.median_filter(grid, MEDIAN_FILTER)
if MF_THRESHOLD:
grid = numpy.where(
numpy.fabs(f_grid - grid) > MF_THRESHOLD, f_grid, grid)
else:
grid = f_grid
# Bicubic spline interpolation
xratio = float(my_xdim) / self.back_size_x
yratio = float(my_ydim) / self.back_size_y
# First arg: starting point. Second arg: ending point. Third arg:
# 1j * number of points. (Why is this complex? Sometimes, NumPy has an
# utterly baffling API...)
slicex = slice(-0.5, -0.5 + xratio, 1j * my_xdim)
slicey = slice(-0.5, -0.5 + yratio, 1j * my_ydim)
my_map = numpy.ma.MaskedArray(numpy.zeros(self.data.shape),
mask=self.data.mask)
# Remove the MaskedArrayFutureWarning warning and keep old numpy < 1.11
# behavior
my_map.unshare_mask()
my_map[useful_chunk[0]] = ndimage.map_coordinates(
grid,
numpy.mgrid[slicex, slicey],
mode='nearest',
order=INTERPOLATE_ORDER)
# If the input grid was entirely masked, then the output map must
# also be masked: there's no useful data here. We don't search for
# sources on a masked background/RMS, so this data will be cleanly
# skipped by the rest of the sourcefinder
if numpy.ma.getmask(grid).all():
my_map.mask = True
elif roundup:
# In some cases, the spline interpolation may produce values
# lower than the minimum value in the map. If required, these
# can be trimmed off. No point doing this if the map is already
# fully masked, though.
my_map = numpy.ma.MaskedArray(data=numpy.where(
my_map >= numpy.min(grid), my_map, numpy.min(grid)),
mask=my_map.mask)
return my_map
def __grids(self):
"""Calculate background and RMS grids of this image.
These grids can be interpolated up to make maps of the original image
dimensions: see _interpolate().
This is called automatically when ImageData.backmap,
ImageData.rmsmap or ImageData.fdrmap is first accessed.
"""
# We set up a dedicated logging subchannel, as the sigmaclip loop
# logging is very chatty:
sigmaclip_logger = logging.getLogger(__name__ + '.sigmaclip')
# there's no point in working with the whole of the data array
# if it's masked.
useful_chunk = ndimage.find_objects(numpy.where(self.data.mask, 0, 1))
assert (len(useful_chunk) == 1)
useful_data = self.data[useful_chunk[0]]
my_xdim, my_ydim = useful_data.shape
rmsgrid, bggrid = [], []
for startx in range(0, my_xdim, self.back_size_x):
rmsrow, bgrow = [], []
for starty in range(0, my_ydim, self.back_size_y):
chunk = useful_data[startx:startx + self.back_size_x,
starty:starty + self.back_size_y].ravel()
if not chunk.any():
rmsrow.append(False)
bgrow.append(False)
continue
chunk, sigma, median, num_clip_its = stats.sigma_clip(
chunk, self.beam)
if len(chunk) == 0 or not chunk.any():
rmsrow.append(False)
bgrow.append(False)
else:
mean = numpy.mean(chunk)
rmsrow.append(sigma)
# In the case of a crowded field, the distribution will be
# skewed and we take the median as the background level.
# Otherwise, we take 2.5 * median - 1.5 * mean. This is the
# same as SExtractor: see discussion at
# <http://terapix.iap.fr/forum/showthread.php?tid=267>.
# (mean - median) / sigma is a quick n' dirty skewness
# estimator devised by Karl Pearson.
if numpy.fabs(mean - median) / sigma >= 0.3:
sigmaclip_logger.debug(
'bg skewed, %f clipping iterations', num_clip_its)
bgrow.append(median)
else:
sigmaclip_logger.debug(
'bg not skewed, %f clipping iterations',
num_clip_its)
bgrow.append(2.5 * median - 1.5 * mean)
rmsgrid.append(rmsrow)
bggrid.append(bgrow)
rmsgrid = numpy.ma.array(rmsgrid,
mask=numpy.where(
numpy.array(rmsgrid) == False, 1, 0))
bggrid = numpy.ma.array(bggrid,
mask=numpy.where(
numpy.array(bggrid) == False, 1, 0))
return {'rms': rmsgrid, 'bg': bggrid}
def _pyse(
self, detectionthresholdmap, analysisthresholdmap,
deblend_nthresh, force_beam, labelled_data=None, labels=[]
):
"""
Run Python-based source extraction on this image.
Args:
detectionthresholdmap (numpy.ndarray):
analysisthresholdmap (numpy.ndarray):
deblend_nthresh (int): number of subthresholds for deblending. 0
disables.
force_beam (bool): force all extractions to have major/minor axes
equal to the restoring beam
labelled_data (numpy.ndarray): labelled island map (output of
numpy.ndimage.label()). Will be calculated automatically if not
provided.
labels (list): list of labels in the island map to use for
fitting.
Returns:
(..utility.containers.ExtractionResults):
This is described in detail in the "Source Extraction System" document
by John Swinbank, available from TKP svn.
"""
# Map our chunks onto a list of islands.
island_list = []
if labelled_data is None:
labels, labelled_data = self.label_islands(
detectionthresholdmap, analysisthresholdmap
)
# Get a bounding box for each island:
# NB Slices ordered by label value (1...N,)
# 'None' returned for missing label indices.
slices = ndimage.find_objects(labelled_data)
for label in labels:
chunk = slices[label-1]
analysis_threshold = (analysisthresholdmap[chunk] /
self.rmsmap[chunk]).max()
# In selected_data only the pixels with the "correct"
# (see above) labels are retained. Other pixel values are
# set to -(bignum).
# In this way, disconnected pixels within (rectangular)
# slices around islands (particularly the large ones) do
# not affect the source measurements.
selected_data = numpy.ma.where(
labelled_data[chunk] == label,
self.data_bgsubbed[chunk].data, -extract.BIGNUM
).filled(fill_value=-extract.BIGNUM)
island_list.append(
extract.Island(
selected_data,
self.rmsmap[chunk],
chunk,
analysis_threshold,
detectionthresholdmap[chunk],
self.beam,
deblend_nthresh,
DEBLEND_MINCONT,
STRUCTURING_ELEMENT
)
)
# If required, we can save the 'left overs' from the deblending and
# fitting processes for later analysis. This needs setting up here:
if self.residuals:
self.residuals_from_gauss_fitting = numpy.zeros(self.data.shape)
self.residuals_from_deblending = numpy.zeros(self.data.shape)
for island in island_list:
self.residuals_from_deblending[island.chunk] += (
island.data.filled(fill_value=0.))
# Deblend each of the islands to its consituent parts, if necessary
if deblend_nthresh:
deblended_list = map(lambda x: x.deblend(), island_list)
#deblended_list = [x.deblend() for x in island_list]
island_list = list(utils.flatten(deblended_list))
# Iterate over the list of islands and measure the source in each,
# appending it to the results list.
results = containers.ExtractionResults()
for island in island_list:
if force_beam:
fixed = {'semimajor': self.beam[0],
'semiminor': self.beam[1],
'theta': self.beam[2]}
else:
fixed = None
fit_results = island.fit(fixed=fixed)
if fit_results:
measurement, residual = fit_results
else:
# Failed to fit; drop this island and go to the next.
continue
try:
det = extract.Detection(measurement, self, chunk=island.chunk)
if (det.ra.error == float('inf') or
det.dec.error == float('inf')):
logger.warn('Bad fit from blind extraction at pixel coords:'
'%f %f - measurement discarded'
'(increase fitting margin?)', det.x, det.y )
else:
results.append(det)
except RuntimeError as e:
logger.error("Island not processed; unphysical?")
if self.residuals:
self.residuals_from_deblending[island.chunk] -= (
island.data.filled(fill_value=0.))
self.residuals_from_gauss_fitting[island.chunk] += residual
def is_usable(det):
# Check that both ends of each axis are usable; that is, that they
# fall within an unmasked part of the image.
# The axis will not likely fall exactly on a pixel number, so
# check all the surroundings.
def check_point(x, y):
x = (numpy.floor(x), numpy.ceil(x))
y = (numpy.floor(y), numpy.ceil(y))
for position in itertools.product(x, y):
try:
if self.data.mask[position[0], position[1]]:
# Point falls in mask
return False
except IndexError:
# Point falls completely outside image
return False
# Point is ok
return True
for point in (
(det.start_smaj_x, det.start_smaj_y),
(det.start_smin_x, det.start_smin_y),
(det.end_smaj_x, det.end_smaj_y),
(det.end_smin_x, det.end_smin_y)
):
if not check_point(*point):
logger.debug("Unphysical source at pixel %f, %f" % (det.x.value, det.y.value))
return False
return True
# Filter will return a list; ensure we return an ExtractionResults.
return containers.ExtractionResults(filter(is_usable, results))
def _interpolate(self, grid, roundup=False):
"""
Interpolate a grid to produce a map of the dimensions of the image.
Args:
grid (numpy.ma.MaskedArray)
Kwargs:
roundup (bool)
Returns:
(numpy.ma.MaskedArray)
Used to transform the RMS, background or FDR grids produced by
L{_grids()} to a map we can compare with the image data.
If roundup is true, values of the resultant map which are lower than
the input grid are trimmed.
"""
# there's no point in working with the whole of the data array if it's
# masked.
useful_chunk = ndimage.find_objects(numpy.where(self.data.mask, 0, 1))
assert(len(useful_chunk) == 1)
my_xdim, my_ydim = self.data[useful_chunk[0]].shape
if MEDIAN_FILTER:
f_grid = ndimage.median_filter(grid, MEDIAN_FILTER)
if MF_THRESHOLD:
grid = numpy.where(
numpy.fabs(f_grid - grid) > MF_THRESHOLD, f_grid, grid
)
else:
grid = f_grid
# Bicubic spline interpolation
xratio = float(my_xdim)/self.back_size_x
yratio = float(my_ydim)/self.back_size_y
# First arg: starting point. Second arg: ending point. Third arg:
# 1j * number of points. (Why is this complex? Sometimes, NumPy has an
# utterly baffling API...)
slicex = slice(-0.5, -0.5+xratio, 1j*my_xdim)
slicey = slice(-0.5, -0.5+yratio, 1j*my_ydim)
my_map = numpy.ma.MaskedArray(numpy.zeros(self.data.shape),
mask = self.data.mask)
my_map[useful_chunk[0]] = ndimage.map_coordinates(
grid, numpy.mgrid[slicex, slicey],
mode='nearest', order=INTERPOLATE_ORDER)
# If the input grid was entirely masked, then the output map must
# also be masked: there's no useful data here. We don't search for
# sources on a masked background/RMS, so this data will be cleanly
# skipped by the rest of the sourcefinder
if numpy.ma.getmask(grid).all():
my_map.mask = True
elif roundup:
# In some cases, the spline interpolation may produce values
# lower than the minimum value in the map. If required, these
# can be trimmed off. No point doing this if the map is already
# fully masked, though.
my_map = numpy.ma.MaskedArray(
data = numpy.where(
my_map >= numpy.min(grid), my_map, numpy.min(grid)),
mask = my_map.mask
)
return my_map
def __grids(self):
"""Calculate background and RMS grids of this image.
These grids can be interpolated up to make maps of the original image
dimensions: see _interpolate().
This is called automatically when ImageData.backmap,
ImageData.rmsmap or ImageData.fdrmap is first accessed.
"""
# there's no point in working with the whole of the data array
# if it's masked.
useful_chunk = ndimage.find_objects(numpy.where(self.data.mask, 0, 1))
assert(len(useful_chunk) == 1)
useful_data = self.data[useful_chunk[0]]
my_xdim, my_ydim = useful_data.shape
rmsgrid, bggrid = [], []
for startx in xrange(0, my_xdim, self.back_size_x):
rmsrow, bgrow = [], []
for starty in xrange(0, my_ydim, self.back_size_y):
chunk = useful_data[
startx:startx + self.back_size_x,
starty:starty + self.back_size_y
].ravel()
if not chunk.any():
rmsrow.append(False)
bgrow.append(False)
continue
chunk, sigma, median, num_clip_its = stats.sigma_clip(
chunk, self.beam)
if len(chunk) == 0 or not chunk.any():
rmsrow.append(False)
bgrow.append(False)
else:
mean = numpy.mean(chunk)
rmsrow.append(sigma)
# In the case of a crowded field, the distribution will be
# skewed and we take the median as the background level.
# Otherwise, we take 2.5 * median - 1.5 * mean. This is the
# same as SExtractor: see discussion at
# <http://terapix.iap.fr/forum/showthread.php?tid=267>.
# (mean - median) / sigma is a quick n' dirty skewness
# estimator devised by Karl Pearson.
if numpy.fabs(mean - median) / sigma >= 0.3:
logger.debug(
'bg skewed, %f clipping iterations', num_clip_its)
bgrow.append(median)
else:
logger.debug(
'bg not skewed, %f clipping iterations', num_clip_its)
bgrow.append(2.5 * median - 1.5 * mean)
rmsgrid.append(rmsrow)
bggrid.append(bgrow)
rmsgrid = numpy.ma.array(
rmsgrid, mask=numpy.where(numpy.array(rmsgrid) == False, 1, 0))
bggrid = numpy.ma.array(
bggrid, mask=numpy.where(numpy.array(bggrid) == False, 1, 0))
return {'rms': rmsgrid, 'bg': bggrid}
