def test_set_pixstack():
"""Ensure that setting the pixel stack size works."""
old = sep.get_extract_pixstack()
new = old * 2
sep.set_extract_pixstack(new)
assert new == sep.get_extract_pixstack()
sep.set_extract_pixstack(old)
def find_sources_with_sep(img):
"""Return sources (x, y) sorted by brightness. Use SEP package.
"""
import sep
if isinstance(img, np.ma.MaskedArray):
image = img.filled(fill_value=np.median(img)).astype('float32')
else:
image = img.astype('float32')
bkg = sep.Background(image)
thresh = 3. * bkg.globalrms
try:
sources = sep.extract(image - bkg.back(), thresh)
except Exception as e:
buff_message = 'internal pixel buffer full'
if e.message[0:26] == buff_message:
sep.set_extract_pixstack(600000)
try:
sources = sep.extract(image - bkg.back(), thresh)
except Exception as e:
if e.message[0:26] == buff_message:
sep.set_extract_pixstack(900000)
sources = sep.extract(image - bkg.back(), thresh)
sources.sort(order='flux')
return np.array([[asrc['x'], asrc['y']] for asrc in sources[::-1]])
def __init__(self, config_fn=None, config={}):
if config_fn is not None:
config = utils.read_config(config_fn)
self.extract_pixstack = config.pop('extract_pixstack', 300000)
sep.set_extract_pixstack(self.extract_pixstack)
self.step_kws = config
self.sources = {}
def test_long_error_msg():
"""Ensure that the error message is created successfully when
there is an error detail."""
# set extract pixstack to an insanely small value; this will trigger
# a detailed error message when running sep.extract()
old = sep.get_extract_pixstack()
sep.set_extract_pixstack(5)
data = np.ones((10, 10), dtype=np.float64)
with pytest.raises(Exception) as excinfo:
sep.extract(data, 0.1)
msg = excinfo.value.args[0]
assert type(msg) == str # check that message is the native string type
assert msg.startswith("internal pixel buffer full: The limit")
# restore
sep.set_extract_pixstack(old)
def do_stage(self, images):
for i, image in enumerate(images):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise ** 2.0) ** 0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data, self.threshold, minarea=self.min_area,
err=error, deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data, sources['x'], sources['y'],
sources['a'], sources['b'],
np.pi / 2.0, 2.5 * kronrad,
subpix=1, err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) * (sources['a'] ** 2.0 + sources['b'] ** 2.0)) ** 0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data, sources['x'], sources['y'],
6.0 * sources['a'], [0.25, 0.5, 0.75],
normflux=sources['flux'], subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'], sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(), sources['x'], sources['y'],
sources['a'], sources['b'], np.pi / 2.0,
2.5 * sources['kronrad'], subpix=1)
#masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (2.5 * sources['kronrad']) ** 2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
'flux', 'fluxerr', 'background', 'fwhm',
'a', 'b', 'theta', 'kronrad', 'ellipticity',
'fluxrad25', 'fluxrad50', 'fluxrad75',
'x2', 'y2', 'xy', 'flag']
# Add the units and description to the catalogs
image.catalog['x'].unit = 'pixel'
image.catalog['x'].description = 'X coordinate of the object'
image.catalog['y'].unit = 'pixel'
image.catalog['y'].description = 'Y coordinate of the object'
image.catalog['xwin'].unit = 'pixel'
image.catalog['xwin'].description = 'Windowed X coordinate of the object'
image.catalog['ywin'].unit = 'pixel'
image.catalog['ywin'].description = 'Windowed Y coordinate of the object'
image.catalog['xpeak'].unit = 'pixel'
image.catalog['xpeak'].description = 'X coordinate of the peak'
image.catalog['ypeak'].unit = 'pixel'
image.catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
image.catalog['flux'].unit = 'counts'
image.catalog['flux'].description = 'Flux within a Kron-like elliptical aperture'
image.catalog['fluxerr'].unit = 'counts'
image.catalog['fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
image.catalog['background'].unit = 'counts'
image.catalog['background'].description = 'Average background value in the aperture'
image.catalog['fwhm'].unit = 'pixel'
image.catalog['fwhm'].description = 'FWHM of the object'
image.catalog['a'].unit = 'pixel'
image.catalog['a'].description = 'Semi-major axis of the object'
image.catalog['b'].unit = 'pixel'
image.catalog['b'].description = 'Semi-minor axis of the object'
image.catalog['theta'].unit = 'degrees'
image.catalog['theta'].description = 'Position angle of the object'
image.catalog['kronrad'].unit = 'pixel'
image.catalog['kronrad'].description = 'Kron radius used for extraction'
image.catalog['ellipticity'].description = 'Ellipticity'
image.catalog['fluxrad25'].unit = 'pixel'
image.catalog['fluxrad25'].description = 'Radius containing 25% of the flux'
image.catalog['fluxrad50'].unit = 'pixel'
image.catalog['fluxrad50'].description = 'Radius containing 50% of the flux'
image.catalog['fluxrad75'].unit = 'pixel'
image.catalog['fluxrad75'].description = 'Radius containing 75% of the flux'
image.catalog['x2'].unit = 'pixel^2'
image.catalog['x2'].description = 'Variance on X coordinate of the object'
image.catalog['y2'].unit = 'pixel^2'
image.catalog['y2'].description = 'Variance on Y coordinate of the object'
image.catalog['xy'].unit = 'pixel^2'
image.catalog['xy'].description = 'XY covariance of the object'
image.catalog['flag'].description = 'Bit mask combination of extraction and photometry flags'
image.catalog.sort('flux')
image.catalog.reverse()
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (mean_background,
'[counts] Sigma clipped mean of frame background')
logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (median_background,
'[counts] Median of frame background')
logs.add_tag(logging_tags, 'L1MEDIAN', float(median_background))
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (std_background,
'[counts] Robust std dev of frame background')
logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))
# Save some image statistics to the header
good_objects = image.catalog['flag'] == 0
seeing = np.median(image.catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing, '[arcsec] Frame FWHM in arcsec')
logs.add_tag(logging_tags, 'L1FWHM', float(seeing))
mean_ellipticity = stats.sigma_clipped_mean(sources['ellipticity'][good_objects],
3.0)
image.header['L1ELLIP'] = (mean_ellipticity, 'Mean image ellipticity (1-B/A)')
logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))
mean_position_angle = stats.sigma_clipped_mean(sources['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (mean_position_angle,
'[deg] PA of mean image ellipticity')
logs.add_tag(logging_tags, 'L1ELLIPA', float(mean_position_angle))
self.logger.info('Extracted sources', extra=logging_tags)
except Exception as e:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
self.logger.error(e, extra=logging_tags)
return images
def detect_blobs(image, mask, threshold, min_area, deblend_nthresh=500,
deblend_cont=0):
'''Detects blobs in `image` using an implementation of
`SExtractor <http://www.astromatic.net/software/sextractor>`_ [1].
Parameters
----------
image: numpy.ndarray[Union[numpy.uint8, numpy.uint16]]
grayscale image in which blobs should be detected
mask: numpy.ndarray[numpy.bool]
binary image that masks pixel regions in which no blobs should be
detected
threshold: int, optional
factor by which pixel values must be above background
to be considered part of a blob (default: ``5``)
min_area: int, optional
minimal size of a blob
deblend_ntresh: int, optional
number of deblending thresholds (default: ``500``)
deblend_cont: int, optional
minimum contrast ratio for deblending (default: ``0``)
Returns
-------
Tuple[numpy.ndarray[numpy.int32]]
detected blobs and the corresponding centroids
References
----------
.. [1] Bertin, E. & Arnouts, S. 1996: SExtractor: Software for source
extraction, Astronomy & Astrophysics Supplement 317, 393
'''
sep.set_extract_pixstack(10**7)
img = image.astype('float')
# We pad the image with mirrored pixels to prevent border artifacts.
pad = 50
left = img[:, 1:pad]
right = img[:, -pad:-1]
detect_img = np.c_[np.fliplr(left), img, np.fliplr(right)]
upper = detect_img[1:pad, :]
lower = detect_img[-pad:-1, :]
detect_img = np.r_[np.flipud(upper), detect_img, np.flipud(lower)]
logger.info('detect blobs via thresholding and deblending')
detection, blobs = sep.extract(
detect_img, threshold,
minarea=min_area, segmentation_map=True,
deblend_nthresh=deblend_nthresh, deblend_cont=deblend_cont,
filter_kernel=None, clean=False
)
centroids = np.zeros(detect_img.shape, dtype=np.int32)
y = detection['y'].astype(int)
x = detection['x'].astype(int)
# WTF? In rare cases object coorindates lie outside of the image.
n = len(detection)
y[y > detect_img.shape[0]] = detect_img.shape[0]
x[x > detect_img.shape[1]] = detect_img.shape[1]
centroids[y, x] = np.arange(1, n + 1)
# Remove the padded border pixels
blobs = blobs[pad-1:-(pad-1), pad-1:-(pad-1)].copy()
centroids = centroids[pad-1:-(pad-1), pad-1:-(pad-1)].copy()
# Blobs detected outside of regions of interest are discarded.
blobs[mask > 0] = 0
blobs[mh.bwperim(np.invert(mask)) > 0] = 0
mh.labeled.relabel(blobs, inplace=True)
# We need to ensure that centroids are labeled the same way as blobs.
centroids[centroids > 0] = blobs[centroids > 0]
return (blobs, centroids)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import numpy as np
import sep
from astropy.io import fits
from astropy.io import ascii
from astroML import crossmatch as cx
import pandas as pd
sep.set_extract_pixstack(900000)
data_path = os.path.abspath('/home/bos0109/sersic/work/rhino/data/extract_test/stellar/CSTAR/')
images_path = os.path.join(data_path, 'images')
master_Ryan = os.path.join(images_path, 'master10_wcs.fits')
cat_path = os.path.join(data_path, 'cats/stars.dat')
cat = ascii.read(cat_path, names = ['cstarid', 'x', 'y', 'imag'])
data = fits.getdata(master_Ryan)
data = data.byteswap().newbyteorder()
bkg = sep.Background(data)
data = data - bkg
def go_test(cat, image_data, thresh):
threshold = thresh*bkg.globalrms
#make the extraction
sources = sep.extract(image_data, threshold)
cat.to_pandas()
sources = pd.DataFrame(sources)
