def find_sources_with_sep(img):
"""Return sources (x, y) sorted by brightness. Use SEP package.
"""
import sep
if isinstance(img, np.ma.MaskedArray):
image = img.filled(fill_value=np.median(img)).astype('float32')
else:
image = img.astype('float32')
bkg = sep.Background(image)
thresh = 3. * bkg.globalrms
try:
sources = sep.extract(image - bkg.back(), thresh)
except Exception as e:
buff_message = 'internal pixel buffer full'
if e.message[0:26] == buff_message:
sep.set_extract_pixstack(600000)
try:
sources = sep.extract(image - bkg.back(), thresh)
except Exception as e:
if e.message[0:26] == buff_message:
sep.set_extract_pixstack(900000)
sources = sep.extract(image - bkg.back(), thresh)
sources.sort(order='flux')
return np.array([[asrc['x'], asrc['y']] for asrc in sources[::-1]])
def test_set_pixstack():
"""Ensure that setting the pixel stack size works."""
old = sep.get_extract_pixstack()
new = old * 2
sep.set_extract_pixstack(new)
assert new == sep.get_extract_pixstack()
sep.set_extract_pixstack(old)
def run_sep_extractor(self):
# Not entirely sure what this is for but the likeliness of a memory
# error occurring in call to sep.extract is inversely proportional to
# the pixstack...
sep.set_extract_pixstack(10000000)
data = self.image_data_formatted
# generate a background map of data
bkg = sep.Background(data)
# subtract the background map from data
data_sub = data - bkg
self.data_sub_bkg = data_sub
threshold = 100 #2 * np.std(data_sub) + np.min(data_sub)
star_objects = sep.extract(data_sub,
threshold,
minarea=9,
mask=self.image_mask,
gain=3,
deblend_nthresh=32,
deblend_cont=0.0005)
pix_thresh = 100
good_objects = star_objects[star_objects['flag'] < 8]
good_objects = good_objects[good_objects['npix'] < pix_thresh]
return good_objects
def find_sources_with_sep(img):
"""Return sources (x, y) sorted by brightness. Use SEP package.
"""
import sep
if isinstance(img, np.ma.MaskedArray):
image = img.filled(fill_value=np.median(img)).astype('float32')
else:
image = img.astype('float32')
bkg = sep.Background(image)
thresh = 3. * bkg.globalrms
try:
sources = sep.extract(image - bkg.back(), thresh)
except Exception as e:
buff_message = 'internal pixel buffer full'
if e.message[0:26] == buff_message:
sep.set_extract_pixstack(600000)
try:
sources = sep.extract(image - bkg.back(), thresh)
except Exception as e:
if e.message[0:26] == buff_message:
sep.set_extract_pixstack(900000)
sources = sep.extract(image - bkg.back(), thresh)
sources.sort(order='flux')
return np.array([[asrc['x'], asrc['y']] for asrc in sources[::-1]])
def __init__(self, config_fn=None, config={}):
if config_fn is not None:
config = utils.read_config(config_fn)
self.extract_pixstack = config.pop('extract_pixstack', 300000)
sep.set_extract_pixstack(self.extract_pixstack)
self.step_kws = config
self.sources = {}
def __init__(self, config_fn=None, config={}, log_level='info'):
logger._set_defaults(level=log_level.upper())
if config_fn is not None:
config = utils.read_config(config_fn)
self.extract_pixstack = config.pop('extract_pixstack', 300000)
sep.set_extract_pixstack(self.extract_pixstack)
self.config = config
self.sources = {}
def test_long_error_msg():
"""Ensure that the error message is created successfully when
there is an error detail."""
# set extract pixstack to an insanely small value; this will trigger
# a detailed error message when running sep.extract()
old = sep.get_extract_pixstack()
sep.set_extract_pixstack(5)
data = np.ones((10, 10), dtype=np.float64)
with pytest.raises(Exception) as excinfo:
sep.extract(data, 0.1)
msg = excinfo.value.args[0]
assert type(msg) == str # check that message is the native string type
assert msg.startswith("internal pixel buffer full: The limit")
# restore
sep.set_extract_pixstack(old)
def run_sep(data, header, background_mask_threshold=None):
"""Compute mask, background, err; then sep.extract sources.
"""
extract_SN_threshold = config['SEP']['extract_SN_threshold']
background_mask_threshold_scale_factor = config['SEP']['background_mask_threshold_scale_factor']
bg_box_size = config['SEP']['bg_box_size']
min_area = config['SEP']['min_area']
# Pupils cover more pixels than point sources.
# So, set the number of source pixels to be 10% of the total (default=300000 pixels)
sep.set_extract_pixstack(max(300000, int(data.size * 0.1)))
if background_mask_threshold is None:
background_mask_threshold = background_mask_threshold_scale_factor * np.sqrt(np.median(data)) + np.median(data)
background = sep.Background(np.ascontiguousarray(data),
mask=np.ascontiguousarray(data > background_mask_threshold),
bw=bg_box_size, bh=bg_box_size)
read_noise_e = float(header['RDNOISE'])
extract_err = np.sqrt(data + read_noise_e ** 2.0)
return sep.extract(data - background, extract_SN_threshold,
err=extract_err, minarea=min_area, deblend_cont=1.0, filter_kernel=None)
def _best_srcs(self):
"""Property, a dictionary of best sources detected in the image.
Keys are:
fitshape: tuple, the size of the stamps on each source detected
sources: a table of sources, with the imformation from sep
positions: an array, with the position of each source stamp
n_sources: the total number of sources extracted
"""
if not hasattr(self, '_best_sources'):
try:
srcs = sep.extract(self.bkg_sub_img,
thresh=6 * self.bkg.globalrms,
mask=self.masked.mask)
except Exception:
sep.set_extract_pixstack(700000)
srcs = sep.extract(self.bkg_sub_img,
thresh=8 * self.bkg.globalrms,
mask=self.masked.mask)
except ValueError:
srcs = sep.extract(self.bkg_sub_img.byteswap().newbyteorder(),
thresh=8 * self.bkg.globalrms,
mask=self.masked.mask)
if len(srcs) < 20:
try:
srcs = sep.extract(self.bkg_sub_img,
thresh=5 * self.bkg.globalrms,
mask=self.masked.mask)
except Exception:
sep.set_extract_pixstack(900000)
srcs = sep.extract(self.bkg_sub_img,
thresh=5 * self.bkg.globalrms,
mask=self.masked.mask)
if len(srcs) < 10:
print 'No sources detected'
#~ print 'raw sources = {}'.format(len(srcs))
p_sizes = np.percentile(srcs['npix'], q=[25, 55, 75])
best_big = srcs['npix'] >= p_sizes[0]
best_small = srcs['npix'] <= p_sizes[2]
best_flag = srcs['flag'] <= 1
fluxes_quartiles = np.percentile(srcs['flux'], q=[15, 85])
low_flux = srcs['flux'] > fluxes_quartiles[0]
# hig_flux = srcs['flux'] < fluxes_quartiles[1]
# best_srcs = srcs[best_big & best_flag & best_small & hig_flux & low_flux]
best_srcs = srcs[best_flag & best_small & low_flux & best_big]
if self._shape is not None:
fitshape = self._shape
else:
p_sizes = 3. * np.sqrt(
np.percentile(best_srcs['npix'], q=[35, 65, 95]))
if p_sizes[1] >= 21:
dx = int(p_sizes[1])
if dx % 2 != 1: dx += 1
fitshape = (dx, dx)
else:
fitshape = (21, 21)
if len(best_srcs) > 1800:
jj = np.random.choice(len(best_srcs), 1800, replace=False)
best_srcs = best_srcs[jj]
print 'Sources good to calculate = {}'.format(len(best_srcs))
self._best_sources = {'sources': best_srcs, 'fitshape': fitshape}
self.db = npdb.NumPyDB_cPickle(self.dbname, mode='store')
pos = []
jj = 0
for row in best_srcs:
position = [row['y'], row['x']]
sub_array_data = extract_array(self.bkg_sub_img,
fitshape,
position,
fill_value=self.bkg.globalrms)
sub_array_data = sub_array_data / np.sum(sub_array_data)
# Patch.append(sub_array_data)
self.db.dump(sub_array_data, jj)
pos.append(position)
jj += 1
# self._best_sources['patches'] = np.array(Patch)
self._best_sources['positions'] = np.array(pos)
self._best_sources['n_sources'] = jj
# self._best_sources['detected'] = srcs
# self.db = npdb.NumPyDB_cPickle(self._dbname, mode='store')
#~ print 'returning best sources\n'
return self._best_sources
def do_stage(self, images):
for i, image in enumerate(images):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise**2.0)**0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data,
mask=mask,
bw=32,
bh=32,
fw=3,
fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data,
mask=mask,
bw=32,
bh=32,
fw=3,
fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data,
self.threshold,
minarea=self.min_area,
err=error,
deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'],
sources['y'], sources['a'],
sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * kronrad,
subpix=1,
err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) *
(sources['a']**2.0 + sources['b']**2.0))**0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data,
sources['x'],
sources['y'],
6.0 * sources['a'],
[0.25, 0.5, 0.75],
normflux=sources['flux'],
subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'],
sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(),
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 *
sources['kronrad'],
subpix=1)
#masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (
2.5 * sources['kronrad']
)**2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[
background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak',
'ypeak', 'flux', 'fluxerr',
'background', 'fwhm', 'a', 'b',
'theta', 'kronrad', 'ellipticity',
'fluxrad25', 'fluxrad50', 'fluxrad75',
'x2', 'y2', 'xy', 'flag']
# Add the units and description to the catalogs
image.catalog['x'].unit = 'pixel'
image.catalog['x'].description = 'X coordinate of the object'
image.catalog['y'].unit = 'pixel'
image.catalog['y'].description = 'Y coordinate of the object'
image.catalog['xwin'].unit = 'pixel'
image.catalog[
'xwin'].description = 'Windowed X coordinate of the object'
image.catalog['ywin'].unit = 'pixel'
image.catalog[
'ywin'].description = 'Windowed Y coordinate of the object'
image.catalog['xpeak'].unit = 'pixel'
image.catalog['xpeak'].description = 'X coordinate of the peak'
image.catalog['ypeak'].unit = 'pixel'
image.catalog[
'ypeak'].description = 'Windowed Y coordinate of the peak'
image.catalog['flux'].unit = 'counts'
image.catalog[
'flux'].description = 'Flux within a Kron-like elliptical aperture'
image.catalog['fluxerr'].unit = 'counts'
image.catalog[
'fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
image.catalog['background'].unit = 'counts'
image.catalog[
'background'].description = 'Average background value in the aperture'
image.catalog['fwhm'].unit = 'pixel'
image.catalog['fwhm'].description = 'FWHM of the object'
image.catalog['a'].unit = 'pixel'
image.catalog[
'a'].description = 'Semi-major axis of the object'
image.catalog['b'].unit = 'pixel'
image.catalog[
'b'].description = 'Semi-minor axis of the object'
image.catalog['theta'].unit = 'degrees'
image.catalog[
'theta'].description = 'Position angle of the object'
image.catalog['kronrad'].unit = 'pixel'
image.catalog[
'kronrad'].description = 'Kron radius used for extraction'
image.catalog['ellipticity'].description = 'Ellipticity'
image.catalog['fluxrad25'].unit = 'pixel'
image.catalog[
'fluxrad25'].description = 'Radius containing 25% of the flux'
image.catalog['fluxrad50'].unit = 'pixel'
image.catalog[
'fluxrad50'].description = 'Radius containing 50% of the flux'
image.catalog['fluxrad75'].unit = 'pixel'
image.catalog[
'fluxrad75'].description = 'Radius containing 75% of the flux'
image.catalog['x2'].unit = 'pixel^2'
image.catalog[
'x2'].description = 'Variance on X coordinate of the object'
image.catalog['y2'].unit = 'pixel^2'
image.catalog[
'y2'].description = 'Variance on Y coordinate of the object'
image.catalog['xy'].unit = 'pixel^2'
image.catalog['xy'].description = 'XY covariance of the object'
image.catalog[
'flag'].description = 'Bit mask combination of extraction and photometry flags'
image.catalog.sort('flux')
image.catalog.reverse()
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (
mean_background,
'[counts] Sigma clipped mean of frame background')
logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (
median_background, '[counts] Median of frame background')
logs.add_tag(logging_tags, 'L1MEDIAN',
float(median_background))
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (
std_background,
'[counts] Robust std dev of frame background')
logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))
# Save some image statistics to the header
good_objects = image.catalog['flag'] == 0
seeing = np.median(
image.catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing,
'[arcsec] Frame FWHM in arcsec')
logs.add_tag(logging_tags, 'L1FWHM', float(seeing))
mean_ellipticity = stats.sigma_clipped_mean(
sources['ellipticity'][good_objects], 3.0)
image.header['L1ELLIP'] = (mean_ellipticity,
'Mean image ellipticity (1-B/A)')
logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))
mean_position_angle = stats.sigma_clipped_mean(
sources['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (
mean_position_angle, '[deg] PA of mean image ellipticity')
logs.add_tag(logging_tags, 'L1ELLIPA',
float(mean_position_angle))
self.logger.info('Extracted sources', extra=logging_tags)
except Exception as e:
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
self.logger.error(e, extra=logging_tags)
return images
def detect_blobs(image, mask, threshold, min_area, deblend_nthresh=500,
deblend_cont=0):
'''Detects blobs in `image` using an implementation of
`SExtractor <http://www.astromatic.net/software/sextractor>`_ [1].
Parameters
----------
image: numpy.ndarray[Union[numpy.uint8, numpy.uint16]]
grayscale image in which blobs should be detected
mask: numpy.ndarray[numpy.bool]
binary image that masks pixel regions in which no blobs should be
detected
threshold: int, optional
factor by which pixel values must be above background
to be considered part of a blob (default: ``5``)
min_area: int, optional
minimal size of a blob
deblend_ntresh: int, optional
number of deblending thresholds (default: ``500``)
deblend_cont: int, optional
minimum contrast ratio for deblending (default: ``0``)
Returns
-------
Tuple[numpy.ndarray[numpy.int32]]
detected blobs and the corresponding centroids
References
----------
.. [1] Bertin, E. & Arnouts, S. 1996: SExtractor: Software for source
extraction, Astronomy & Astrophysics Supplement 317, 393
'''
sep.set_extract_pixstack(10**7)
img = image.astype('float')
# We pad the image with mirrored pixels to prevent border artifacts.
pad = 50
left = img[:, 1:pad]
right = img[:, -pad:-1]
detect_img = np.c_[np.fliplr(left), img, np.fliplr(right)]
upper = detect_img[1:pad, :]
lower = detect_img[-pad:-1, :]
detect_img = np.r_[np.flipud(upper), detect_img, np.flipud(lower)]
logger.info('detect blobs via thresholding and deblending')
detection, blobs = sep.extract(
detect_img, threshold,
minarea=min_area, segmentation_map=True,
deblend_nthresh=deblend_nthresh, deblend_cont=deblend_cont,
filter_kernel=None, clean=False
)
centroids = np.zeros(detect_img.shape, dtype=np.int32)
y = detection['y'].astype(int)
x = detection['x'].astype(int)
# WTF? In rare cases object coorindates lie outside of the image.
n = len(detection)
y[y > detect_img.shape[0]] = detect_img.shape[0]
x[x > detect_img.shape[1]] = detect_img.shape[1]
centroids[y, x] = np.arange(1, n + 1)
# Remove the padded border pixels
blobs = blobs[pad-1:-(pad-1), pad-1:-(pad-1)].copy()
centroids = centroids[pad-1:-(pad-1), pad-1:-(pad-1)].copy()
# Blobs detected outside of regions of interest are discarded.
blobs[mask > 0] = 0
blobs[mh.bwperim(np.invert(mask)) > 0] = 0
mh.labeled.relabel(blobs, inplace=True)
# We need to ensure that centroids are labeled the same way as blobs.
centroids[centroids > 0] = blobs[centroids > 0]
return (blobs, centroids)
def join_images(science_raw, science_mask, reference_raw, reference_mask, sigma_cut=5., use_pixels=False, show=False,
percent=99, size_cut=True, pixstack_limit=None):
"""Join two images to fittable vectors"""
science = np.ma.array(science_raw, mask=science_mask, copy=True)
reference = np.ma.array(reference_raw, mask=reference_mask, copy=True)
science_std, _ = fit_noise(science)
reference_std, _ = fit_noise(reference)
if use_pixels:
# remove pixels less than `percent` percentile above sky level to speed fitting
science.mask[science <= np.nanpercentile(science.compressed(), percent)] = True
reference.mask[reference <= np.nanpercentile(reference.compressed(), percent)] = True
# flatten into 1d arrays of good pixels
science.mask |= reference.mask
reference.mask |= science.mask
science_flatten = science.compressed()
reference_flatten = reference.compressed()
logging.info('Found {0} usable pixels for gain matching'.format(science_flatten.size))
if science_flatten.size == 0:
logging.error('No pixels in common at this percentile ({0}); lower and try again'.format(percent))
else:
if pixstack_limit is None:
pixstack_limit = science.size // 20
if pixstack_limit > 300000:
sep.set_extract_pixstack(pixstack_limit)
science_sources = sep.extract(np.ascontiguousarray(science.data), thresh=sigma_cut, err=science_std, mask=np.ascontiguousarray(science.mask))
reference_sources = sep.extract(np.ascontiguousarray(reference.data), thresh=sigma_cut, err=reference_std, mask=np.ascontiguousarray(reference.mask))
science_sources = science_sources[science_sources['errx2'] != np.inf] # exclude partially masked sources
reference_sources = reference_sources[reference_sources['errx2'] != np.inf]
dx = science_sources['x'] - reference_sources['x'][:, np.newaxis]
dy = science_sources['y'] - reference_sources['y'][:, np.newaxis]
separation = np.sqrt(dx**2 + dy**2)
sigma_eqv = np.sqrt((reference_sources['a']**2 + reference_sources['b']**2) / 2.)
matches = (np.min(separation, axis=1) < 2. * sigma_eqv)
if size_cut:
# cut unusually large/small sources (assumes most sources are real)
med_sigma = np.median(sigma_eqv) # median sigma if all sources were circular Gaussians
absdev_sigma = np.abs(sigma_eqv - med_sigma)
std_sigma = np.median(absdev_sigma) * np.sqrt(np.pi / 2)
matches &= (absdev_sigma < 3 * std_sigma)
inds = np.argmin(separation, axis=1)
science_flatten = science_sources['flux'][inds][matches]
reference_flatten = reference_sources['flux'][matches]
logging.info('Found {0} stars in common for gain matching'.format(science_flatten.size))
if science_flatten.size <= 2:
logging.error('Too few stars in common at {0}-sigma; lower and try again'.format(sigma_cut))
raise ValueError()
if show:
import matplotlib.pyplot as plt
plt.ion()
plt.figure(1)
plt.clf()
vmin, vmax = np.nanpercentile(science, (1, 99))
plt.imshow(science, vmin=vmin, vmax=vmax)
plt.title('Science')
if not use_pixels:
plt.plot(reference_sources['x'][matches], reference_sources['y'][matches], 'o', mfc='none', mec='r')
plt.figure(2)
plt.clf()
vmin, vmax = np.nanpercentile(reference, (1, 99))
plt.imshow(reference, vmin=vmin, vmax=vmax)
plt.title('Reference')
if not use_pixels:
plt.plot(reference_sources['x'][matches], reference_sources['y'][matches], 'o', mfc='none', mec='r')
plt.figure(3)
plt.clf()
plt.loglog(reference_flatten, science_flatten, '.')
plt.xlabel('Reference')
plt.ylabel('Science')
return reference_flatten, science_flatten
def sextract(self,
band,
sub_background=False,
force_segmap=None,
use_mask=False,
incl_apphot=False):
# perform sextractor on single band only (may expand for matched source phot)
# Generate segmap and segmask
if (band == conf.DETECTION_NICKNAME) | (band
== conf.MODELING_NICKNAME):
idx = 0
else:
idx = self._band2idx(band)
image = self.images[idx].copy()
wgts = self.weights[idx].copy()
wgts[wgts == 0] = -1
var = 1. / wgts
var[wgts == -1] = 0
mask = self.masks[idx].copy()
background = self.backgrounds[idx]
# Supply a segmap to "match" detection
if force_segmap is not None:
var[force_segmap] = 0
if (self.weights == 1).all() | (conf.USE_DETECTION_WEIGHT == False):
# No weight supplied by user
var = None
thresh = conf.THRESH #* background[1] # WTF is this about?!
# if not sub_background:
# thresh += background[0]
self.logger.debug(
f'Detection is to be performed with weights? {conf.USE_DETECTION_WEIGHT}'
)
else:
thresh = conf.THRESH
convfilt = None
if conf.FILTER_KERNEL is not None:
dirname = os.path.dirname(__file__)
filename = os.path.join(
dirname, '../../config/conv_filters/' + conf.FILTER_KERNEL)
if os.path.exists(filename):
convfilt = np.array(
np.array(ascii.read(filename, data_start=1)).tolist())
else:
raise FileExistsError(
f"Convolution file at {filename} does not exist!")
if use_mask:
mask = mask
else:
mask = None
self.logger.debug(
f'Detection is to be performed with thresh = {thresh}')
# Set extraction pixel limit buffer
sep.set_extract_pixstack(conf.PIXSTACK_SIZE)
if sub_background:
self.logger.debug('Background will be subtracted.')
background = sep.Background(self.images[idx],
bw=conf.DETECT_BW,
bh=conf.DETECT_BH,
fw=conf.DETECT_FW,
fh=conf.DETECT_FH)
if conf.USE_FLAT:
image -= background.globalback
self.logger.debug(
f'Subtracted flat background level ({background.globalback:4.4f})'
)
else:
image -= background.back()
# var = np.ones_like(var)
kwargs = dict(var=var,
mask=mask,
minarea=conf.MINAREA,
filter_kernel=convfilt,
filter_type=conf.FILTER_TYPE,
segmentation_map=True,
clean=conf.CLEAN,
clean_param=conf.CLEAN_PARAM,
deblend_nthresh=conf.DEBLEND_NTHRESH,
deblend_cont=conf.DEBLEND_CONT)
catalog, segmap = sep.extract(image, thresh, **kwargs)
catalog['y'] -= 0.75 # HACK
catalog['x'] -= 0.1 #HACK
# plt.ion()
# fig, ax = plt.subplots(ncols=2, figsize=(40,20))
# from matplotlib.colors import LogNorm
# ax[0].imshow(image, norm=LogNorm(), vmin=1E-8, vmax=10, cmap='Greys')
# ax[0].scatter(catalog['x'], catalog['y'], s=1)
# ax[1].imshow(segmap)
# ax[1].scatter(catalog['x'], catalog['y'], s=1)
# # plt.show()
# fig.savefig(conf.PLOT_DIR + '/checksep.pdf')
# plt.close('all')
if len(catalog) != 0:
catalog = Table(catalog)
catalog.add_column(Column(catalog['x'], name='x_orig'))
catalog.add_column(Column(catalog['y'], name='y_orig'))
if self.wcs is not None:
wx = catalog[
'x_orig'] #+ self.mosaic_origin[1] - conf.BRICK_BUFFER
wy = catalog[
'y_orig'] #+ self.mosaic_origin[0] - conf.BRICK_BUFFER
wwx, wwy = wx, wy
# wwx, wwy = wx - self.mosaic_origin[0] + conf.BRICK_BUFFER, wy - self.mosaic_origin[1] + conf.BRICK_BUFFER
skyc = self.wcs.all_pix2world(wwx, wwy, 0)
# print(- self.mosaic_origin[0] + conf.BRICK_BUFFER, - self.mosaic_origin[1] + conf.BRICK_BUFFER)
catalog.add_column(
Column(skyc[0], name=f'RA_{conf.DETECTION_NICKNAME}'))
catalog.add_column(
Column(skyc[1], name=f'DEC_{conf.DETECTION_NICKNAME}'))
# Aperture Photometry
if incl_apphot:
flux = np.zeros(len(catalog),
dtype=(float, len(conf.APER_PHOT)))
flux_err = flux.copy()
flag = np.zeros_like(catalog['x'], dtype=bool)
for i, radius in enumerate(conf.APER_PHOT): # in arcsec
flux[:, i], flux_err[:, i], flag = sep.sum_circle(
image,
x=catalog['x'],
y=catalog['y'],
r=radius / conf.PIXEL_SCALE,
var=var)
mag = -2.5 * np.log10(flux) + conf.MODELING_ZPT
mag_err = 1.09 * flux_err / flux
catalog.add_column(Column(flux, name='flux_aper'))
catalog.add_column(Column(flux_err, name='fluxerr_aper'))
catalog.add_column(Column(mag, name='mag_aper'))
catalog.add_column(Column(mag_err, name='magerr_aper'))
catalog.add_column(Column(flag, name='flag_aper'))
self.catalog = catalog
self.n_sources = len(catalog)
self.segmap = segmap
return catalog, segmap
else:
raise ValueError('No objects found by SExtractor.')
(v0_2) - January-2019 Spec2SDSS_gri now incorporates the redshift (factor of (1+z)**5) to the wavelength specific intensities when generating the bandpass AB surface brightnesses. The redshift factor is now removed from intensity scaling step in this version.
(v0_3) - January-2019 Computes ra,dec from the source mask when determining image position. This avoids image registration offsets in row and column numbers for each band. Prepared for public release.
'''
import numpy as np
import os, sys, string, time
from scipy.interpolate import RectBivariateSpline
import scipy.ndimage
import warnings
from astropy.io import fits
from astropy.utils.exceptions import AstropyWarning
from astropy.convolution import Gaussian2DKernel
from astropy.convolution import convolve
from astropy.cosmology import FlatLambdaCDM
import sep
sep.set_extract_pixstack(9999999)
realsim_dir = os.path.dirname(os.path.abspath(__file__))
def rebin(array, dimensions=None, scale=None):
"""
Return the array 'array' to the new 'dimensions'
conserving the flux in the bins. The sum of the
array will remain the same as the original array.
Congrid from the scipy recipies does not generally
conserve surface brightness with reasonable accuracy.
As such, even accounting for the ratio of old and new
image areas, it does not conserve flux. This function
nicely solves the problem and more accurately
redistributes the flux in the output. This function
def do_stage(self, image):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise**2.0)**0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data,
self.threshold,
minarea=self.min_area,
err=error,
deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# We remove anything with a detection flag >= 8
# This includes memory overflows and objects that are too close the edge
sources = sources[sources['flag'] < 8]
sources = array_utils.prune_nans_from_table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * kronrad,
subpix=1,
err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Do circular aperture photometry for diameters of 1" to 6"
for diameter in [1, 2, 3, 4, 5, 6]:
flux, fluxerr, flag = sep.sum_circle(data,
sources['x'],
sources['y'],
diameter / 2.0 /
image.pixel_scale,
gain=1.0,
err=error)
sources['fluxaper{0}'.format(diameter)] = flux
sources['fluxerr{0}'.format(diameter)] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) *
(sources['a']**2.0 + sources['b']**2.0))**0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data,
sources['x'],
sources['y'],
6.0 * sources['a'],
[0.25, 0.5, 0.75],
normflux=sources['flux'],
subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'],
sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(),
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * sources['kronrad'],
subpix=1)
# masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (
2.5 * sources['kronrad']
)**2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[
background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
'flux', 'fluxerr', 'peak', 'fluxaper1',
'fluxerr1', 'fluxaper2', 'fluxerr2', 'fluxaper3',
'fluxerr3', 'fluxaper4', 'fluxerr4', 'fluxaper5',
'fluxerr5', 'fluxaper6', 'fluxerr6',
'background', 'fwhm', 'a', 'b', 'theta',
'kronrad', 'ellipticity', 'fluxrad25',
'fluxrad50', 'fluxrad75', 'x2', 'y2', 'xy',
'flag']
# Add the units and description to the catalogs
catalog['x'].unit = 'pixel'
catalog['x'].description = 'X coordinate of the object'
catalog['y'].unit = 'pixel'
catalog['y'].description = 'Y coordinate of the object'
catalog['xwin'].unit = 'pixel'
catalog['xwin'].description = 'Windowed X coordinate of the object'
catalog['ywin'].unit = 'pixel'
catalog['ywin'].description = 'Windowed Y coordinate of the object'
catalog['xpeak'].unit = 'pixel'
catalog['xpeak'].description = 'X coordinate of the peak'
catalog['ypeak'].unit = 'pixel'
catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
catalog['flux'].unit = 'count'
catalog[
'flux'].description = 'Flux within a Kron-like elliptical aperture'
catalog['fluxerr'].unit = 'count'
catalog[
'fluxerr'].description = 'Error on the flux within Kron aperture'
catalog['peak'].unit = 'count'
catalog['peak'].description = 'Peak flux (flux at xpeak, ypeak)'
for diameter in [1, 2, 3, 4, 5, 6]:
catalog['fluxaper{0}'.format(diameter)].unit = 'count'
catalog['fluxaper{0}'.format(
diameter
)].description = 'Flux from fixed circular aperture: {0}" diameter'.format(
diameter)
catalog['fluxerr{0}'.format(diameter)].unit = 'count'
catalog['fluxerr{0}'.format(
diameter
)].description = 'Error on Flux from circular aperture: {0}"'.format(
diameter)
catalog['background'].unit = 'count'
catalog[
'background'].description = 'Average background value in the aperture'
catalog['fwhm'].unit = 'pixel'
catalog['fwhm'].description = 'FWHM of the object'
catalog['a'].unit = 'pixel'
catalog['a'].description = 'Semi-major axis of the object'
catalog['b'].unit = 'pixel'
catalog['b'].description = 'Semi-minor axis of the object'
catalog['theta'].unit = 'degree'
catalog['theta'].description = 'Position angle of the object'
catalog['kronrad'].unit = 'pixel'
catalog['kronrad'].description = 'Kron radius used for extraction'
catalog['ellipticity'].description = 'Ellipticity'
catalog['fluxrad25'].unit = 'pixel'
catalog[
'fluxrad25'].description = 'Radius containing 25% of the flux'
catalog['fluxrad50'].unit = 'pixel'
catalog[
'fluxrad50'].description = 'Radius containing 50% of the flux'
catalog['fluxrad75'].unit = 'pixel'
catalog[
'fluxrad75'].description = 'Radius containing 75% of the flux'
catalog['x2'].unit = 'pixel^2'
catalog[
'x2'].description = 'Variance on X coordinate of the object'
catalog['y2'].unit = 'pixel^2'
catalog[
'y2'].description = 'Variance on Y coordinate of the object'
catalog['xy'].unit = 'pixel^2'
catalog['xy'].description = 'XY covariance of the object'
catalog[
'flag'].description = 'Bit mask of extraction/photometry flags'
catalog.sort('flux')
catalog.reverse()
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (
mean_background,
'[counts] Sigma clipped mean of frame background')
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (median_background,
'[counts] Median of frame background')
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (
std_background, '[counts] Robust std dev of frame background')
# Save some image statistics to the header
good_objects = catalog['flag'] == 0
for quantity in ['fwhm', 'ellipticity', 'theta']:
good_objects = np.logical_and(
good_objects, np.logical_not(np.isnan(catalog[quantity])))
if good_objects.sum() == 0:
image.header['L1FWHM'] = ('NaN',
'[arcsec] Frame FWHM in arcsec')
image.header['L1ELLIP'] = ('NaN',
'Mean image ellipticity (1-B/A)')
image.header['L1ELLIPA'] = (
'NaN', '[deg] PA of mean image ellipticity')
else:
seeing = np.median(
catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing,
'[arcsec] Frame FWHM in arcsec')
mean_ellipticity = stats.sigma_clipped_mean(
catalog['ellipticity'][good_objects], 3.0)
image.header['L1ELLIP'] = (mean_ellipticity,
'Mean image ellipticity (1-B/A)')
mean_position_angle = stats.sigma_clipped_mean(
catalog['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (
mean_position_angle, '[deg] PA of mean image ellipticity')
logging_tags = {
key: float(image.header[key])
for key in [
'L1MEAN', 'L1MEDIAN', 'L1SIGMA', 'L1FWHM', 'L1ELLIP',
'L1ELLIPA'
]
}
logger.info('Extracted sources',
image=image,
extra_tags=logging_tags)
# adding catalog (a data table) to the appropriate images attribute.
image.data_tables['catalog'] = DataTable(data_table=catalog,
name='CAT')
except Exception:
logger.error(logs.format_exception(), image=image)
return image
def best_sources(self):
"""Property, a dictionary of best sources detected in the image.
Keys are:
fitshape: tuple, the size of the stamps on each source detected
sources: a table of sources, with the imformation from sep
positions: an array, with the position of each source stamp
n_sources: the total number of sources extracted
"""
if not hasattr(self, '_best_sources'):
# print('looking for srcs')
try:
srcs = sep.extract(self.bkg_sub_img.data,
thresh=8 * self.__bkg.globalrms,
mask=self.mask,
minarea=9)
except Exception:
try:
sep.set_extract_pixstack(3600000)
srcs = sep.extract(self.bkg_sub_img.data,
thresh=35 * self.__bkg.globalrms,
mask=self.mask,
minarea=9)
except Exception:
raise
if len(srcs) < self.min_sources:
print("""found {} sources, looking for at least {}.
Trying again""".format(len(srcs), self.min_sources))
old_srcs = srcs
try:
srcs = sep.extract(self.bkg_sub_img.data,
thresh=3 * self.__bkg.globalrms,
mask=self.mask,
minarea=5)
except Exception:
sep.set_extract_pixstack(900000)
srcs = sep.extract(self.bkg_sub_img.data,
thresh=3 * self.__bkg.globalrms,
mask=self.mask,
minarea=9)
if len(old_srcs) > len(srcs):
srcs = old_srcs
if len(srcs) == 0:
raise ValueError('Few sources detected on image')
elif len(srcs) == 1:
m, med, st = sigma_clipped_stats(
self.bkg_sub_img.data.flatten()) # noqa
if st <= 0.1:
raise ValueError('Image is constant, possible saturated')
if m >= 65535.:
raise ValueError('Image is saturated')
else:
raise ValueError('only one sources. Possible saturation')
p_sizes = np.percentile(srcs['npix'], q=[20, 50, 80])
best_big = srcs['npix'] >= p_sizes[0]
best_small = srcs['npix'] <= p_sizes[2]
best_flag = srcs['flag'] == 0
fluxes_quartiles = np.percentile(srcs['flux'], q=[15, 85])
low_flux = srcs['flux'] >= fluxes_quartiles[0]
hig_flux = srcs['flux'] <= fluxes_quartiles[1]
best_srcs = srcs[best_big & best_flag & best_small & hig_flux
& low_flux]
if len(best_srcs) == 0:
print('Best sources are too few- Using everything we have!')
best_srcs = srcs
# raise ValueError('Few sources detected on image')
if len(best_srcs) > 1800:
jj = np.random.choice(len(best_srcs), 1800, replace=False)
best_srcs = best_srcs[jj]
print(('Sources found = {}'.format(len(best_srcs))))
self._best_sources = best_srcs
return self._best_sources
def best_sources(self):
"""Property, a dictionary of best sources detected in the image.
Keys are:
fitshape: tuple, the size of the stamps on each source detected
sources: a table of sources, with the imformation from sep
positions: an array, with the position of each source stamp
n_sources: the total number of sources extracted
"""
if not hasattr(self, "_best_sources"):
try:
srcs = sep.extract(
self.bkg_sub_img.data,
thresh=8 * self.__bkg.globalrms,
mask=self.mask,
minarea=9,
)
except Exception:
try:
sep.set_extract_pixstack(3600000)
srcs = sep.extract(
self.bkg_sub_img.data,
thresh=35 * self.__bkg.globalrms,
mask=self.mask,
minarea=9,
)
except Exception:
raise
if len(srcs) < self.min_sources:
old_srcs = srcs
try:
srcs = sep.extract(
self.bkg_sub_img.data,
thresh=3 * self.__bkg.globalrms,
mask=self.mask,
minarea=5,
)
except Exception:
sep.set_extract_pixstack(900000)
srcs = sep.extract(
self.bkg_sub_img.data,
thresh=3 * self.__bkg.globalrms,
mask=self.mask,
minarea=9,
)
if len(old_srcs) > len(srcs):
srcs = old_srcs
if len(srcs) == 0:
raise ValueError("Few sources detected on image")
elif len(srcs) == 1:
m, med, st = sigma_clipped_stats(
self.bkg_sub_img.data.flatten()) # noqa
if st <= 0.1:
raise ValueError("Image is constant, possible saturated")
if m >= 65535.0:
raise ValueError("Image is saturated")
else:
raise ValueError("Only one source. Possible saturation")
p_sizes = np.percentile(srcs["npix"], q=[20, 50, 80])
best_big = srcs["npix"] >= p_sizes[0]
best_small = srcs["npix"] <= p_sizes[2]
best_flag = srcs["flag"] == 0
fluxes_quartiles = np.percentile(srcs["flux"], q=[15, 85])
low_flux = srcs["flux"] >= fluxes_quartiles[0]
hig_flux = srcs["flux"] <= fluxes_quartiles[1]
best_srcs = srcs[best_big & best_flag & best_small & hig_flux
& low_flux]
if len(best_srcs) == 0:
self.warning(
"Best sources are too few- Using everything we have!")
best_srcs = srcs
if len(best_srcs) > 1800:
jj = np.random.choice(len(best_srcs), 1800, replace=False)
best_srcs = best_srcs[jj]
self._best_sources = best_srcs
return self._best_sources
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import numpy as np
import sep
from astropy.io import fits
from astropy.io import ascii
from astroML import crossmatch as cx
import pandas as pd
sep.set_extract_pixstack(900000)
data_path = os.path.abspath('/home/bos0109/sersic/work/rhino/data/extract_test/stellar/CSTAR/')
images_path = os.path.join(data_path, 'images')
master_Ryan = os.path.join(images_path, 'master10_wcs.fits')
cat_path = os.path.join(data_path, 'cats/stars.dat')
cat = ascii.read(cat_path, names = ['cstarid', 'x', 'y', 'imag'])
data = fits.getdata(master_Ryan)
data = data.byteswap().newbyteorder()
bkg = sep.Background(data)
data = data - bkg
def go_test(cat, image_data, thresh):
threshold = thresh*bkg.globalrms
#make the extraction
sources = sep.extract(image_data, threshold)
cat.to_pandas()
sources = pd.DataFrame(sources)
def detect_blobs(image, mask, threshold, min_area, deblend_nthresh=500,
deblend_cont=0):
'''Detects blobs in `image` using an implementation of
`SExtractor <http://www.astromatic.net/software/sextractor>`_ [1].
Parameters
----------
image: numpy.ndarray[Union[numpy.uint8, numpy.uint16]]
grayscale image in which blobs should be detected
mask: numpy.ndarray[numpy.bool]
binary image that masks pixel regions in which no blobs should be
detected
threshold: int, optional
factor by which pixel values must be above background
to be considered part of a blob (default: ``5``)
min_area: int, optional
minimal size of a blob
deblend_ntresh: int, optional
number of deblending thresholds (default: ``500``)
deblend_cont: int, optional
minimum contrast ratio for deblending (default: ``0``)
Returns
-------
Tuple[numpy.ndarray[numpy.int32]]
detected blobs and the corresponding centroids
References
----------
.. [1] Bertin, E. & Arnouts, S. 1996: SExtractor: Software for source
extraction, Astronomy & Astrophysics Supplement 317, 393
'''
sep.set_extract_pixstack(10**7)
img = image.astype('float')
# We pad the image with mirrored pixels to prevent border artifacts.
pad = 50
left = img[:, 1:pad]
right = img[:, -pad:-1]
detect_img = np.c_[np.fliplr(left), img, np.fliplr(right)]
upper = detect_img[1:pad, :]
lower = detect_img[-pad:-1, :]
detect_img = np.r_[np.flipud(upper), detect_img, np.flipud(lower)]
logger.info('detect blobs via thresholding and deblending')
detection, blobs = sep.extract(
detect_img, threshold,
minarea=min_area, segmentation_map=True,
deblend_nthresh=deblend_nthresh, deblend_cont=deblend_cont,
filter_kernel=None, clean=False
)
centroids = np.zeros(detect_img.shape, dtype=np.int32)
y = detection['y'].astype(int)
x = detection['x'].astype(int)
# WTF? In rare cases object coorindates lie outside of the image.
n = len(detection)
y[y > detect_img.shape[0]] = detect_img.shape[0]
x[x > detect_img.shape[1]] = detect_img.shape[1]
centroids[y, x] = np.arange(1, n + 1)
# Remove the padded border pixels
blobs = blobs[pad-1:-(pad-1), pad-1:-(pad-1)].copy()
centroids = centroids[pad-1:-(pad-1), pad-1:-(pad-1)].copy()
# Blobs detected outside of regions of interest are discarded.
blobs[mask > 0] = 0
blobs[mh.bwperim(np.invert(mask)) > 0] = 0
mh.labeled.relabel(blobs, inplace=True)
# We need to ensure that centroids are labeled the same way as blobs.
centroids[centroids > 0] = blobs[centroids > 0]
return (blobs, centroids)
def get_objects_sep(image, header=None, mask=None, thresh=4.0, aper=3.0, bkgann=None, r0=0.5, gain=1, edge=0, minnthresh=2, minarea=5, relfluxradius=2.0, wcs=None, use_fwhm=False, use_mask_bg=False, use_mask_large=False, subtract_bg=True, npix_large=300, sn=10.0, verbose=True, get_fwhm=False, **kwargs):
if r0 > 0.0:
kernel = make_kernel(r0)
else:
kernel = None
if verbose:
print("Preparing background mask")
if mask is None:
mask = np.zeros_like(image, dtype=np.bool)
mask_bg = np.zeros_like(mask)
mask_segm = np.zeros_like(mask)
if use_mask_bg:
# Simple heuristics to mask regions with rapidly varying background
if verbose:
print("Masking rapidly changing background")
bg2 = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64)
for _ in xrange(3):
bg1 = sep.Background(image, mask=mask|mask_bg, bw=256, bh=256)
ibg = bg2.back() - bg1.back()
tmp = np.abs(ibg - np.median(ibg)) > 5.0*mad_std(ibg)
mask_bg |= dilate(tmp, np.ones([100, 100]))
mask_bg = dilate(tmp, np.ones([100, 100]))
if verbose:
print("Building background map")
bg = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64)
if subtract_bg:
image1 = image - bg.back()
else:
image1 = image.copy()
sep.set_extract_pixstack(image.shape[0]*image.shape[1])
if use_mask_large:
# Mask regions around huge objects as they are most probably corrupted by saturation and blooming
if verbose:
print("Extracting initial objects")
obj0,segm = sep.extract(image1, err=bg.rms(), thresh=thresh, minarea=minarea, mask=mask|mask_bg, filter_kernel=kernel, segmentation_map=True, **kwargs)
if verbose:
print("Dilating large objects")
mask_segm = np.isin(segm, [_+1 for _,npix in enumerate(obj0['npix']) if npix > npix_large])
mask_segm = dilate(mask_segm, np.ones([50, 50]))
if verbose:
print("Extracting final objects")
obj0 = sep.extract(image1, err=bg.rms(), thresh=thresh, minarea=minarea, mask=mask|mask_bg|mask_segm, filter_kernel=kernel, **kwargs)
if use_fwhm:
# Estimate FHWM and use it to get optimal aperture size
idx = obj0['flag'] == 0
fwhm = 2.0*np.sqrt(np.hypot(obj0['a'][idx], obj0['b'][idx])*np.log(2))
fwhm = 2.0*sep.flux_radius(image1, obj0['x'][idx], obj0['y'][idx], relfluxradius*fwhm*np.ones_like(obj0['x'][idx]), 0.5, mask=mask)[0]
fwhm = np.median(fwhm)
aper = max(1.5*fwhm, aper)
if verbose:
print("FWHM = %.2g, aperture = %.2g" % (fwhm, aper))
# Windowed positional parameters are often biased in crowded fields, let's avoid them for now
# xwin,ywin,flag = sep.winpos(image1, obj0['x'], obj0['y'], 0.5, mask=mask)
xwin,ywin = obj0['x'], obj0['y']
# Filter out objects too close to frame edges
idx = (np.round(xwin) > edge) & (np.round(ywin) > edge) & (np.round(xwin) < image.shape[1]-edge) & (np.round(ywin) < image.shape[0]-edge) # & (obj0['flag'] == 0)
if minnthresh:
idx &= (obj0['tnpix'] >= minnthresh)
if verbose:
print("Measuring final objects")
flux,fluxerr,flag = sep.sum_circle(image1, xwin[idx], ywin[idx], aper, err=bg.rms(), gain=gain, mask=mask|mask_bg|mask_segm, bkgann=bkgann)
# For debug purposes, let's make also the same aperture photometry on the background map
bgflux,bgfluxerr,bgflag = sep.sum_circle(bg.back(), xwin[idx], ywin[idx], aper, err=bg.rms(), gain=gain, mask=mask|mask_bg|mask_segm)
bg = bgflux/np.pi/aper**2
# Fluxes to magnitudes
mag,magerr = np.zeros_like(flux), np.zeros_like(flux)
mag[flux>0] = -2.5*np.log10(flux[flux>0])
# magerr[flux>0] = 2.5*np.log10(1.0 + fluxerr[flux>0]/flux[flux>0])
magerr[flux>0] = 2.5/np.log(10)*fluxerr[flux>0]/flux[flux>0]
# better FWHM estimation - FWHM=HFD for Gaussian
if get_fwhm:
fwhm = 2.0*sep.flux_radius(image1, xwin[idx], ywin[idx], relfluxradius*aper*np.ones_like(xwin[idx]), 0.5, mask=mask)[0]
else:
fwhm = np.zeros_like(xwin[idx])
flag |= obj0['flag'][idx]
# Quality cuts
fidx = (flux > 0) & (magerr < 1.0/sn)
if wcs is None and header is not None:
# If header is provided, we may build WCS from it
wcs = WCS(header)
if wcs is not None:
# If WCS is provided we may convert x,y to ra,dec
ra,dec = wcs.all_pix2world(obj0['x'][idx], obj0['y'][idx], 0)
else:
ra,dec = np.zeros_like(obj0['x'][idx]),np.zeros_like(obj0['y'][idx])
if verbose:
print("All done")
return {'x':xwin[idx][fidx], 'y':ywin[idx][fidx], 'flux':flux[fidx], 'fluxerr':fluxerr[fidx], 'mag':mag[fidx], 'magerr':magerr[fidx], 'flags':obj0['flag'][idx][fidx]|flag[fidx], 'ra':ra[fidx], 'dec':dec[fidx], 'bg':bg[fidx], 'fwhm':fwhm[fidx], 'aper':aper, 'bkgann':bkgann, 'a':obj0['a'][idx][fidx], 'b':obj0['b'][idx][fidx], 'theta':obj0['theta'][idx][fidx]}
s = raw_input('type glob pattern for file list: ')
flist = glob.glob(s)
print flist
#creates an empty file to fill with opened input data
inptarray = []
#Open the images given at the command line and place them in an array
for i in range(len(flist)):
step1 = flist[i]
step2 = fits.open(step1)
#align1 = step2[0].data
data_sub = step2[0].data
####objects = sep.extract(data_sub, 1.5, err=bkg.globalrms)
sep.set_extract_pixstack(5000000)
data_sub = data_sub.byteswap().newbyteorder()
objects = sep.extract(data_sub, 2000)
# how many objects were detected
len(objects)
# plot background-subtracted image
fig, ax = plt.subplots()
m, s = np.mean(data_sub), np.std(data_sub)
im = ax.imshow(data_sub,
interpolation='nearest',
cmap='gray',
vmin=m - s,
vmax=m + s,
origin='lower')
def extract_features(self, subregions, mask=None):
"""Extract image features for each subregion. Image should be cropped
and masked.
:param subregions: subregions to be used
:param mask: mask to be applied in source extraction, optional
:return: None
"""
# set internal pixel buffer
sep.set_extract_pixstack(10000000)
# extract time from header and derive frame properties
try:
time = Time(self.header['DATE-OBS'], format='isot')
features = OrderedDict([
('time', time.isot),
('filename', self.filename.split(os.path.sep)[-1]),
('moon_alt', observatory.moon_altaz(time).alt.deg),
('sun_alt', observatory.sun_altaz(time).alt.deg),
('moon_phase', 1-observatory.moon_phase(time).value/np.pi),
])
except KeyError as e:
conf.logger.error('missing time data in file {}: {}.'.format(
self.filename, e))
return False
# derive and subtract sky background
bkg = sep.Background(self.data.astype(np.float64),
bw=conf.SEP_BKGBOXSIZE, bh=conf.SEP_BKGBOXSIZE,
fw=conf.SEP_BKGXRANGE, fh=conf.SEP_BKGYRANGE)
data_sub = self.data - bkg.back()
# if mask is provided, it is applied in the proper derivation of
# source brightness thresholds
if mask is not None:
threshold = (np.ma.median(np.ma.array(data_sub,
mask=(1-mask))) +
np.median(bkg.rms())*conf.SEP_SIGMA)
src = sep.extract(data_sub, threshold, minarea=conf.SEP_MINAREA,
mask=(1-mask),
deblend_nthresh=conf.SEP_DEBLENDN,
deblend_cont=conf.SEP_DEBLENDV)
else:
threshold = (np.median(data_sub) +
np.median(bkg.rms())*conf.SEP_SIGMA)
src = sep.extract(data_sub, threshold, minarea=conf.SEP_MINAREA,
mask=mask,
deblend_nthresh=conf.SEP_DEBLENDN,
deblend_cont=conf.SEP_DEBLENDV)
# apply max_flag cutoff (reject flawed sources)
src = src[src['flag'] <= conf.SEP_MAXFLAG]
# feature extraction per subregion
features['srcdens'] = []
features['bkgmedian'] = []
features['bkgmean'] = []
features['bkgstd'] = []
for i, sub in enumerate(subregions):
features['srcdens'].append(len(
src[sub[src['y'].astype(np.int),
src['x'].astype(np.int)]])/np.sum(sub[mask== 1]))
features['bkgmedian'].append(np.median(bkg.back()[sub]))
features['bkgmean'].append(np.mean(bkg.back()[sub]))
features['bkgstd'].append(np.std(bkg.back()[sub]))
self.subregions = subregions
self.features = features
def do_stage(self, images):
for i, image in enumerate(images):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise ** 2.0) ** 0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data, self.threshold, minarea=self.min_area,
err=error, deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data, sources['x'], sources['y'],
sources['a'], sources['b'],
np.pi / 2.0, 2.5 * kronrad,
subpix=1, err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) * (sources['a'] ** 2.0 + sources['b'] ** 2.0)) ** 0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data, sources['x'], sources['y'],
6.0 * sources['a'], [0.25, 0.5, 0.75],
normflux=sources['flux'], subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'], sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(), sources['x'], sources['y'],
sources['a'], sources['b'], np.pi / 2.0,
2.5 * sources['kronrad'], subpix=1)
#masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (2.5 * sources['kronrad']) ** 2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
'flux', 'fluxerr', 'background', 'fwhm',
'a', 'b', 'theta', 'kronrad', 'ellipticity',
'fluxrad25', 'fluxrad50', 'fluxrad75',
'x2', 'y2', 'xy', 'flag']
# Add the units and description to the catalogs
image.catalog['x'].unit = 'pixel'
image.catalog['x'].description = 'X coordinate of the object'
image.catalog['y'].unit = 'pixel'
image.catalog['y'].description = 'Y coordinate of the object'
image.catalog['xwin'].unit = 'pixel'
image.catalog['xwin'].description = 'Windowed X coordinate of the object'
image.catalog['ywin'].unit = 'pixel'
image.catalog['ywin'].description = 'Windowed Y coordinate of the object'
image.catalog['xpeak'].unit = 'pixel'
image.catalog['xpeak'].description = 'X coordinate of the peak'
image.catalog['ypeak'].unit = 'pixel'
image.catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
image.catalog['flux'].unit = 'counts'
image.catalog['flux'].description = 'Flux within a Kron-like elliptical aperture'
image.catalog['fluxerr'].unit = 'counts'
image.catalog['fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
image.catalog['background'].unit = 'counts'
image.catalog['background'].description = 'Average background value in the aperture'
image.catalog['fwhm'].unit = 'pixel'
image.catalog['fwhm'].description = 'FWHM of the object'
image.catalog['a'].unit = 'pixel'
image.catalog['a'].description = 'Semi-major axis of the object'
image.catalog['b'].unit = 'pixel'
image.catalog['b'].description = 'Semi-minor axis of the object'
image.catalog['theta'].unit = 'degrees'
image.catalog['theta'].description = 'Position angle of the object'
image.catalog['kronrad'].unit = 'pixel'
image.catalog['kronrad'].description = 'Kron radius used for extraction'
image.catalog['ellipticity'].description = 'Ellipticity'
image.catalog['fluxrad25'].unit = 'pixel'
image.catalog['fluxrad25'].description = 'Radius containing 25% of the flux'
image.catalog['fluxrad50'].unit = 'pixel'
image.catalog['fluxrad50'].description = 'Radius containing 50% of the flux'
image.catalog['fluxrad75'].unit = 'pixel'
image.catalog['fluxrad75'].description = 'Radius containing 75% of the flux'
image.catalog['x2'].unit = 'pixel^2'
image.catalog['x2'].description = 'Variance on X coordinate of the object'
image.catalog['y2'].unit = 'pixel^2'
image.catalog['y2'].description = 'Variance on Y coordinate of the object'
image.catalog['xy'].unit = 'pixel^2'
image.catalog['xy'].description = 'XY covariance of the object'
image.catalog['flag'].description = 'Bit mask combination of extraction and photometry flags'
image.catalog.sort('flux')
image.catalog.reverse()
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (mean_background,
'[counts] Sigma clipped mean of frame background')
logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (median_background,
'[counts] Median of frame background')
logs.add_tag(logging_tags, 'L1MEDIAN', float(median_background))
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (std_background,
'[counts] Robust std dev of frame background')
logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))
# Save some image statistics to the header
good_objects = image.catalog['flag'] == 0
seeing = np.median(image.catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing, '[arcsec] Frame FWHM in arcsec')
logs.add_tag(logging_tags, 'L1FWHM', float(seeing))
mean_ellipticity = stats.sigma_clipped_mean(sources['ellipticity'][good_objects],
3.0)
image.header['L1ELLIP'] = (mean_ellipticity, 'Mean image ellipticity (1-B/A)')
logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))
mean_position_angle = stats.sigma_clipped_mean(sources['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (mean_position_angle,
'[deg] PA of mean image ellipticity')
logs.add_tag(logging_tags, 'L1ELLIPA', float(mean_position_angle))
self.logger.info('Extracted sources', extra=logging_tags)
except Exception as e:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
self.logger.error(e, extra=logging_tags)
return images
def make_catalog(data, header):
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(data.shape[1] * data.shape[0] * 0.05))
data = data.copy()
error = (np.abs(data) + header['RDNOISE']**2.0)**0.5
mask = data > 0.9 * header['SATURATE']
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# Do an initial source detection
sources = sep.extract(data,
THRESHOLD,
mask=mask,
minarea=MIN_AREA,
err=error,
deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# We remove anything with a detection flag >= 8
# This includes memory overflows and objects that are too close the edge
sources = sources[sources['flag'] < 8]
sources = prune_nans_from_table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * kronrad,
subpix=1,
err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) * (sources['a']**2.0 + sources['b']**2.0))**0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['theta'] = np.degrees(sources['theta'])
return save_catalog_meta_data(sources)