def objective(threshold, fwhm, sigma_radius, roundlo, roundhi, sharplo,
sharphi):
res_table = DAOStarFinder(threshold=median + std * threshold,
fwhm=fwhm,
sigma_radius=sigma_radius,
sharplo=sharplo,
sharphi=sharphi,
roundlo=roundlo,
roundhi=roundhi,
exclude_border=True)(img)
if not res_table:
return 3000
xys = structured_to_unstructured(
np.array(res_table['xcentroid', 'ycentroid']))
seen_indices = set()
offsets = []
for xy in xys:
dist, index = lookup_tree.query(xy)
if dist > 2 or index in seen_indices:
offsets.append(np.nan)
else:
offsets.append(dist)
seen_indices.add(index)
offsets += [np.nan] * len(seen_indices - set(lookup_tree.indices))
offsets += [np.nan] * abs(len(ref_table) - len(res_table))
offsets = np.array(offsets)
offsets -= np.nanmean(offsets)
offsets[np.isnan(offsets)] = 3.
return np.sqrt(np.sum(np.array(offsets)**2))
def locateStarsInImage(imageArray):
mean, median, std = sigma_clipped_stats(imageArray, sigma=sigmaParameter)
daofind = DAOStarFinder(fwhm=fwhmParameter,
threshold=thresholdParameter * std)
sources = daofind(imageArray - median)
return sources
def Seeing(data, r=11):
from astropy.stats import sigma_clipped_stats
from photutils.detection import DAOStarFinder
from astropy.nddata.utils import Cutout2D
# Source detection
mean, median, std = sigma_clipped_stats(data, sigma=3)
starfind = DAOStarFinder(fwhm=4.0,
threshold=5. * std,
exclude_border=True,
sky=median)
sources = starfind(data - median)
x = sources['xcentroid']
y = sources['ycentroid']
# FWHM over all the detected sources
fwhm = []
for i in range(len(x)):
cut = Cutout2D(data, [x[i], y[i]], r, mode='partial')
sec = cut.data
xc, yc = cut.to_cutout_position([x[i], y[i]])
fwhm.append(FWHM(sec, xc, yc))
return np.array(fwhm)
def getnewtargetlist(imagedata):
_logger.info("Redoing target list")
mean, median, std = sigma_clipped_stats(imagedata[200:-200, 200:-200],
sigma=3.0)
daofind = DAOStarFinder(fwhm=3, threshold=5. * std)
sources = daofind(imagedata - median)
retTables = Table(
[sources['xcentroid'], sources['ycentroid'], sources['flux']],
names=['x', 'y', 'FLUX'])
return retTables
def daodetect(image,nsigma=1.5,fwhm=3.0):
""" Detection with DAOFinder."""
threshold = np.median(image.error)*nsigma
daofind = DAOStarFinder(fwhm=fwhm, threshold=threshold, sky=0.0)
objects = daofind(image.data-image.sky, mask=image.mask)
# homogenize the columns
objects['xcentroid'].name = 'x'
objects['ycentroid'].name = 'y'
return objects
def bar():
# TODO fails with friggin index error in model again:
config = Config()
image, input_table = read_or_generate_image('gauss_cluster_N1000')
mean, median, std = sigma_clipped_stats(image, sigma=config.clip_sigma)
star_guesses = make_stars_guess(image,
DAOStarFinder(median * 3, 4.),
cutout_size=51)
config.epsf_guess = make_epsf_combine(star_guesses)
config.epsfbuilder_iters = 4
def astrometry_extract(data,
fwhm=3.,
ksigma=5.,
csigma=3.,
indexing=1,
bintable=True):
''' Extracts the star positions (0-indexing), but not extended ones.
Note
----
This is just a convenience function for DAOStarFinder. First used for the
astrometry client. This is why the xy positions are sorted by flux.
Parameters
----------
data: ndarray
The array containing the pixel values
fwhm: float
The estimated FWHM of stellar objects in the image.
ksigma, csigma: float
The threshold for the detection will be calculated by median plus
``ksigma`` times standard deviation, where the median and standard
deviation is calculated from the ``csigma``-sigma clipping on to the
original image (``data``).
indexing: int, float
Whether to use 0 or 1 indexing. The user may use any floating number
for their own indexing, although 0 or 1 is the most usual case.
bintable: bool
Whether to convert to FITS BINTABLE format. This is required for astrometry.net
Example
-------
>>> xy = extracter(orig.data, fwhm=4, ksigma=5, csigma=3) + 1 # 1-indexing
>>> np.savetxt(srcpath, xy, fmt='%d')
>>> plt.plot(*xy.T, 'rx', ms=10)
'''
avg, med, std = sigma_clipped_stats(data, sigma=csigma, iters=1)
finder = DAOStarFinder(fwhm=fwhm,
threshold=med + ksigma * std,
exclude_border=True)
sources = finder(data)
sources.sort(["flux"])
xy = np.vstack((sources["xcentroid"].round().astype(int).tolist(),
sources["ycentroid"].round().astype(int).tolist())).T
xy += indexing
if bintable:
x = fits.Column(name='x', format='Iw', array=xy[:, 0])
y = fits.Column(name='y', format='Iw', array=xy[:, 1])
xy = fits.BinTableHDU.from_columns([x, y])
return xy
return xy
def photutils_daostarfinder(img, thresholds, fwhms):
mask = np.zeros_like(img, dtype=np.bool)
results = []
for t, f in zip(thresholds, fwhms):
dao = DAOStarFinder(threshold=t, fwhm=f)
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore', category=NoDetectionsWarning
)
dao_dataframe = dao.find_stars(img, mask=mask)
dao_dataframe = dao.find_stars(img, mask=mask)
if dao_dataframe is not None:
dao_dataframe = dao_dataframe.to_pandas()
results += [dao_dataframe]
_mask = np.zeros_like(img, dtype=np.bool)
y_limit, x_limit = np.array(img.shape) - 1
spot_mask_positions = (dao_dataframe
.loc[:, ['xcentroid', 'ycentroid']]
.transform(np.round).astype('int')
.query('({y} >= 0) & ({y} <= @y_limit)'.format(y='ycentroid'))
.query('({x} >= 0) & ({x} <= @x_limit)'.format(x='xcentroid'))
)
_mask[
spot_mask_positions.ycentroid,
spot_mask_positions.xcentroid
] = True
# fwhm ~= 2.355 sigma
mask += binary_dilation(_mask, selem=disk(2*f))
if len(results) > 0:
return pd.concat(results)
else:
return pd.DataFrame(columns=['xcentroid', 'ycentroid'])
def get_finder(image: np.ndarray, config: Config) -> photutils.StarFinderBase:
"""construct a StarFinder from a given configuration and image(needed for threshold)"""
mean, median, std = sigma_clipped_stats(image, sigma=config.clip_sigma)
threshold = median + config.threshold_factor * std
finder = DAOStarFinder(threshold=threshold,
fwhm=config.fwhm_guess,
sigma_radius=config.sigma_radius,
sharplo=config.sharplo,
sharphi=config.sharphi,
roundlo=config.roundlo,
roundhi=config.roundhi,
exclude_border=config.exclude_border)
return finder
def psfphotometry(imagefile,
ra=None,
dec=None,
x=None,
y=None,
fwhm=5.0,
zp=0.0,
gain=1.0,
doDifferential=False,
xfield=None,
yfield=None,
xfirst=None,
yfirst=None):
hdulist = fits.open(imagefile)
header = fits.getheader(imagefile)
if x == None:
w = WCS(header)
x0, y0 = w.wcs_world2pix(ra, dec, 1)
gain = 1.0
else:
x0, y0 = x, y
if len(hdulist) > 3:
image = hdulist[1].data
elif len(hdulist) == 2:
image = hdulist[0].data
else:
image = hdulist[0].data
image_shape = image.shape
#daogroup = DAOGroup(crit_separation=8)
daogroup = DAOGroup(crit_separation=25)
mmm_bkg = MMMBackground()
#iraffind = IRAFStarFinder(threshold=2.0*mmm_bkg(image),
# fwhm=4.0)
fitter = LevMarLSQFitter()
gaussian_prf = IntegratedGaussianPRF(flux=1, sigma=1.7)
gaussian_prf.sigma.fixed = False
gaussian_prf.flux.fixed = False
psffile = imagefile.replace(".fits", ".psf")
fid = open(psffile, 'w')
if len(image_shape) == 3:
nhdu, xshape, yshape = image.shape
dateobs = utcparser(hdulist[0].header["UTCSTART"])
mjd = dateobs.mjd
if "KINCYCTI" in hdulist[0].header:
mjdall = mjd + np.arange(
nhdu) * hdulist[0].header["KINCYCTI"] / 86400.0
else:
mjdall = mjd + np.arange(
nhdu) * hdulist[0].header["EXPTIME"] / 86400.0
mjds, mags, magerrs, fluxes, fluxerrs = [], [], [], [], []
for jj in range(nhdu):
if np.mod(jj, 10) == 0:
print('PSF fitting: %d/%d' % (jj, nhdu))
image = hdulist[0].data[jj, :, :]
mjd = mjdall[jj]
n, median, std = sigma_clipped_stats(image, sigma=3.0)
daofind = DAOStarFinder(fwhm=2.0, threshold=2. * std)
#phot_obj = IterativelySubtractedPSFPhotometry(finder=daofind,
# group_maker=daogroup,
# bkg_estimator=mmm_bkg,
# psf_model=gaussian_prf,
# fitter=fitter,
# fitshape=(21, 21),
# niters=10)
image = image - np.median(image)
image_slice = np.zeros(image.shape)
slsize = 25
xmin = np.max([0, int(x0 - slsize)])
xmax = np.min([int(x0 + slsize), image.shape[0]])
ymin = np.max([0, int(y0 - slsize)])
ymax = np.min([int(y0 + slsize), image.shape[1]])
image_slice[ymin:ymax, xmin:xmax] = 1
if doDifferential:
xmin_f = np.max([0, int(xfield - slsize)])
xmax_f = np.min([int(xfield + slsize), image.shape[0]])
ymin_f = np.max([0, int(yfield - slsize)])
ymax_f = np.min([int(yfield + slsize), image.shape[1]])
image_slice[ymin_f:ymax_f, xmin_f:xmax_f] = 1
image = image * image_slice
if (xfirst is None) or (yfirst is None):
phot_obj = BasicPSFPhotometry(finder=daofind,
group_maker=daogroup,
psf_model=gaussian_prf,
fitter=fitter,
fitshape=(21, 21),
bkg_estimator=mmm_bkg)
phot_results = phot_obj(image)
else:
gaussian_prf = IntegratedGaussianPRF(flux=1, sigma=1.7)
gaussian_prf.sigma.fixed = False
gaussian_prf.flux.fixed = False
gaussian_prf.x_0.fixed = False
gaussian_prf.y_0.fixed = False
phot_obj = BasicPSFPhotometry(group_maker=daogroup,
psf_model=gaussian_prf,
fitter=fitter,
fitshape=(21, 21),
bkg_estimator=mmm_bkg)
pos = Table(names=['x_0', 'y_0'],
data=[[xfirst, xfield], [yfirst, yfield]])
phot_results_tmp = phot_obj(image, init_guesses=pos)
resimage = phot_obj.get_residual_image()
pos = Table(names=['x_0', 'y_0'], data=[[x0], [y0]])
gaussian_prf = IntegratedGaussianPRF(flux=1, sigma=1.7)
gaussian_prf.sigma.fixed = False
gaussian_prf.flux.fixed = False
gaussian_prf.x_0.fixed = True
gaussian_prf.y_0.fixed = True
phot_obj = BasicPSFPhotometry(group_maker=daogroup,
psf_model=gaussian_prf,
fitter=fitter,
fitshape=(7, 7),
bkg_estimator=mmm_bkg)
phot_results = phot_obj(resimage, init_guesses=pos)
phot_results = vstack([phot_results_tmp, phot_results])
#if True:
if False:
#sources = iraffind(image)
sources = daofind(image)
import matplotlib.pyplot as plt
positions = np.transpose(
(sources['ycentroid'], sources['xcentroid']))
apertures = CircularAperture(positions, r=4.)
fig, axs = plt.subplots(1, 2)
plt.axes(axs[0])
plt.imshow(image.T,
origin='lower',
cmap='viridis',
aspect=1,
interpolation='nearest',
vmin=np.percentile(image[image > 0], 10),
vmax=np.percentile(image[image > 0], 90))
apertures.plot(color='red')
plt.xlim([ymin, ymax])
plt.ylim([xmin, xmax])
resimage = phot_obj.get_residual_image()
plt.axes(axs[1])
plt.imshow(resimage.T,
origin='lower',
cmap='viridis',
aspect=1,
interpolation='nearest',
vmin=0,
vmax=np.percentile(resimage[resimage > 0], 90))
apertures.plot(color='red')
plt.xlim([ymin, ymax])
plt.ylim([xmin, xmax])
plt.savefig('test_%d.png' % jj)
plt.close()
fig, axs = plt.subplots(1, 2)
plt.axes(axs[0])
plt.imshow(image.T,
origin='lower',
cmap='viridis',
aspect=1,
interpolation='nearest',
vmin=np.percentile(image[image > 0], 10),
vmax=np.percentile(image[image > 0], 90))
apertures.plot(color='red')
plt.xlim([ymin_f, ymax_f])
plt.ylim([xmin_f, xmax_f])
resimage = phot_obj.get_residual_image()
plt.axes(axs[1])
plt.imshow(resimage.T,
origin='lower',
cmap='viridis',
aspect=1,
interpolation='nearest',
vmin=np.percentile(resimage[resimage > 0], 10),
vmax=np.percentile(resimage[resimage > 0], 90))
apertures.plot(color='red')
plt.xlim([ymin_f, ymax_f])
plt.ylim([xmin_f, xmax_f])
plt.savefig('test_f_%d.png' % jj)
plt.close()
#phot_results.pprint_all()
#print(stop)
dist = np.sqrt((phot_results["x_fit"] - x0)**2 +
(phot_results["y_fit"] - y0)**2)
idx = np.argmin(dist)
flux = phot_results[idx]["flux_fit"]
fluxerr = phot_results[idx]["flux_unc"]
magerr = 1.0857 * fluxerr / flux #1.0857 = 2.5/log(10)
mag = zp - 2.5 * np.log10(flux)
if doDifferential:
dist = np.sqrt((phot_results["x_fit"] - xfield)**2 +
(phot_results["y_fit"] - yfield)**2)
idy = np.argmin(dist)
flux_field = phot_results[idy]["flux_fit"]
fluxerr_field = phot_results[idy]["flux_unc"]
magerr_field = 1.0857 * fluxerr_field / flux_field #1.0857 = 2.5/log(10)
mag_field = zp - 2.5 * np.log10(flux_field)
mag = mag - mag_field
magerr = np.sqrt(magerr**2 + magerr_field**2)
fluxerr = np.sqrt((fluxerr / flux)**2 +
(fluxerr_field / flux_field)**2)
flux = flux / flux_field
fluxerr = flux * fluxerr
#print(phot_results[idy]["flux_fit"], phot_results[idx]["flux_fit"])
mjds.append(mjd)
mags.append(mag)
magerrs.append(magerr)
fluxes.append(flux)
fluxerrs.append(fluxerr)
fid.write('%.5f %.5f %.5f %.5f %.5f\n' %
(dateobs.mjd, mag, magerr, flux, fluxerr))
fid.close()
return np.array(mjds), np.array(mags), np.array(magerrs), np.array(
fluxes), np.array(fluxerrs)
else:
mjds, mags, magerrs, fluxes, fluxerrs = [], [], [], [], []
for ii, hdu in enumerate(hdulist):
if ii == 0: continue
header = hdulist[ii].header
image = hdulist[ii].data
if not "DATE" in header:
print("Warning: 'DATE missing from %s hdu %d/%d" %
(imagefile, ii, len(hdulist)))
continue
dateobs = Time(header["DATE"])
phot_results = phot_obj(image)
dist = np.sqrt((phot_results["x_fit"] - x0)**2 +
(phot_results["y_fit"] - y0)**2)
idx = np.argmin(dist)
flux = phot_results[idx]["flux_fit"]
fluxerr = phot_results[idx]["flux_unc"]
magerr = 1.0857 * fluxerr / flux #1.0857 = 2.5/log(10)
mag = zp - 2.5 * np.log10(flux)
mjds.append(dateobs.mjd)
mags.append(mag)
magerrs.append(magerr)
fluxes.append(flux)
fluxerrs.append(fluxerr)
fid.write('%.5f %.5f %.5f %.5f %.5f\n' %
(dateobs.mjd, mag, magerr, flux, fluxerr))
fid.close()
return np.array(mjds), np.array(mags), np.array(magerrs), np.array(
fluxes), np.array(fluxerrs)
# 0 PRIMARY 1 PrimaryHDU 258 ()
# 1 SCI 1 ImageHDU 85 (2048, 2048) float32
# 2 ERR 1 ImageHDU 10 (2048, 2048) float32
# 3 DQ 1 ImageHDU 11 (2048, 2048) int32 (rescales to uint32)
# 4 AREA 1 ImageHDU 9 (2048, 2048) float32
# 5 VAR_POISSON 1 ImageHDU 9 (2048, 2048) float32
# 6 VAR_RNOISE 1 ImageHDU 9 (2048, 2048) float32
# 7 VAR_FLAT 1 ImageHDU 9 (2048, 2048) float32
# 8 ASDF 1 BinTableHDU 11 1R x 1C [17197B]
# The data is the hdu[1]
data = hdu[1].data
# Let's run a quick fitting using DAOStarFinder
mean, median, std = sigma_clipped_stats(data, sigma=3.0)
daofind = DAOStarFinder(fwhm=3.0, threshold=5. * std)
sources = daofind(data - median)
# And now, let's make some cuts to remove objects that are too faint or too bright
flux_min = 5
flux_max = 50
flux_range = np.where((sources['flux'] > flux_min)
& (sources['flux'] < flux_max))[0]
init_tbl = Table()
init_tbl['x_0'] = sources['xcentroid'][flux_range]
init_tbl['y_0'] = sources['ycentroid'][flux_range]
init_tbl['flux_0'] = sources['flux'][flux_range]
# And now, let's make a plot of the original image.
plt.figure(figsize=(9, 9))
def make_tweakreg_catalog(model,
kernel_fwhm,
snr_threshold,
sharplo=0.2,
sharphi=1.0,
roundlo=-1.0,
roundhi=1.0,
brightest=None,
peakmax=None):
"""
Create a catalog of point-line sources to be used for image
alignment in tweakreg.
Parameters
----------
model : `ImageModel`
The input `ImageModel` of a single image. The input image is
assumed to be background subtracted.
kernel_fwhm : float
The full-width at half-maximum (FWHM) of the 2D Gaussian kernel
used to filter the image before thresholding. Filtering the
image will smooth the noise and maximize detectability of
objects with a shape similar to the kernel.
snr_threshold : float
The signal-to-noise ratio per pixel above the ``background`` for
which to consider a pixel as possibly being part of a source.
sharplo : float, optional
The lower bound on sharpness for object detection.
sharphi : float, optional
The upper bound on sharpness for object detection.
roundlo : float, optional
The lower bound on roundness for object detection.
roundhi : float, optional
The upper bound on roundness for object detection.
brightest : int, None, optional
Number of brightest objects to keep after sorting the full object list.
If ``brightest`` is set to `None`, all objects will be selected.
peakmax : float, None, optional
Maximum peak pixel value in an object. Only objects whose peak pixel
values are *strictly smaller* than ``peakmax`` will be selected.
This may be used to exclude saturated sources. By default, when
``peakmax`` is set to `None`, all objects will be selected.
.. warning::
`DAOStarFinder` automatically excludes objects whose peak
pixel values are negative. Therefore, setting ``peakmax`` to a
non-positive value would result in exclusion of all objects.
Returns
-------
catalog : `~astropy.Table`
An astropy Table containing the source catalog.
"""
if not isinstance(model, ImageModel):
raise TypeError('The input model must be an ImageModel.')
threshold_img = detect_threshold(model.data, nsigma=snr_threshold)
# TODO: use threshold image based on error array
threshold = threshold_img[0, 0] # constant image
daofind = DAOStarFinder(fwhm=kernel_fwhm,
threshold=threshold,
sharplo=sharplo,
sharphi=sharphi,
roundlo=roundlo,
roundhi=roundhi,
brightest=brightest,
peakmax=peakmax)
# Mask the non-imaging area (e.g. MIRI)
mask = (dqflags.pixel['NON_SCIENCE'] & model.dq).astype(bool)
sources = daofind(model.data, mask=mask)
columns = ['id', 'xcentroid', 'ycentroid', 'flux']
if sources:
catalog = sources[columns]
else:
catalog = Table(names=columns,
dtype=(np.int_, np.float_, np.float_, np.float_))
return catalog
starfinder_result = run_optimizer(starfinder_optimizer,
starfinder_obj,
n_evaluations=1400)
x = starfinder_result.x
print(
list(zip(starfinder_result.space.dimension_names,
starfinder_result.x)))
mean, median, std = sigma_clipped_stats(img, sigma=3)
threshold = median + x[0] * std
finder = DAOStarFinder(threshold=threshold,
fwhm=x[1],
sigma_radius=x[2],
roundlo=x[3],
roundhi=x[4],
sharplo=x[5],
sharphi=x[6],
exclude_border=True)
result_table = finder(img)
plt.imshow(img, norm=LogNorm(), cmap='inferno')
plt.plot(input_table['x'],
input_table['y'],
'o',
fillstyle='none',
markeredgewidth=0.5,
markeredgecolor='red',
label=f'reference N={len(input_table)}')
plt.plot(result_table['xcentroid'],
#plt.imshow(image)
#plt.show()
fitsStars = fits.open("Images/FitsImages/L_2019-04-08_22-59-51_c.fits")
imgStars = fitsStars[0].data.astype(np.float64)
#bkg = MMMBackground()
#background = bkg(imgStars)
#gaussian_prf = PRF()
#gaussian_prf.sigma.fixed = False
#photTester = DAOP(8,background,5,gaussian_prf,(11,11))
daogroup = DAOGroup(crit_separation=8)
mmm_bkg = MMMBackground()
iraffind = DAOStarFinder(threshold=2 * mmm_bkg(imgStars), fwhm=4.5)
fitter = LevMarLSQFitter()
gaussian_prf = IntegratedGaussianPRF(sigma=2.05)
gaussian_prf.sigma.fixed = False
photTester = IterativelySubtractedPSFPhotometry(finder=iraffind,
group_maker=daogroup,
bkg_estimator=mmm_bkg,
psf_model=gaussian_prf,
fitter=fitter,
fitshape=(11, 11),
niters=2)
photResults = photTester(imgStars)
print(photResults['x_fit', 'y_fit', 'flux_fit'])
finalImg = photTester.get_residual_image()
def fwhm(image,syntax,sigma_lvl = None,fwhm = None):
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
'''
Find full width half maxiumu of an image via gaussian fitting to isolated bright stars
Can be very tempromental for bad images
'''
from astropy.stats import sigma_clipped_stats
from photutils.detection import DAOStarFinder
from autophot.packages.functions import gauss_fwhm,gauss_2d
import numpy as np
import pandas as pd
import sys,os
import lmfit
from astropy.stats import sigma_clip
import logging
logger = logging.getLogger(__name__)
if sigma_lvl != None:
min_source_no = 0
max_source_no = np.inf
else:
max_source_no = syntax['max_source_lim']
min_source_no = syntax['min_source_lim']
if sigma_lvl == None:
threshold_value = syntax['threshold_value']
int_fwhm = syntax['fwhm_guess']
else:
threshold_value = sigma_lvl
if fwhm != None:
int_fwhm = fwhm
else:
int_fwhm = syntax['fwhm_guess']
if fwhm != None:
syntax['int_scale'] = syntax['scale']
isolated_sources = []
img_seg = [0]
try:
for idx in range(len(list(img_seg))):
mean, median, std = sigma_clipped_stats(image,
sigma=syntax['fwhm_sigma'],
maxiters=syntax['fwhm_iters'])
if sigma_lvl == None:
logger.debug('Image stats: Mean %.3f :: Median %.3f :: std %.3f' % (mean,median,std))
syntax['global_mean'] = mean
syntax['global_median'] = median
syntax['global_std'] = std
# decrease
m = 0
# increase
n = 0
# backstop
failsafe = 0
decrease_increment = False
# How much to drop/increase each iteration each
fudge_factor = syntax['fudge_factor']
bkg_detect_check=[]
search_image = image.copy()
# Remove target by masking with area with that of median image value - just so it's not picked up
if syntax['target_name'] != None and fwhm == None:
logger.info('Target location : (x,y) -> (%.3f,%.3f)' % (syntax['target_x_pix'] , syntax['target_y_pix']))
search_image[int(syntax['target_y_pix'])-syntax['int_scale']: int(syntax['target_y_pix']) + syntax['int_scale'],
int(syntax['target_x_pix'])-syntax['int_scale']: int(syntax['target_x_pix']) + syntax['int_scale']] = syntax['global_median'] * np.ones((int(2*syntax['int_scale']),int(2*syntax['int_scale'])))
while True:
try:
# If iterations get to big - terminate
if failsafe>syntax['source_max_iter']:
logger.info(' Source detection gives up!')
break
else:
failsafe +=1
# check if threshold value is still good
threshold_value_check = threshold_value + n - m
# if <=0 reverse previous drop and and fine_fudge factor
if threshold_value_check <= syntax['lim_SNR']:
logger.warning('Threshold value has gone below threshold - increasing by smaller increment ')
# revert privious decrease
decrease_increment = True
n=syntax['fine_fudge_factor']
# m = 0 to stop any further decrease
threshold_value += m
m = 0
else:
threshold_value = round(threshold_value + n - m,3)
# if threshold goes negative usesmaller fudge factor
if decrease_increment:
fudge_factor = syntax['fine_fudge_factor']
daofind = DAOStarFinder(fwhm = np.ceil(int_fwhm),
threshold = threshold_value*std,
sharplo = 0.2,sharphi = 1.0,
roundlo = -1.0,roundhi = 1.0
)
sources = daofind(search_image - median)
if sources == None:
logger.warning('Sources == None')
m = fudge_factor
continue
sources = sources.to_pandas()
logger.info('Number of sources before cleaning - [s = %.1f]: %d ' % (threshold_value,len(sources)))
# f_x = len(sources) / (threshold_value*std)
# relative_change = f_x - np.nanmedian(f_x) / np.nanmedian(f_x)
if len(sources) == 0:
logger.warning('No sources')
m = fudge_factor
continue
# bkg_detect_check.append(relative_change)
try:
sources['xcentroid']
'Make sure some are detceted, if not try again '
except Exception as e:
logger.exception(e)
break
if len(sources) > 10000 and m !=0:
logger.warning('Picking up noise')
fudge_factor = syntax['fine_fudge_factor']
n = syntax['fine_fudge_factor']
m = 0
decrease_increment = True
continue
elif len(sources) > max_source_no:
logger.warning('Too many sources')
if m != 0 :
decrease_increment = True
n = syntax['fine_fudge_factor']
fudge_factor = syntax['fine_fudge_factor']
else:
n = fudge_factor
continue
elif len(sources) < min_source_no:
logger.warning('Too few sources')
m = fudge_factor
continue
elif len(sources) == 0:
logger.warning('No sources')
m = fudge_factor
continue
if len(sources) > 30:
if syntax['remove_boundary_sources']:
with_boundary = len(sources)
sources = sources[sources['xcentroid'] < image.shape[1] - syntax['pix_bound'] ]
sources = sources[sources['xcentroid'] > syntax['pix_bound'] ]
sources = sources[sources['ycentroid'] < image.shape[0] - syntax['pix_bound'] ]
sources = sources[sources['ycentroid'] > syntax['pix_bound'] ]
logger.debug('Removed %d sources near boundary' % (with_boundary - len(sources)))
x = np.array(sources['xcentroid'])
y = np.array(sources['ycentroid'])
iso_temp = []
pix_dist = []
if len(sources) < min_source_no:
logger.warning('Less than min source after boundary removal')
m = fudge_factor
continue
if sigma_lvl != None or len(sources) < 10:
isolated_sources = pd.DataFrame({'x_pix':x,'y_pix':y})
else:
for idx in range(len(x)):
try:
x0 = x[idx]
y0 = y[idx]
dist = np.sqrt((x0-np.array(x))**2+(y0-np.array(y))**2)
dist = dist[np.where(dist!=0)]
isolated_dist = 0
if syntax['isolate_sources']:
isolated_dist = syntax['iso_scale']
if len(dist) == 0:
dist = [0]
if min(list(dist)) > isolated_dist:
df = np.array((float(x0),float(y0)))
iso_temp.append(df)
pix_dist.append(dist)
except Exception as e:
logger.exception(e)
pass
if len(iso_temp) == 0:
logger.warning('Less than min source after isolating sources')
m = fudge_factor
continue
isolated_sources= pd.DataFrame(data = iso_temp)
isolated_sources.columns = ['x_pix','y_pix']
isolated_sources.reset_index()
x_rc = []
y_rc = []
sigma=[]
medianlst=[]
x_pix = np.arange(0,2 * syntax['int_scale'])
y_pix = np.arange(0,2 * syntax['int_scale'])
image_copy = image.copy()
for idx in isolated_sources.index:
try:
x0 = isolated_sources['x_pix'].loc[[idx]]
y0 = isolated_sources['y_pix'].loc[[idx]]
close_up = image_copy[int(y0)- syntax['int_scale']: int(y0) + syntax['int_scale'],
int(x0)- syntax['int_scale']: int(x0) + syntax['int_scale']]
if close_up.shape != (int(2*syntax['int_scale']),int(2*syntax['int_scale'])):
sigma.append(np.nan)
x_rc.append(np.nan)
y_rc.append(np.nan)
medianlst.append(np.nan)
logger.warning('wrong close-up size')
continue
mean, median_val, std = sigma_clipped_stats(close_up,
sigma=syntax['fwhm_sigma'],
maxiters=syntax['fwhm_iters'])
medianlst.append(median_val)
xx, yy = np.meshgrid(x_pix, y_pix)
if syntax['remove_sat']:
try:
saturation_lvl = syntax['sat_lvl']
except:
saturation_lvl = 2**16
if np.nanmax(close_up) >= saturation_lvl:
sigma.append(np.nan)
x_rc.append(np.nan)
y_rc.append(np.nan)
continue
try:
pars = lmfit.Parameters()
pars.add('A',value = np.nanmax(close_up),min = 0)
pars.add('x0',value = close_up.shape[0]/2)
pars.add('y0',value = close_up.shape[0]/2)
pars.add('sigma',value = 3,max = syntax['max_fit_fwhm'] / 2*np.sqrt(2*np.log(2)) )
pars.add('sky',value = np.nanmedian(close_up))
def residual(p):
p = p.valuesdict()
return (close_up - gauss_2d((xx,yy),p['x0'],p['y0'],p['sky'],p['A'],p['sigma']).reshape(close_up.shape)).flatten()
mini = lmfit.Minimizer(residual, pars,nan_policy = 'omit')
result = mini.minimize(method = 'least_squares')
sigma_fit = abs(result.params['sigma'].value)
sigma.append(sigma_fit)
x_rc.append(result.params['x0'] - syntax['int_scale'] + x0)
y_rc.append(result.params['y0'] - syntax['int_scale'] + y0)
except Exception as e:
logger.exception(e)
sigma.append(np.nan)
x_rc.append(np.nan)
y_rc.append(np.nan)
pass
except Exception as e:
logger.exception(e)
sigma.append(np.nan)
x_rc.append(np.nan)
y_rc.append(np.nan)
medianlst.append(median_val)
continue
if sigma_lvl == None:
isolated_sources['sigma'] = pd.Series(sigma)
isolated_sources['median'] = pd.Series(medianlst)
if isolated_sources['sigma'].values == np.array([]):
logger.info('> No sigma values taken <')
continue
try:
if len(isolated_sources) > 30:
isolate_mask = sigma_clip(isolated_sources['sigma'].values, sigma=1.5).mask
isolated_sources = isolated_sources[~isolate_mask]
except:
isolated_sources = []
if len(isolated_sources) < min_source_no:
logger.warning('Less than min source after sigma clipping: %d' % len(isolated_sources))
threshold_value += m
if n ==0:
decrease_increment = True
n = syntax['fine_fudge_factor']
fudge_factor = syntax['fine_fudge_factor']
else:
n = fudge_factor
# m = fudge_factor
m=0
# n = fudge_factor
continue
logger.info('Isolated sources found [ %.1f sigma ]: %d' % (threshold_value,len(isolated_sources)))
sigma = np.nanmedian(isolated_sources['sigma'])
mean_fwhm = gauss_fwhm(sigma)
syntax['scale'] = int(np.ceil(syntax['scale_multipler'] * mean_fwhm))
else:
isolated_sources['sigma'] = pd.Series(sigma)
mean_fwhm = fwhm
break
except Exception as e:
logger.exception(e)
continue
return mean_fwhm,isolated_sources,syntax
except Exception as e:
logger.exception(e)
return np.nan
def psf_fit(data_array, data_file, psf_grid, fheader, imagemodel):
# Let's run a quick fitting using DAOStarFinder
mean, median, std = sigma_clipped_stats(data_array, sigma=3.0)
daofind = DAOStarFinder(fwhm=3.0, threshold=5. * std)
sources = daofind(data_array - median)
# And now, let's make some cuts to remove objects that are too faint or too bright
flux_min = 5 #5 #0
flux_max = 50 #50 #1000
flux_range = np.where((sources['flux'] > flux_min)
& (sources['flux'] < flux_max))[0]
init_tbl = Table()
init_tbl['x_0'] = sources['xcentroid'][flux_range]
init_tbl['y_0'] = sources['ycentroid'][flux_range]
init_tbl['flux_0'] = sources['flux'][flux_range]
# And now, let's make a plot of the original image.
plt.figure(figsize=(9, 9))
imshow_norm(data_array,
interval=PercentileInterval(99.),
stretch=SqrtStretch())
plt.colorbar()
plt.savefig(data_file + '.png', dpi=300)
plt.clf()
# And now, let's make a plot of the image showing the positions of the objects.
plt.figure(figsize=(9, 9))
imshow_norm(data_array,
interval=PercentileInterval(99.),
stretch=SqrtStretch())
plt.scatter(init_tbl['x_0'], init_tbl['y_0'], s=10, color='black')
plt.colorbar()
plt.savefig(data_file + '_with_daostarfinder_objects.png', dpi=300)
plt.clf()
eval_xshape = int(np.ceil(psf_grid.data.shape[2] / psf_grid.oversampling))
eval_yshape = int(np.ceil(psf_grid.data.shape[1] / psf_grid.oversampling))
# And now, let's run the PSF Photometry
sigma_psf = 3.
daogroup = DBSCANGroup(2.0 * sigma_psf * gaussian_sigma_to_fwhm)
mmm_bkg = MMMBackground()
fit_shape = (eval_yshape, eval_xshape)
phot = BasicPSFPhotometry(daogroup,
mmm_bkg,
psf_grid,
fit_shape,
finder=None,
aperture_radius=3.)
# This is the part that takes the longest, so I print out the date/time before and after.
now = datetime.now()
dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
print("Starting the fit: date and time = ", dt_string)
tbl = phot(data_array, init_guesses=init_tbl)
now = datetime.now()
dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
print("Ending the fit: date and time = ", dt_string)
# Now I format the output
tbl['x_fit'].format = '%.1f'
tbl['y_fit'].format = '%.1f'
tbl['flux_fit'].format = '%.4e'
tbl['flux_unc'].format = '%.4e'
tbl['x_0_unc'].format = '%.4e'
tbl['y_0_unc'].format = '%.4e'
diff = phot.get_residual_image()
hdu_out = fits.PrimaryHDU(diff, header=fheader)
hdul_out = fits.HDUList([hdu_out])
hdul_out.writeto(data_file + '_residual.fits')
# And create a residual image from the fit
plt.figure(figsize=(9, 9))
imshow_norm(diff, interval=PercentileInterval(99.), stretch=SqrtStretch())
#plt.scatter(tbl['x_fit'], tbl['y_fit'], s=80, facecolors='none', edgecolors='r')
plt.colorbar()
plt.savefig(data_file + '_residual.png', dpi=300)
plt.clf()
# Calculate the RA and DEC values from the x_fit and y_fit values.
RA_fit = np.zeros(len(tbl['x_fit']))
DEC_fit = np.zeros(len(tbl['x_fit']))
RA_fit, DEC_fit = imagemodel.meta.wcs(tbl['x_fit'], tbl['y_fit'])
tbl.add_column(DEC_fit, index=0, name='DEC_fit')
tbl.add_column(RA_fit, index=0, name='RA_fit')
# And write out the table to a file.
tbl.write(data_file + '_psf_fit_output.fits')
print('\n')
imheader = hdu_list[0].header
imdata = hdu_list[0].data
hdu_list.close()
#%% get image background statistics
mean, median, std = sigma_clipped_stats(imdata, sigma=3.0)
print(
f'Image Background: mean = {mean:.5g}, median = {median:.5g}, standard deviation = {std:.3g}\n'
)
seeing = 6.0
#%% find the sources
daofind = DAOStarFinder(fwhm=seeing, sky=median, threshold=5. * std)
sources = daofind.find_stars(imdata - median)
print('\nPrint source locations:')
sources['id', 'xcentroid',
'ycentroid'].pprint() #print out positions of sources
print('\n')
#%% perform aperture photometry
# extract source postions from table; transpose is needed for proper orientation
positions = np.transpose((sources['xcentroid'], sources['ycentroid']))
# define the aperture
r_a = 3 * seeing
apertures = CircularAperture(positions, r=r_a)
# define the annulus
r_in = r_a + 3
time.sleep(0.5)
for job in jobs:
optimizer.tell(job.args, job.result.get())
except KeyboardInterrupt:
pass
res = optimizer.get_result()
with open(result_filename, 'wb') as f:
dill.dump(optimizer, f)
threshold, fwhm, sigma_radius, roundlo, roundhi, sharplo, sharphi = res.x
res_table = DAOStarFinder(threshold=median + std * threshold,
fwhm=fwhm,
sigma_radius=sigma_radius,
sharplo=sharplo,
sharphi=sharphi,
roundlo=roundlo,
roundhi=roundhi,
exclude_border=True)(img)
plt.ion()
plt.imshow(img, norm=LogNorm())
plt.plot(res_table['xcentroid'],
res_table['ycentroid'],
'ro',
markersize=0.5)
plot_evaluations(res, dimensions=dimensions)
plt.figure()
ax = plot_convergence(res)
ax.set_yscale('log')
def baz():
config = Config.instance()
# throw away border pixels to make psf fit into original image
psf = read_or_generate_helper('anisocado_psf', config)
psf = center_cutout(
psf, (51, 51)) # cutout center of psf or else it takes forever to fit
config.output_folder = 'output_cheating_astrometry'
filename = 'scopesim_grid_16_perturb0'
image, input_table = read_or_generate_image(filename, config)
origin = np.array(psf.shape) / 2
# type: ignore
epsf = photutils.psf.EPSFModel(psf,
flux=None,
origin=origin,
oversampling=1,
normalize=False)
epsf = photutils.psf.prepare_psf_model(epsf, renormalize_psf=False)
image, input_table = read_or_generate_image(filename, config)
mean, median, std = sigma_clipped_stats(image, sigma=config.clip_sigma)
threshold = median + config.threshold_factor * std
fwhm = estimate_fwhm(epsf.psfmodel)
finder = DAOStarFinder(threshold=threshold, fwhm=fwhm)
grouper = DAOGroup(config.separation_factor * fwhm)
shape = (epsf.psfmodel.shape / epsf.psfmodel.oversampling).astype(np.int64)
epsf.fwhm = astropy.modeling.Parameter(
'fwhm', 'this is not the way to add this I think')
epsf.fwhm.value = fwhm
bkgrms = MADStdBackgroundRMS()
photometry = BasicPSFPhotometry(finder=finder,
group_maker=grouper,
bkg_estimator=bkgrms,
psf_model=epsf,
fitter=LevMarLSQFitter(),
fitshape=shape)
result_table = photometry(image)
star_guesses = make_stars_guess(image, finder, 51)
plot_filename = os.path.join(config.output_folder,
filename + '_photometry_vs_sources')
plot_image_with_source_and_measured(image,
input_table,
result_table,
output_path=plot_filename)
if len(result_table) != 0:
plot_filename = os.path.join(config.output_folder,
filename + '_measurement_offset')
plot_xy_deviation(input_table, result_table, output_path=plot_filename)
else:
print(f"No sources found for {filename} with {config}")
plt.figure()
plt.imshow(epsf.psfmodel.data)
save(os.path.join(config.output_folder, filename + '_epsf'), plt.gcf())
plt.figure()
plt.imshow(concat_star_images(star_guesses))
save(os.path.join(config.output_folder, filename + '_star_guesses'),
plt.gcf())
res = PhotometryResult(image, input_table, result_table, epsf,
star_guesses)
def create_dao_like_coordlists(fitsfile,
sourcelist_filename,
make_region_file=False,
dao_fwhm=3.5,
bkgsig_sf=2.):
"""Make daofind-like coordinate lists
Parameters
----------
fitsfile : string
Name of the drizzle-combined filter product to used to generate photometric sourcelists.
sourcelist_filename : string
Name of optionally generated ds9-compatible region file
dao_fwhm : float
(`~photutils.detection.DAOstarfinder` param 'fwhm') The full-width half-maximum (FWHM) of the major axis of the
Gaussian kernel in units of pixels. Default value = 3.5.
make_region_file : Boolean
Generate ds9-compatible region file? Default value = True
bkgsig_sf : float
multiplictive scale factor applied to background sigma value to compute DAOfind input parameter
'threshold'. Default value = 2.
Returns
-------
sources : astropy table
Table containing x, y coordinates of identified sources
"""
hdulist = fits.open(fitsfile)
image = hdulist['SCI'].data
image -= np.nanmedian(image)
bkg_sigma = mad_std(image, ignore_nan=True)
daofind = DAOStarFinder(fwhm=dao_fwhm, threshold=bkgsig_sf * bkg_sigma)
sources = daofind(image)
hdulist.close()
for col in sources.colnames:
sources[col].info.format = '%.8g' # for consistent table output
# Write out ecsv file
tbl_length = len(sources)
sources.write(sourcelist_filename, format="ascii.ecsv")
log.info("Created coord list file '{}' with {} sources".format(
sourcelist_filename, tbl_length))
if make_region_file:
out_table = sources.copy()
# Remove all other columns besides xcentroid and ycentroid
out_table.keep_columns(['xcentroid', 'ycentroid'])
# Add offset of 1.0 in X and Y to line up sources in region file with image displayed in ds9.
out_table['xcentroid'].data[:] += np.float64(1.0)
out_table['ycentroid'].data[:] += np.float64(1.0)
reg_filename = sourcelist_filename.replace(".ecsv", ".reg")
out_table.write(reg_filename, format="ascii")
log.info("Created region file '{}' with {} sources".format(
reg_filename, len(out_table)))
return (sources)
def psf_photometry(self,
mbi,
label=None,
subtract_galaxy=True,
sersic_model=None,
save_residual_images=False,
**kwargs):
self.mbi = mbi
self._setup_psf()
if subtract_galaxy:
self.phot_image = self.subtract_galaxy(label, sersic_model,
**kwargs)
else:
self.phot_image = mbi.image
if self.use_hsc_bright_mask['phot']:
logger.info('applying hsc bright object mask')
for b in mbi.bands:
mask = mbi.get_hsc_bright_object_mask(b).astype(bool)
self.phot_image[b][mask] = 0
catalog = LsstStruct()
self.stddev = LsstStruct()
if save_residual_images:
self.residual_image = LsstStruct()
for band in mbi.bands:
if len(mbi.stats) > 0:
self.stddev[band] = mbi.stats[band].stdev
else:
self.stddev[band] = self.bkgrms(self.phot_image[band])
daogroup = DAOGroup(self.crit_separation * self.psf_fwhm[band])
self.daofinder_opt['fwhm'] = self.psf_fwhm[band]
self.phot_opts['aperture_radius'] = self.aperture_radius
self.phot_opts['aperture_radius'] *= self.psf_fwhm[band]
daofind = DAOStarFinder(self.threshold * self.stddev[band],
exclude_border=True,
**self.daofinder_opt)
logger.info('performing ' + band + '-band psf photometry')
photometry = IterativelySubtractedPSFPhotometry(
finder=daofind,
group_maker=daogroup,
psf_model=self.psf_model[band],
**self.phot_opts)
with warnings.catch_warnings():
message = '.*The fit may be unsuccessful;.*'
warnings.filterwarnings('ignore',
message=message,
category=AstropyUserWarning)
catalog[band] = photometry(image=self.phot_image[band])
if save_residual_images:
logger.info('generating residual image')
self.residual_image[band] = subtract_psf(
mbi.image[band], self.psf_model[band], catalog[band])
return catalog
def find(image,
fwhm,
method='daophot',
background='1D',
frame='diff',
diag=False):
'''
Find all stars above the sky background level using DAOFind-like algorithm
Required inputs:
image = 2D array of image on which to perform find
fwhm = FWHM in pixels (1)
Optional inputs:
method = Either 'daophot' or 'peaks' to select different finding algorithms
background = '2D' or '1D' to select 2- or 1-D background estimators
frame = 'diff' or 'single' to set background behaviour for difference or single frames
Example
-------
>>> np.random.seed(0)
>>> im = np.ones((10,10)) + np.random.uniform(size=(10,10))
>>> im *= u.ph
>>> im[5,5] += 5 * u.ph
>>> star_tbl, bkg_image, threshold = find(im, 1, method='peaks', background='1D', frame='single')
>>> np.equal(len(star_tbl), 1)
True
'''
from photutils.detection import DAOStarFinder, find_peaks
from astropy.stats import sigma_clipped_stats
if frame == 'diff':
# Determine background RMS:
bkg_image, sky = estimate_background(image,
method=background,
sigma=5,
diag=diag)
find_image = image
elif frame == 'single':
# Create and subtract a background image and determine background RMS:
bkg_image, sky = estimate_background(image,
method=background,
sigma=2,
diag=diag)
find_image = image - bkg_image
# Look for sources at twice the background RMS level
threshold = 2 * sky
# Make sure the image and threshold units are the same
threshold = threshold.to(image.unit)
# Find stars
if method == 'daophot':
finder = DAOStarFinder(threshold.value, fwhm)
star_tbl = finder.find_stars(find_image.value)
star_tbl['x'], star_tbl['y'] = \
star_tbl['xcentroid'], star_tbl['ycentroid']
elif method == 'peaks':
star_tbl = find_peaks(find_image.value, threshold.value, box_size=3)
star_tbl['x'], star_tbl['y'] = \
star_tbl['x_peak'], star_tbl['y_peak']
# Remove entries outside the image frame (mainly an issue with daophot), then reset the ID column:
index = ((star_tbl['x'] < 0) | (star_tbl['y'] < 0) |
(star_tbl['x'] > image.shape[0]) |
(star_tbl['y'] > image.shape[1]))
star_tbl.remove_rows(index)
star_tbl['id'] = np.arange(len(star_tbl)) + 1
if diag:
print("Sky background rms: {}".format(sky))
print("Found {} stars".format(len(star_tbl)))
return star_tbl, bkg_image, threshold