def get_winpos(data, x, y, a, subsampling=5):
"""
Get windowed position. These are more accurate positions calculated
by SEP (SExtractor) created by iteratively recentering on the area
contained by the half-flux radius.
Parameters
----------
data: `~numpy.array`
Image data
x: array-like
X coordinates of the sources
y: array-like
Y coordinates of the sources
subsampling: int, optional
Number of subpixels for each image pixel
(used to calculate the half flux radius).
Default=5.
"""
import sep
r, flag = sep.flux_radius(data, x, y,
6.*a, 0.5, subpix=subsampling)
sig = 2. / 2.35 * r
xwin, ywin, flags = sep.winpos(data, x, y, sig)
return xwin, ywin
def test_flux_radius():
data = np.ones(data_shape)
fluxfrac = [0.2**2, 0.3**2, 0.7**2, 1.]
true_r = [2., 3., 7., 10.]
r, _ = sep.flux_radius(data, x, y, 10.*np.ones_like(x),
[0.2**2, 0.3**2, 0.7**2, 1.], subpix=5)
for i in range(len(fluxfrac)):
assert_allclose(r[:, i], true_r[i], rtol=0.01)
def _measure(self, img, sources, mask=None):
logger.info('measuring source parameters')
# HACK: issues with numerical precision
# must have pi/2 <= theta <= npi/2
sources[np.abs(np.abs(sources['theta']) - np.pi/2) < 1e-6] = np.pi/2
for p in ['x', 'y', 'a', 'b', 'theta']:
sources = sources[~np.isnan(sources[p])]
# calculate "AUTO" parameters
kronrad, krflag = sep.kron_radius(
img, sources['x'], sources['y'], sources['a'], sources['b'],
sources['theta'], 6.0, mask=mask)
flux, fluxerr, flag = sep.sum_ellipse(
img, sources['x'], sources['y'], sources['a'], sources['b'],
sources['theta'], 2.5*kronrad, subpix=5, mask=mask)
flag |= krflag # combine flags into 'flag'
sources = sources[~np.isnan(flux)]
flux = flux[~np.isnan(flux)]
sources = sources[flux > 0]
flux = flux[flux > 0]
mag_auto = utils.zpt - 2.5*np.log10(flux)
r, flag = sep.flux_radius(
img, sources['x'], sources['y'], 6.*sources['a'], 0.5,
normflux=flux, subpix=5, mask=mask)
sources['mag_auto'] = mag_auto
sources['flux_auto'] = flux
sources['flux_radius'] = r * utils.pixscale
# approximate fwhm
r_squared = sources['a']**2 + sources['b']**2
sources['fwhm'] = 2 * np.sqrt(np.log(2) * r_squared) * utils.pixscale
q = sources['b'] / sources['a']
area = np.pi * q * sources['flux_radius']**2
sources['mu_ave_auto'] = sources['mag_auto'] + 2.5 * np.log10(2*area)
area_arcsec = np.pi * (self.psf_fwhm/2)**2 * utils.pixscale**2
flux, fluxerr, flag = sep.sum_circle(
img, sources['x'], sources['y'], self.psf_fwhm/2,
subpix=5, mask=mask)
flux[flux<=0] = np.nan
mu_0 = utils.zpt - 2.5*np.log10(flux / area_arcsec)
sources['mu_0_aper'] = mu_0
return sources
def find_flux(tbdat_sub, objects, kronrad, kronflag): flux, fluxerr, flag = sep.sum_ellipse(tbdat_sub, objects['x'], objects['y'], objects['a'], objects['b'], objects['theta'], pho_auto_A = (2.5*kronrad), err = bkg.globalrms, subpix=1) flag |=kronflag #combines all flags r_min = 1.75 #minimum diameter = 3.5 use_circle = kronrad * np.sqrt(a * b) < r_min cflux, cfluxerr, cflag = sep.sum_circle(tbdat_sub, objects['x'][use_circle], objects['y'][use_circle], r_min, subpix=1) flux[use_circle] = cflux fluxerr[use_circle] = cfluxerr flag[use_circle] = cflag r, rflag = sep.flux_radius(data, x, y, 6.0*objects['a'], rmax = 0.5, normflux = flux, subpix =5) sig = 2.0 / (2.35*r) # r from sep.flux_radius() above, with fluxfrac = 0.5 xwin, ywin, wflag = sep.winpos(tbdat_sub, objects['x'], objects['y'], sig) return flux, fluxerr, flag, r, xwin, ywin
def make_HSC_detect_mask(bin_msk,
img,
objects,
segmap,
r=10.0,
radius=1.5,
threshold=0.01):
'''Make HSC detection and bright star mask,
based on HSC binary mask flags.
Parameters:
-----------
bin_msk: 2-D np.array, can be loaded from HSC image cutouts
objects: table, returned from sep.extract_obj
segmap: 2-D np.array, returned from sep.extract_obj
r: float, blow-up parameter
radius: float, convolution radius
threshold: float, threshold of making mask after convolution
Returns:
-----------
HSC_detect_mask: 2-D boolean np.array
See also:
-----------------
convert_HSC_binary_mask(bin_msk)
'''
import sep
TDmask = convert_HSC_binary_mask(bin_msk)
cen_mask = np.zeros(bin_msk.shape, dtype=np.bool)
cen_obj = objects[segmap[int(bin_msk.shape[0] / 2.),
int(bin_msk.shape[1] / 2.)] - 1]
fraction_radius = sep.flux_radius(img, cen_obj['x'], cen_obj['y'],
10 * cen_obj['a'], 0.5)[0]
ba = np.divide(cen_obj['b'], cen_obj['a'])
sep.mask_ellipse(cen_mask,
cen_obj['x'],
cen_obj['y'],
fraction_radius,
fraction_radius * ba,
cen_obj['theta'],
r=r)
from astropy.convolution import convolve, Gaussian2DKernel
HSC_mask = (TDmask[:, :, 5]).astype(bool) * (
~cen_mask) + TDmask[:, :, 9].astype(bool)
# Convolve the image with a Gaussian kernel with the width of 1.5 pixel
cvl = convolve(HSC_mask.astype('float'), Gaussian2DKernel(radius))
HSC_detect_mask = cvl >= threshold
return HSC_detect_mask
def _get_flux_auto(self, objs):
flux_auto = np.zeros(objs.size) - 9999.0
fluxerr_auto = np.zeros(objs.size) - 9999.0
flux_radius = np.zeros(objs.size) - 9999.0
kron_radius = np.zeros(objs.size) - 9999.0
w, = np.where((objs['a'] >= 0.0) & (objs['b'] >= 0.0)
& (objs['theta'] >= -np.pi / 2.)
& (objs['theta'] <= np.pi / 2.))
if w.size > 0:
kron_radius[w], krflag = sep.kron_radius(
self.image,
objs['x'][w],
objs['y'][w],
objs['a'][w],
objs['b'][w],
objs['theta'][w],
6.0,
)
objs['flag'][w] |= krflag
aper_rad = 2.5 * kron_radius
flux_auto[w], fluxerr_auto[w], flag_auto = \
sep.sum_ellipse(
self.image,
objs['x'][w],
objs['y'][w],
objs['a'][w],
objs['b'][w],
objs['theta'][w],
aper_rad[w],
subpix=1,
)
objs['flag'][w] |= flag_auto
flux_radius[w], frflag = sep.flux_radius(
self.image,
objs['x'][w],
objs['y'][w],
6. * objs['a'][w],
PHOT_FLUXFRAC,
normflux=flux_auto[w],
subpix=5,
)
objs['flag'][w] |= frflag # combine flags into 'flag'
return flux_auto, fluxerr_auto, flux_radius, kron_radius
def measureHfd(data):
bkg = sep.Background(data)
thresh = 3 * bkg.globalrms
objects = sep.extract(data - bkg, thresh)
# taken from g-tecs autoFocus.py
hfr, mask = sep.flux_radius(data,
objects['x'],
objects['y'],
30 * np.ones_like(objects['x']),
0.5,
normflux=objects['cflux'])
mask = np.logical_and(mask == 0, objects['peak'] < 40000)
hfd = 2 * hfr[mask]
if hfd.size > 3:
mean, median, std = sigma_clipped_stats(hfd, sigma=2.5, iters=10)
return median, std
return 0.0, 0.0
def findSpot(data, sigma):
image=data
#m, s = np.mean(image), np.std(image)
bkg = sep.Background(image, bw=32, bh=32, fw=3, fh=3)
objs = sep.extract(image-bkg, sigma, err=bkg.globalrms)
aper_radius=3
# Calculate the Kron Radius
kronrad, krflag = sep.kron_radius(image, objs['x'], objs['y'], \
objs['a'], objs['b'], objs['theta'], aper_radius)
r_min = 3
use_circle = kronrad * np.sqrt(objs['a'] * objs['b'])
cinx=np.where(use_circle <= r_min)
einx=np.where(use_circle > r_min)
# Calculate the equivalent of FLUX_AUTO
flux, fluxerr, flag = sep.sum_ellipse(image, objs['x'][einx], objs['y'][einx], \
objs['a'][einx], objs['b'][einx], objs['theta'][einx], 2.5*kronrad[einx],subpix=1)
cflux, cfluxerr, cflag = sep.sum_circle(image, objs['x'][cinx], objs['y'][cinx],
objs['a'][cinx], subpix=1)
# Adding half pixel to measured coordinate.
objs['x'] = objs['x']+0.5
objs['y'] = objs['y']+0.5
objs['flux'][einx]=flux
objs['flux'][cinx]=cflux
r, flag = sep.flux_radius(image, objs['x'], objs['y'], \
6*objs['a'], 0.3,normflux=objs['flux'], subpix=5)
flag |= krflag
objs=rfn.append_fields(objs, 'r', data=r, usemask=False)
objects=objs[:]
return objects
def measureHfd(img):
data = fits.open(img)[0].data
y, x = data.shape
xcen = int(x / 2)
ycen = int(y / 2)
# grab the middle 2k x 2k
stats_area = np.array(data[ycen-1024: ycen+1024, \
xcen-1024: xcen+1024]).astype(np.int32).copy(order='C')
bkg = sep.Background(stats_area)
thresh = 3 * bkg.globalrms
objects = sep.extract(stats_area - bkg, thresh)
# taken from g-tecs autoFocus.py
hfr, mask = sep.flux_radius(stats_area,
objects['x'],
objects['y'],
30 * np.ones_like(objects['x']),
0.5,
normflux=objects['cflux'])
mask = np.logical_and(mask == 0, objects['peak'] < 40000)
hfd = 2 * hfr[mask]
if hfd.size > 3:
mean, median, std = sigma_clipped_stats(hfd, sigma=2.5, iters=10)
return median, std
return 0.0, 0.0
def sourceExtract(data,
thresh=3,
bkg=False,
bkg_rms=None,
err=None,
mask=None,
min_area=5,
deblend_cont=0.05,
segment=False,
extras=False):
"""
Extract all sources above a certain threshold in the given image
Parameters
----------
data : array-like
CCD image frame from which to extract sources
thresh : float, optional
Number of sigma a detection must be above the background to
be flagged as a source - if err not given, bkg_rms is needed
Default = 3
bkg : bool, optional
Toggle to model spatially varying background and subtract from
data - by default assumes this has been done separately
Default = False
bkg_rms : float, optional
Estimation of the global background noise - used to determine
threshold if err is None - can calculate global background
rms when subtracting background model
Default = None
err : array-like, optional
Error array for the CCD frame - supersedes bkg_rms when
determining the threshold
Default = None
min_area : int, optional
Minimum number of pixels to be flagged as a source
Default = 5
deblend_cont : float, optional
Minimum contrast ratio used by SEP for deblending
Default = 0.05
segment : bool, optional
Toggle to generate a segmentation map for the given image
Default = False
extras : bool, optional
Toggle to calculate ellipticity, FWHM, Kron radius and
flux radius
Default = False
Returns
-------
sources : astropy Table object
Table containing quantities determined by sep for each source
detected in the given image
segmentation_map : array-like, optional
Array of integers with same shape as data - pixels not
belonging to any object have value 0, whilst all pixels
belonging to ith object have value (e.g. sources[i]) have
value i+1 - only returned if seg_map is True
Raises
------
None
"""
# subtract spatially varying background model if requested
if bkg:
data, bkg_rms = subtractBackground(data)
# determine threshold for extraction
if err is None:
thresh *= bkg_rms
# extract sources
if not segment:
sources = sep.extract(data,
thresh,
err=err,
mask=mask,
deblend_cont=deblend_cont)
else:
sources, seg_map = sep.extract(data,
thresh,
err=err,
mask=mask,
deblend_cont=deblend_cont,
segmentation_map=seg_map)
sources = Table(sources)
# remove nans from table
sources = pruneNansFromTable(sources)
if extras:
# calculate ellipticity parameter
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# calculate full width half maxima
sources['fwhm'] = calculateFWHM(sources['a'], sources['b'])
# compute kron radii
try:
sources['kronr'], krflag = sep.kron_radius(data, sources['x'],
sources['y'],
sources['a'],
sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
except Exception as e:
print(e)
pass
# compute flux radii
try:
sources['fluxr'], frflag = sep.flux_radius(data,
sources['x'],
sources['y'],
6.0 * sources['a'],
0.5,
subpix=5)
sources['flag'] |= frflag
except Exception as e:
print(e)
pass
if segment:
return sources, seg_map
else:
return sources
def test_vs_sextractor():
"""Test behavior of sep versus sextractor.
Note: we turn deblending off for this test. This is because the
deblending algorithm uses a random number generator. Since the sequence
of random numbers is not the same between sextractor and sep or between
different platforms, object member pixels (and even the number of objects)
can differ when deblending is on.
Deblending is turned off by setting DEBLEND_MINCONT=1.0 in the sextractor
configuration file and by setting deblend_cont=1.0 in sep.extract().
"""
data = np.copy(image_data) # make an explicit copy so we can 'subfrom'
bkg = sep.Background(data, bw=64, bh=64, fw=3, fh=3)
# Test that SExtractor background is same as SEP:
bkgarr = bkg.back(dtype=np.float32)
assert_allclose(bkgarr, image_refback, rtol=1.e-5)
# Extract objects (use deblend_cont=1.0 to disable deblending).
bkg.subfrom(data)
objs = sep.extract(data, 1.5*bkg.globalrms, deblend_cont=1.0)
objs = np.sort(objs, order=['y'])
# Read SExtractor result
refobjs = np.loadtxt(IMAGECAT_FNAME, dtype=IMAGECAT_DTYPE)
refobjs = np.sort(refobjs, order=['y'])
# Found correct number of sources at the right locations?
assert_allclose(objs['x'], refobjs['x'] - 1., atol=1.e-3)
assert_allclose(objs['y'], refobjs['y'] - 1., atol=1.e-3)
# Test aperture flux
flux, fluxerr, flag = sep.sum_circle(data, objs['x'], objs['y'], 5.,
err=bkg.globalrms)
assert_allclose(flux, refobjs['flux_aper'], rtol=2.e-4)
assert_allclose(fluxerr, refobjs['fluxerr_aper'], rtol=1.0e-5)
# check if the flags work at all (comparison values
assert ((flag & sep.APER_TRUNC) != 0).sum() == 4
assert ((flag & sep.APER_HASMASKED) != 0).sum() == 0
# Test "flux_auto"
kr, flag = sep.kron_radius(data, objs['x'], objs['y'], objs['a'],
objs['b'], objs['theta'], 6.0)
flux, fluxerr, flag = sep.sum_ellipse(data, objs['x'], objs['y'],
objs['a'], objs['b'],
objs['theta'], r=2.5 * kr,
err=bkg.globalrms, subpix=1)
# For some reason, one object doesn't match. It's very small
# and kron_radius is set to 0.0 in SExtractor, but 0.08 in sep.
# Could be due to a change in SExtractor between v2.8.6 (used to
# generate "truth" catalog) and v2.18.11 (from which sep was forked).
i = 56 # index is 59 when deblending is on.
kr[i] = 0.0
flux[i] = 0.0
fluxerr[i] = 0.0
# We use atol for radius because it is reported to nearest 0.01 in
# reference objects.
assert_allclose(2.5*kr, refobjs['kron_radius'], atol=0.01, rtol=0.)
assert_allclose(flux, refobjs['flux_auto'], rtol=0.0005)
assert_allclose(fluxerr, refobjs['fluxerr_auto'], rtol=0.0005)
# Test ellipse representation conversion
cxx, cyy, cxy = sep.ellipse_coeffs(objs['a'], objs['b'], objs['theta'])
assert_allclose(cxx, objs['cxx'], rtol=1.e-4)
assert_allclose(cyy, objs['cyy'], rtol=1.e-4)
assert_allclose(cxy, objs['cxy'], rtol=1.e-4)
a, b, theta = sep.ellipse_axes(objs['cxx'], objs['cyy'], objs['cxy'])
assert_allclose(a, objs['a'], rtol=1.e-4)
assert_allclose(b, objs['b'], rtol=1.e-4)
assert_allclose(theta, objs['theta'], rtol=1.e-4)
#test round trip
cxx, cyy, cxy = sep.ellipse_coeffs(a, b, theta)
assert_allclose(cxx, objs['cxx'], rtol=1.e-4)
assert_allclose(cyy, objs['cyy'], rtol=1.e-4)
assert_allclose(cxy, objs['cxy'], rtol=1.e-4)
# test flux_radius
fr, flags = sep.flux_radius(data, objs['x'], objs['y'], 6.*refobjs['a'],
[0.1, 0.5, 0.6], normflux=refobjs['flux_auto'],
subpix=5)
assert_allclose(fr, refobjs["flux_radius"], rtol=0.04, atol=0.01)
# test winpos
sig = 2. / 2.35 * fr[:, 1] # flux_radius = 0.5
xwin, ywin, flag = sep.winpos(data, objs['x'], objs['y'], sig)
assert_allclose(xwin, refobjs["xwin"] - 1., rtol=0., atol=0.0025)
assert_allclose(ywin, refobjs["ywin"] - 1., rtol=0., atol=0.0025)
async def __call__(self, image: Image) -> Image:
"""Find stars in given image and append catalog.
Args:
image: Image to find stars in.
Returns:
Image with attached catalog.
"""
import sep
loop = asyncio.get_running_loop()
# got data?
if image.data is None:
log.warning("No data found in image.")
return image
# no mask?
mask = image.mask if image.mask is not None else np.zeros(
image.data.shape, dtype=bool)
# remove background
data, bkg = SepSourceDetection.remove_background(image.data, mask)
# extract sources
sources = await loop.run_in_executor(
None,
partial(
sep.extract,
data,
self.threshold,
err=bkg.globalrms,
minarea=self.minarea,
deblend_nthresh=self.deblend_nthresh,
deblend_cont=self.deblend_cont,
clean=self.clean,
clean_param=self.clean_param,
mask=image.mask,
),
)
# convert to astropy table
sources = pd.DataFrame(sources)
# only keep sources with detection flag < 8
sources = sources[sources["flag"] < 8]
x, y = sources["x"], sources["y"]
# Calculate the ellipticity
sources["ellipticity"] = 1.0 - (sources["b"] / sources["a"])
# calculate the FWHMs of the stars
fwhm = 2.0 * (np.log(2) * (sources["a"]**2.0 + sources["b"]**2.0))**0.5
sources["fwhm"] = fwhm
# clip theta to [-pi/2,pi/2]
sources["theta"] = sources["theta"].clip(lower=np.pi / 2,
upper=np.pi / 2)
# Kron radius
kronrad, krflag = sep.kron_radius(data, x, y, sources["a"],
sources["b"], sources["theta"], 6.0)
sources["flag"] |= krflag
sources["kronrad"] = kronrad
# equivalent of FLUX_AUTO
gain = image.header["DET-GAIN"] if "DET-GAIN" in image.header else None
flux, fluxerr, flag = await loop.run_in_executor(
None,
partial(
sep.sum_ellipse,
data,
x,
y,
sources["a"],
sources["b"],
sources["theta"],
2.5 * kronrad,
subpix=5,
mask=image.mask,
gain=gain,
),
)
sources["flag"] |= flag
sources["flux"] = flux
# radii at 0.25, 0.5, and 0.75 flux
flux_radii, flag = sep.flux_radius(data,
x,
y,
6.0 * sources["a"],
[0.25, 0.5, 0.75],
normflux=sources["flux"],
subpix=5)
sources["flag"] |= flag
sources["fluxrad25"] = flux_radii[:, 0]
sources["fluxrad50"] = flux_radii[:, 1]
sources["fluxrad75"] = flux_radii[:, 2]
# xwin/ywin
sig = 2.0 / 2.35 * sources["fluxrad50"]
xwin, ywin, flag = sep.winpos(data, x, y, sig)
sources["flag"] |= flag
sources["xwin"] = xwin
sources["ywin"] = ywin
# theta in degrees
sources["theta"] = np.degrees(sources["theta"])
# only keep sources with detection flag < 8
sources = sources[sources["flag"] < 8]
# match fits conventions
sources["x"] += 1
sources["y"] += 1
# pick columns for catalog
cat = sources[[
"x",
"y",
"peak",
"flux",
"fwhm",
"a",
"b",
"theta",
"ellipticity",
"tnpix",
"kronrad",
"fluxrad25",
"fluxrad50",
"fluxrad75",
"xwin",
"ywin",
]]
# copy image, set catalog and return it
img = image.copy()
img.catalog = Table.from_pandas(cat)
return img
def get_objects_sep(image, header=None, mask=None, thresh=4.0, aper=3.0, bkgann=None, r0=0.5, gain=1, edge=0, minnthresh=2, minarea=5, relfluxradius=2.0, wcs=None, use_fwhm=False, use_mask_bg=False, use_mask_large=False, subtract_bg=True, npix_large=300, sn=10.0, verbose=True, get_fwhm=False, **kwargs):
if r0 > 0.0:
kernel = make_kernel(r0)
else:
kernel = None
if verbose:
print("Preparing background mask")
if mask is None:
mask = np.zeros_like(image, dtype=np.bool)
mask_bg = np.zeros_like(mask)
mask_segm = np.zeros_like(mask)
if use_mask_bg:
# Simple heuristics to mask regions with rapidly varying background
if verbose:
print("Masking rapidly changing background")
bg2 = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64)
for _ in xrange(3):
bg1 = sep.Background(image, mask=mask|mask_bg, bw=256, bh=256)
ibg = bg2.back() - bg1.back()
tmp = np.abs(ibg - np.median(ibg)) > 5.0*mad_std(ibg)
mask_bg |= dilate(tmp, np.ones([100, 100]))
mask_bg = dilate(tmp, np.ones([100, 100]))
if verbose:
print("Building background map")
bg = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64)
if subtract_bg:
image1 = image - bg.back()
else:
image1 = image.copy()
sep.set_extract_pixstack(image.shape[0]*image.shape[1])
if use_mask_large:
# Mask regions around huge objects as they are most probably corrupted by saturation and blooming
if verbose:
print("Extracting initial objects")
obj0,segm = sep.extract(image1, err=bg.rms(), thresh=thresh, minarea=minarea, mask=mask|mask_bg, filter_kernel=kernel, segmentation_map=True, **kwargs)
if verbose:
print("Dilating large objects")
mask_segm = np.isin(segm, [_+1 for _,npix in enumerate(obj0['npix']) if npix > npix_large])
mask_segm = dilate(mask_segm, np.ones([50, 50]))
if verbose:
print("Extracting final objects")
obj0 = sep.extract(image1, err=bg.rms(), thresh=thresh, minarea=minarea, mask=mask|mask_bg|mask_segm, filter_kernel=kernel, **kwargs)
if use_fwhm:
# Estimate FHWM and use it to get optimal aperture size
idx = obj0['flag'] == 0
fwhm = 2.0*np.sqrt(np.hypot(obj0['a'][idx], obj0['b'][idx])*np.log(2))
fwhm = 2.0*sep.flux_radius(image1, obj0['x'][idx], obj0['y'][idx], relfluxradius*fwhm*np.ones_like(obj0['x'][idx]), 0.5, mask=mask)[0]
fwhm = np.median(fwhm)
aper = max(1.5*fwhm, aper)
if verbose:
print("FWHM = %.2g, aperture = %.2g" % (fwhm, aper))
# Windowed positional parameters are often biased in crowded fields, let's avoid them for now
# xwin,ywin,flag = sep.winpos(image1, obj0['x'], obj0['y'], 0.5, mask=mask)
xwin,ywin = obj0['x'], obj0['y']
# Filter out objects too close to frame edges
idx = (np.round(xwin) > edge) & (np.round(ywin) > edge) & (np.round(xwin) < image.shape[1]-edge) & (np.round(ywin) < image.shape[0]-edge) # & (obj0['flag'] == 0)
if minnthresh:
idx &= (obj0['tnpix'] >= minnthresh)
if verbose:
print("Measuring final objects")
flux,fluxerr,flag = sep.sum_circle(image1, xwin[idx], ywin[idx], aper, err=bg.rms(), gain=gain, mask=mask|mask_bg|mask_segm, bkgann=bkgann)
# For debug purposes, let's make also the same aperture photometry on the background map
bgflux,bgfluxerr,bgflag = sep.sum_circle(bg.back(), xwin[idx], ywin[idx], aper, err=bg.rms(), gain=gain, mask=mask|mask_bg|mask_segm)
bg = bgflux/np.pi/aper**2
# Fluxes to magnitudes
mag,magerr = np.zeros_like(flux), np.zeros_like(flux)
mag[flux>0] = -2.5*np.log10(flux[flux>0])
# magerr[flux>0] = 2.5*np.log10(1.0 + fluxerr[flux>0]/flux[flux>0])
magerr[flux>0] = 2.5/np.log(10)*fluxerr[flux>0]/flux[flux>0]
# better FWHM estimation - FWHM=HFD for Gaussian
if get_fwhm:
fwhm = 2.0*sep.flux_radius(image1, xwin[idx], ywin[idx], relfluxradius*aper*np.ones_like(xwin[idx]), 0.5, mask=mask)[0]
else:
fwhm = np.zeros_like(xwin[idx])
flag |= obj0['flag'][idx]
# Quality cuts
fidx = (flux > 0) & (magerr < 1.0/sn)
if wcs is None and header is not None:
# If header is provided, we may build WCS from it
wcs = WCS(header)
if wcs is not None:
# If WCS is provided we may convert x,y to ra,dec
ra,dec = wcs.all_pix2world(obj0['x'][idx], obj0['y'][idx], 0)
else:
ra,dec = np.zeros_like(obj0['x'][idx]),np.zeros_like(obj0['y'][idx])
if verbose:
print("All done")
return {'x':xwin[idx][fidx], 'y':ywin[idx][fidx], 'flux':flux[fidx], 'fluxerr':fluxerr[fidx], 'mag':mag[fidx], 'magerr':magerr[fidx], 'flags':obj0['flag'][idx][fidx]|flag[fidx], 'ra':ra[fidx], 'dec':dec[fidx], 'bg':bg[fidx], 'fwhm':fwhm[fidx], 'aper':aper, 'bkgann':bkgann, 'a':obj0['a'][idx][fidx], 'b':obj0['b'][idx][fidx], 'theta':obj0['theta'][idx][fidx]}
b =1
apertures = [2.5,3.4,5,7.5,10,15,20,30,50,70]
catalog = []
f = []
a =1
flux, fluxerr, flag = sep.sum_ellipse(data_sub, objects['x'], objects['y'], a, b, theta,apertures[0],err=bkg.globalrms, gain=1.4)
flux1, fluxerr1, flag1 = sep.sum_ellipse(data_sub, objects['x'], objects['y'], a, b, theta,apertures[1],err=bkg.globalrms, gain=1.4)
flux2, fluxerr2, flag2 = sep.sum_ellipse(data_sub, objects['x'], objects['y'], a, b, theta,apertures[2],err=bkg.globalrms, gain=1.4)
flux3, fluxerr3, flag3 = sep.sum_ellipse(data_sub, objects['x'], objects['y'], a, b, theta,apertures[3],err=bkg.globalrms, gain=1.4)
flux4, fluxerr4, flag4 = sep.sum_ellipse(data_sub, objects['x'], objects['y'], a, b, theta,apertures[4],err=bkg.globalrms, gain=1.4)
flux5, fluxer5r, flag5 = sep.sum_ellipse(data_sub, objects['x'], objects['y'], a, b, theta,apertures[5],err=bkg.globalrms, gain=1.4)
flux6, fluxerr6, flag6 = sep.sum_ellipse(data_sub, objects['x'], objects['y'], a, b, theta,apertures[6],err=bkg.globalrms, gain=1.4)
flux7, fluxerr7, flag7 = sep.sum_ellipse(data_sub, objects['x'], objects['y'], a, b, theta,apertures[7],err=bkg.globalrms, gain=1.4)
flux8, fluxerr8, flag8 = sep.sum_ellipse(data_sub, objects['x'], objects['y'], a, b, theta,apertures[8],err=bkg.globalrms, gain=1.4)
flux9, fluxerr9, flag9 = sep.sum_ellipse(data_sub, objects['x'], objects['y'], a, b, theta,apertures[9],err=bkg.globalrms, gain=1.4)
r, flag = sep.flux_radius(data_sub, objects['x'], objects['y'],6.*objects['a'], 0.5, normflux=flux, subpix=5)
r1, flag1 = sep.flux_radius(data_sub, objects['x'], objects['y'],6.*objects['a'], 0.5, normflux=flux, subpix=5)
r2, flag2 = sep.flux_radius(data_sub, objects['x'], objects['y'], 6.*objects['a'],0.5, normflux=flux, subpix=5)
r3, flag3 = sep.flux_radius(data_sub, objects['x'], objects['y'],6.*objects['a'], 0.5, normflux=flux, subpix=5)
r4, flag4 = sep.flux_radius(data_sub, objects['x'], objects['y'],6.*objects['a'], 0.5, normflux=flux, subpix=5)
r5, flag5 = sep.flux_radius(data_sub, objects['x'], objects['y'],6.*objects['a'], 0.5, normflux=flux, subpix=5)
r6, flag6 = sep.flux_radius(data_sub, objects['x'], objects['y'],6.*objects['a'], 0.5, normflux=flux, subpix=5)
r7, flag7 = sep.flux_radius(data_sub, objects['x'], objects['y'],6.*objects['a'], 0.5, normflux=flux, subpix=5)
r8, flag8 = sep.flux_radius(data_sub, objects['x'], objects['y'],6.*objects['a'], 0.5, normflux=flux, subpix=5)
r9, flag9 = sep.flux_radius(data_sub, objects['x'], objects['y'],6.*objects['a'], 0.5, normflux=flux, subpix=5)
#print(flux)
#return flux
catalog = np.array(flux)
def findStars(wind, thresh, kernel_fwhm, return_bkg=False):
"""
Use sep to find objects in image. Not sure the outputs returned
are correct for xbin != ybin
Parameters
----------
wind : `~hipercam.window.Window`
window in which to detect objects.
thresh : float
threshold for object detection, in muliples of background RMS.
kernel_fwhm : float
Image is convolved with a Gaussian kernel of this FWHM prior to object
detection. Should be set to something similar to the typical FWHM in image.
return_bkg : bool
True to return the background as calculated by sep.Background
Returns
-------
objects : np.ndarray
Extracted object parameters (structured array).
For available fields, see sep documentation.
http://sep.readthedocs.io/en/v1.0.x/api/sep.extract.html#sep.extract
Most quantities are converted to unbinned coordinates and pixels,
apart from npix, and tnpix, the number of pixels in the objects.
bkg : `~sep.Background` [if return_bkg]
The background estimated by 'sep'. Use sep.subfrom
to subtract from data.
"""
# ensure float type, c-contiguous and native byte order
data = wind.data.astype("float")
bkg = sep.Background(data)
bkg.subfrom(data) # in-place background subtraction
sigma = 2.5 * gaussian_fwhm_to_sigma
kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
kernel.normalize()
objects = sep.extract(data,
thresh,
err=bkg.globalrms,
clean=True,
filter_kernel=kernel.array)
# add crude FWHM estimate and HFD measurement
fwhm = 2.0 * np.sqrt(np.log(2.0) * (objects["a"]**2 + objects["b"]**2))
hfr, flags = sep.flux_radius(
data,
objects["x"],
objects["y"],
25 * np.ones_like(objects["x"]),
0.5,
normflux=objects["cflux"],
)
bad_hfd = np.logical_or(flags != 0, hfr >= 25)
hfr[bad_hfd] = np.nan
objects = recfunctions.append_fields(objects, ("fwhm", "hfd"),
(fwhm, 2 * hfr))
# convert to un-binned pixels
objects["xmin"] = (wind.x(objects["xmin"] - (wind.xbin - 1) / 2)).astype(
np.int32)
objects["xmax"] = (wind.x(objects["xmax"] + (wind.xbin - 1) / 2)).astype(
np.int32)
objects["ymin"] = (wind.y(objects["ymin"] - (wind.ybin - 1) / 2)).astype(
np.int32)
objects["ymax"] = (wind.y(objects["ymax"] + (wind.ybin - 1) / 2)).astype(
np.int32)
for key in ("x", "xcpeak", "xpeak"):
objects[key] = wind.x(objects[key])
for key in ("y", "ycpeak", "ypeak"):
objects[key] = wind.y(objects[key])
for key in ("xy", "cxy"):
objects[key] *= wind.xbin * wind.ybin
for key in ("x2", "a", "errx2", "cxx"):
objects[key] *= wind.xbin
for key in ("y2", "b", "erry2", "cyy"):
objects[key] *= wind.ybin
for key in ("fwhm", "hfd"):
objects[key] *= np.sqrt(wind.xbin * wind.ybin)
# change the data type to save on space
objects = objects.astype(SMALL_TYPE)
if return_bkg:
return (objects, bkg)
else:
return objects
def test_vs_sextractor():
data = np.copy(image_data) # make an explicit copy so we can 'subfrom'
bkg = sep.Background(data, bw=64, bh=64, fw=3, fh=3)
# Test that SExtractor background is same as SEP:
bkgarr = bkg.back(dtype=np.float32)
assert_allclose(bkgarr, image_refback, rtol=1.e-5)
# Extract objects
bkg.subfrom(data)
objs = sep.extract(data, 1.5*bkg.globalrms)
objs = np.sort(objs, order=['y'])
# Read SExtractor result
refobjs = np.loadtxt(IMAGECAT_FNAME, dtype=IMAGECAT_DTYPE)
refobjs = np.sort(refobjs, order=['y'])
# Found correct number of sources at the right locations?
assert_allclose(objs['x'], refobjs['x'] - 1., atol=1.e-3)
assert_allclose(objs['y'], refobjs['y'] - 1., atol=1.e-3)
# Test aperture flux
flux, fluxerr, flag = sep.sum_circle(data, objs['x'], objs['y'], 5.,
err=bkg.globalrms)
assert_allclose(flux, refobjs['flux_aper'], rtol=2.e-4)
assert_allclose(fluxerr, refobjs['fluxerr_aper'], rtol=1.0e-5)
# check if the flags work at all (comparison values
assert ((flag & sep.APER_TRUNC) != 0).sum() == 4
assert ((flag & sep.APER_HASMASKED) != 0).sum() == 0
# Test "flux_auto"
kr, flag = sep.kron_radius(data, objs['x'], objs['y'], objs['a'],
objs['b'], objs['theta'], 6.0)
flux, fluxerr, flag = sep.sum_ellipse(data, objs['x'], objs['y'],
objs['a'], objs['b'],
objs['theta'], r=2.5 * kr,
err=bkg.globalrms, subpix=1)
# For some reason, object at index 59 doesn't match. It's very small
# and kron_radius is set to 0.0 in SExtractor, but 0.08 in sep.
# Most of the other values are within 1e-4 except one which is only
# within 0.01. This might be due to a change in SExtractor between
# v2.8.6 (used to generate "truth" catalog) and v2.18.11.
kr[59] = 0.0
flux[59] = 0.0
fluxerr[59] = 0.0
assert_allclose(2.5*kr, refobjs['kron_radius'], rtol=0.01)
assert_allclose(flux, refobjs['flux_auto'], rtol=0.01)
assert_allclose(fluxerr, refobjs['fluxerr_auto'], rtol=0.01)
# Test ellipse representation conversion
cxx, cyy, cxy = sep.ellipse_coeffs(objs['a'], objs['b'], objs['theta'])
assert_allclose(cxx, objs['cxx'], rtol=1.e-4)
assert_allclose(cyy, objs['cyy'], rtol=1.e-4)
assert_allclose(cxy, objs['cxy'], rtol=1.e-4)
a, b, theta = sep.ellipse_axes(objs['cxx'], objs['cyy'], objs['cxy'])
assert_allclose(a, objs['a'], rtol=1.e-4)
assert_allclose(b, objs['b'], rtol=1.e-4)
assert_allclose(theta, objs['theta'], rtol=1.e-4)
#test round trip
cxx, cyy, cxy = sep.ellipse_coeffs(a, b, theta)
assert_allclose(cxx, objs['cxx'], rtol=1.e-4)
assert_allclose(cyy, objs['cyy'], rtol=1.e-4)
assert_allclose(cxy, objs['cxy'], rtol=1.e-4)
# test flux_radius
fr, flags = sep.flux_radius(data, objs['x'], objs['y'], 6.*refobjs['a'],
[0.1, 0.5, 0.6], normflux=refobjs['flux_auto'],
subpix=5)
assert_allclose(fr, refobjs["flux_radius"], rtol=0.04, atol=0.01)
# test winpos
sig = 2. / 2.35 * fr[:, 1] # flux_radius = 0.5
xwin, ywin, flag = sep.winpos(data, objs['x'], objs['y'], sig)
assert_allclose(xwin, refobjs["xwin"] - 1., rtol=0., atol=0.0025)
assert_allclose(ywin, refobjs["ywin"] - 1., rtol=0., atol=0.0025)
def do_stage(self, images):
for i, image in enumerate(images):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise ** 2.0) ** 0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data, self.threshold, minarea=self.min_area,
err=error, deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data, sources['x'], sources['y'],
sources['a'], sources['b'],
np.pi / 2.0, 2.5 * kronrad,
subpix=1, err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) * (sources['a'] ** 2.0 + sources['b'] ** 2.0)) ** 0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data, sources['x'], sources['y'],
6.0 * sources['a'], [0.25, 0.5, 0.75],
normflux=sources['flux'], subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'], sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(), sources['x'], sources['y'],
sources['a'], sources['b'], np.pi / 2.0,
2.5 * sources['kronrad'], subpix=1)
#masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (2.5 * sources['kronrad']) ** 2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
'flux', 'fluxerr', 'background', 'fwhm',
'a', 'b', 'theta', 'kronrad', 'ellipticity',
'fluxrad25', 'fluxrad50', 'fluxrad75',
'x2', 'y2', 'xy', 'flag']
# Add the units and description to the catalogs
image.catalog['x'].unit = 'pixel'
image.catalog['x'].description = 'X coordinate of the object'
image.catalog['y'].unit = 'pixel'
image.catalog['y'].description = 'Y coordinate of the object'
image.catalog['xwin'].unit = 'pixel'
image.catalog['xwin'].description = 'Windowed X coordinate of the object'
image.catalog['ywin'].unit = 'pixel'
image.catalog['ywin'].description = 'Windowed Y coordinate of the object'
image.catalog['xpeak'].unit = 'pixel'
image.catalog['xpeak'].description = 'X coordinate of the peak'
image.catalog['ypeak'].unit = 'pixel'
image.catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
image.catalog['flux'].unit = 'counts'
image.catalog['flux'].description = 'Flux within a Kron-like elliptical aperture'
image.catalog['fluxerr'].unit = 'counts'
image.catalog['fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
image.catalog['background'].unit = 'counts'
image.catalog['background'].description = 'Average background value in the aperture'
image.catalog['fwhm'].unit = 'pixel'
image.catalog['fwhm'].description = 'FWHM of the object'
image.catalog['a'].unit = 'pixel'
image.catalog['a'].description = 'Semi-major axis of the object'
image.catalog['b'].unit = 'pixel'
image.catalog['b'].description = 'Semi-minor axis of the object'
image.catalog['theta'].unit = 'degrees'
image.catalog['theta'].description = 'Position angle of the object'
image.catalog['kronrad'].unit = 'pixel'
image.catalog['kronrad'].description = 'Kron radius used for extraction'
image.catalog['ellipticity'].description = 'Ellipticity'
image.catalog['fluxrad25'].unit = 'pixel'
image.catalog['fluxrad25'].description = 'Radius containing 25% of the flux'
image.catalog['fluxrad50'].unit = 'pixel'
image.catalog['fluxrad50'].description = 'Radius containing 50% of the flux'
image.catalog['fluxrad75'].unit = 'pixel'
image.catalog['fluxrad75'].description = 'Radius containing 75% of the flux'
image.catalog['x2'].unit = 'pixel^2'
image.catalog['x2'].description = 'Variance on X coordinate of the object'
image.catalog['y2'].unit = 'pixel^2'
image.catalog['y2'].description = 'Variance on Y coordinate of the object'
image.catalog['xy'].unit = 'pixel^2'
image.catalog['xy'].description = 'XY covariance of the object'
image.catalog['flag'].description = 'Bit mask combination of extraction and photometry flags'
image.catalog.sort('flux')
image.catalog.reverse()
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (mean_background,
'[counts] Sigma clipped mean of frame background')
logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (median_background,
'[counts] Median of frame background')
logs.add_tag(logging_tags, 'L1MEDIAN', float(median_background))
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (std_background,
'[counts] Robust std dev of frame background')
logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))
# Save some image statistics to the header
good_objects = image.catalog['flag'] == 0
seeing = np.median(image.catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing, '[arcsec] Frame FWHM in arcsec')
logs.add_tag(logging_tags, 'L1FWHM', float(seeing))
mean_ellipticity = stats.sigma_clipped_mean(sources['ellipticity'][good_objects],
3.0)
image.header['L1ELLIP'] = (mean_ellipticity, 'Mean image ellipticity (1-B/A)')
logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))
mean_position_angle = stats.sigma_clipped_mean(sources['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (mean_position_angle,
'[deg] PA of mean image ellipticity')
logs.add_tag(logging_tags, 'L1ELLIPA', float(mean_position_angle))
self.logger.info('Extracted sources', extra=logging_tags)
except Exception as e:
logging_tags = logs.image_config_to_tags(image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename', os.path.basename(image.filename))
self.logger.error(e, extra=logging_tags)
return images
def findStars(wind, thresh, kernel_fwhm):
"""
Use sep to find objects in image.
Parameters
----------
wind : `~hipercam.window.Window`
window in which to detect objects.
thresh : float
threshold for object detection, in muliples of background RMS.
kernel_fwhm: float
Image is convolved with a Gaussian kernel of this FWHM prior to object
detection. Should be set to something similar to the typical FWHM in image.
Returns
-------
objects : np.ndarray
Extracted object parameters (structured array).
For available fields, see sep documentation.
http://sep.readthedocs.io/en/v1.0.x/api/sep.extract.html#sep.extract
Quantities are in un-binned pixels
"""
# ensure float type, c-contiguous and native byte order
data = wind.data.astype('float')
bkg = sep.Background(data)
bkg.subfrom(data) # in-place background subtraction
sigma = 2.5 * gaussian_fwhm_to_sigma
kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
kernel.normalize()
objects = sep.extract(data, thresh, err=bkg.globalrms, clean=True,
filter_kernel=kernel.array)
# add crude FWHM estimate and HFD measurement
fwhm = 2. * np.sqrt(np.log(2.) * (objects['a']**2 + objects['b']**2))
hfr, flags = sep.flux_radius(data, objects['x'], objects['y'], 25*np.ones_like(objects['x']),
0.5, normflux=objects['cflux'])
bad_hfd = np.logical_or(flags != 0, hfr >= 25)
hfr[bad_hfd] = np.nan
objects = recfunctions.append_fields(objects, ('fwhm', 'hfd'), (fwhm, 2*hfr))
# convert to un-binned pixels
for key in ('x', 'xmin', 'xmax', 'xcpeak', 'xpeak'):
objects[key] = objects[key]*wind.xbin + wind.llx
for key in ('y', 'ymin', 'ymax', 'ycpeak', 'ypeak'):
objects[key] = objects[key]*wind.ybin + wind.lly
for key in ('npix', 'tnpix', 'xy', 'cxy'):
objects[key] *= wind.xbin * wind.ybin
for key in ('x2', 'a', 'errx2', 'cxx'):
objects[key] *= wind.xbin
for key in ('y2', 'b', 'erry2', 'cyy'):
objects[key] *= wind.ybin
for key in ('fwhm', 'hfd'):
objects[key] *= np.sqrt(wind.xbin * wind.ybin)
return objects
def do_stage(self, images):
for i, image in enumerate(images):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise**2.0)**0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data,
mask=mask,
bw=32,
bh=32,
fw=3,
fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data,
mask=mask,
bw=32,
bh=32,
fw=3,
fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data,
self.threshold,
minarea=self.min_area,
err=error,
deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'],
sources['y'], sources['a'],
sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * kronrad,
subpix=1,
err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) *
(sources['a']**2.0 + sources['b']**2.0))**0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data,
sources['x'],
sources['y'],
6.0 * sources['a'],
[0.25, 0.5, 0.75],
normflux=sources['flux'],
subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'],
sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(),
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 *
sources['kronrad'],
subpix=1)
#masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (
2.5 * sources['kronrad']
)**2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[
background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
image.catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak',
'ypeak', 'flux', 'fluxerr',
'background', 'fwhm', 'a', 'b',
'theta', 'kronrad', 'ellipticity',
'fluxrad25', 'fluxrad50', 'fluxrad75',
'x2', 'y2', 'xy', 'flag']
# Add the units and description to the catalogs
image.catalog['x'].unit = 'pixel'
image.catalog['x'].description = 'X coordinate of the object'
image.catalog['y'].unit = 'pixel'
image.catalog['y'].description = 'Y coordinate of the object'
image.catalog['xwin'].unit = 'pixel'
image.catalog[
'xwin'].description = 'Windowed X coordinate of the object'
image.catalog['ywin'].unit = 'pixel'
image.catalog[
'ywin'].description = 'Windowed Y coordinate of the object'
image.catalog['xpeak'].unit = 'pixel'
image.catalog['xpeak'].description = 'X coordinate of the peak'
image.catalog['ypeak'].unit = 'pixel'
image.catalog[
'ypeak'].description = 'Windowed Y coordinate of the peak'
image.catalog['flux'].unit = 'counts'
image.catalog[
'flux'].description = 'Flux within a Kron-like elliptical aperture'
image.catalog['fluxerr'].unit = 'counts'
image.catalog[
'fluxerr'].description = 'Erronr on the flux within a Kron-like elliptical aperture'
image.catalog['background'].unit = 'counts'
image.catalog[
'background'].description = 'Average background value in the aperture'
image.catalog['fwhm'].unit = 'pixel'
image.catalog['fwhm'].description = 'FWHM of the object'
image.catalog['a'].unit = 'pixel'
image.catalog[
'a'].description = 'Semi-major axis of the object'
image.catalog['b'].unit = 'pixel'
image.catalog[
'b'].description = 'Semi-minor axis of the object'
image.catalog['theta'].unit = 'degrees'
image.catalog[
'theta'].description = 'Position angle of the object'
image.catalog['kronrad'].unit = 'pixel'
image.catalog[
'kronrad'].description = 'Kron radius used for extraction'
image.catalog['ellipticity'].description = 'Ellipticity'
image.catalog['fluxrad25'].unit = 'pixel'
image.catalog[
'fluxrad25'].description = 'Radius containing 25% of the flux'
image.catalog['fluxrad50'].unit = 'pixel'
image.catalog[
'fluxrad50'].description = 'Radius containing 50% of the flux'
image.catalog['fluxrad75'].unit = 'pixel'
image.catalog[
'fluxrad75'].description = 'Radius containing 75% of the flux'
image.catalog['x2'].unit = 'pixel^2'
image.catalog[
'x2'].description = 'Variance on X coordinate of the object'
image.catalog['y2'].unit = 'pixel^2'
image.catalog[
'y2'].description = 'Variance on Y coordinate of the object'
image.catalog['xy'].unit = 'pixel^2'
image.catalog['xy'].description = 'XY covariance of the object'
image.catalog[
'flag'].description = 'Bit mask combination of extraction and photometry flags'
image.catalog.sort('flux')
image.catalog.reverse()
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (
mean_background,
'[counts] Sigma clipped mean of frame background')
logs.add_tag(logging_tags, 'L1MEAN', float(mean_background))
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (
median_background, '[counts] Median of frame background')
logs.add_tag(logging_tags, 'L1MEDIAN',
float(median_background))
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (
std_background,
'[counts] Robust std dev of frame background')
logs.add_tag(logging_tags, 'L1SIGMA', float(std_background))
# Save some image statistics to the header
good_objects = image.catalog['flag'] == 0
seeing = np.median(
image.catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing,
'[arcsec] Frame FWHM in arcsec')
logs.add_tag(logging_tags, 'L1FWHM', float(seeing))
mean_ellipticity = stats.sigma_clipped_mean(
sources['ellipticity'][good_objects], 3.0)
image.header['L1ELLIP'] = (mean_ellipticity,
'Mean image ellipticity (1-B/A)')
logs.add_tag(logging_tags, 'L1ELLIP', float(mean_ellipticity))
mean_position_angle = stats.sigma_clipped_mean(
sources['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (
mean_position_angle, '[deg] PA of mean image ellipticity')
logs.add_tag(logging_tags, 'L1ELLIPA',
float(mean_position_angle))
self.logger.info('Extracted sources', extra=logging_tags)
except Exception as e:
logging_tags = logs.image_config_to_tags(
image, self.group_by_keywords)
logs.add_tag(logging_tags, 'filename',
os.path.basename(image.filename))
self.logger.error(e, extra=logging_tags)
return images
def find_stars(self, image: Image) -> Table:
"""Find stars in given image and append catalog.
Args:
image: Image to find stars in.
Returns:
Full table with results.
"""
import sep
# get data and make it continuous
data = image.data.copy()
# mask?
mask = image.mask.data if image.mask is not None else None
# estimate background, probably we need to byte swap, and subtract it
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError as e:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# extract sources
try:
sources = sep.extract(data, self.threshold, err=bkg.globalrms, minarea=self.minarea,
deblend_nthresh=self.deblend_nthresh, deblend_cont=self.deblend_cont,
clean=self.clean, clean_param=self.clean_param, mask=mask)
except:
log.exception('An error has occured.')
return Table()
# convert to astropy table
sources = Table(sources)
# only keep sources with detection flag < 8
sources = sources[sources['flag'] < 8]
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# calculate the FWHMs of the stars
fwhm = 2.0 * (np.log(2) * (sources['a'] ** 2.0 + sources['b'] ** 2.0)) ** 0.5
sources['fwhm'] = fwhm
# get gain
gain = image.header['DET-GAIN'] if 'DET-GAIN' in image.header else None
# Kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'], sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# equivalent of FLUX_AUTO
flux, fluxerr, flag = sep.sum_ellipse(data, sources['x'], sources['y'], sources['a'], sources['b'],
sources['theta'], 2.5 * kronrad, subpix=1, mask=mask,
err=bkg.rms(), gain=gain)
sources['flag'] |= flag
sources['flux'] = flux
sources['fluxerr'] = fluxerr
# radii at 0.25, 0.5, and 0.75 flux
flux_radii, flag = sep.flux_radius(data, sources['x'], sources['y'], 6.0 * sources['a'], [0.25, 0.5, 0.75],
normflux=sources['flux'], subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# xwin/ywin
sig = 2. / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'], sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# perform aperture photometry for diameters of 1" to 8"
for diameter in [1, 2, 3, 4, 5, 6, 7, 8]:
flux, fluxerr, flag = sep.sum_circle(data, sources['x'], sources['y'],
diameter / 2. / image.pixel_scale,
mask=mask, err=bkg.rms(), gain=gain)
sources['fluxaper{0}'.format(diameter)] = flux
sources['fluxerr{0}'.format(diameter)] = fluxerr
sources['flag'] |= flag
# average background at each source
# since SEP sums up whole pixels, we need to do the same on an image of ones for the background_area
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(), sources['x'], sources['y'],
sources['a'], sources['b'], np.pi / 2.0,
2.5 * sources['kronrad'], subpix=1)
background_area, _, _ = sep.sum_ellipse(np.ones(shape=bkg.back().shape), sources['x'], sources['y'],
sources['a'], sources['b'], np.pi / 2.0,
2.5 * sources['kronrad'], subpix=1)
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[background_area > 0]
# match fits conventions
sources['x'] += 1.0
sources['xpeak'] += 1
sources['xwin'] += 1.0
sources['xmin'] += 1
sources['xmax'] += 1
sources['y'] += 1.0
sources['ypeak'] += 1
sources['ywin'] += 1.0
sources['ymin'] += 1
sources['ymax'] += 1
sources['theta'] = np.degrees(sources['theta'])
# pick columns for catalog
cat = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
'flux', 'fluxerr', 'peak', 'fluxaper1', 'fluxerr1',
'fluxaper2', 'fluxerr2', 'fluxaper3', 'fluxerr3',
'fluxaper4', 'fluxerr4', 'fluxaper5', 'fluxerr5',
'fluxaper6', 'fluxerr6', 'fluxaper7', 'fluxerr7',
'fluxaper8', 'fluxerr8', 'background', 'fwhm',
'a', 'b', 'theta', 'kronrad', 'ellipticity',
'fluxrad25', 'fluxrad50', 'fluxrad75',
'x2', 'y2', 'xy', 'flag']
# set it
image.catalog = cat
# return full catalog
return sources
def _run_sep(self):
import sep
# THRESH=1.2 # in sky sigma
# DETECT_THRESH=1.6 # in sky sigma
# DEBLEND_MINCONT=0.005
# DETECT_MINAREA = 6 # minimum number of pixels above threshold
objs, seg = sep.extract(
self.detim,
self.detect_thresh,
err=self.detnoise,
segmentation_map=True,
**self.sx_config
)
logger.debug('found %d objects' % objs.size)
if objs.size == 0:
self.cat = objs
return None
flux_auto = np.zeros(objs.size)-9999.0
fluxerr_auto = np.zeros(objs.size)-9999.0
flux_radius = np.zeros(objs.size)-9999.0
kron_radius = np.zeros(objs.size)-9999.0
w, = np.where(
(objs['a'] >= 0.0) &
(objs['b'] >= 0.0) &
between(objs['theta'], -pi/2., pi/2., type='[]')
)
if w.size > 0:
kron_radius[w], krflag = sep.kron_radius(
self.detim,
objs['x'][w],
objs['y'][w],
objs['a'][w],
objs['b'][w],
objs['theta'][w],
6.0,
)
objs['flag'][w] |= krflag
aper_rad = 2.5*kron_radius
flux_auto[w], fluxerr_auto[w], flag_auto = \
sep.sum_ellipse(
self.detim,
objs['x'][w],
objs['y'][w],
objs['a'][w],
objs['b'][w],
objs['theta'][w],
aper_rad[w],
subpix=1,
)
objs['flag'][w] |= flag_auto
# what we did in DES, but note threshold above
# is 1 as opposed to wide survey. deep survey
# was even lower, 0.8?
# used half light radius
PHOT_FLUXFRAC = 0.5
flux_radius[w], frflag = sep.flux_radius(
self.detim,
objs['x'][w],
objs['y'][w],
6.*objs['a'][w],
PHOT_FLUXFRAC,
normflux=flux_auto[w],
subpix=5,
)
objs['flag'][w] |= frflag # combine flags into 'flag'
ncut = 2 # need this to make sure array
new_dt = [
('id', 'i8'),
('number', 'i4'),
('ncutout', 'i4'),
('kron_radius', 'f4'),
('flux_auto', 'f4'),
('fluxerr_auto', 'f4'),
('flux_radius', 'f4'),
('isoarea_image', 'f4'),
('iso_radius', 'f4'),
('box_size', 'i4'),
('file_id', 'i8', ncut),
('orig_row', 'f4', ncut),
('orig_col', 'f4', ncut),
('orig_start_row', 'i8', ncut),
('orig_start_col', 'i8', ncut),
('orig_end_row', 'i8', ncut),
('orig_end_col', 'i8', ncut),
('cutout_row', 'f4', ncut),
('cutout_col', 'f4', ncut),
('dudrow', 'f8', ncut),
('dudcol', 'f8', ncut),
('dvdrow', 'f8', ncut),
('dvdcol', 'f8', ncut),
]
cat = eu.numpy_util.add_fields(objs, new_dt)
cat['id'] = np.arange(cat.size)
cat['number'] = np.arange(1, cat.size+1)
cat['ncutout'] = 1
cat['flux_auto'] = flux_auto
cat['fluxerr_auto'] = fluxerr_auto
cat['flux_radius'] = flux_radius
wcs = self.datalist[0]['wcs']
cat['dudrow'][:, 0] = wcs.dudy
cat['dudcol'][:, 0] = wcs.dudx
cat['dvdrow'][:, 0] = wcs.dvdy
cat['dvdcol'][:, 0] = wcs.dvdx
# use the number of pixels in the seg map as the iso area
for i in range(objs.size):
w = np.where(seg == (i+1))
cat['isoarea_image'][i] = w[0].size
cat['iso_radius'] = np.sqrt(cat['isoarea_image'].clip(min=1)/np.pi)
box_size = self._get_box_sizes(cat)
half_box_size = box_size//2
maxrow, maxcol = self.detim.shape
cat['box_size'] = box_size
cat['orig_row'][:, 0] = cat['y']
cat['orig_col'][:, 0] = cat['x']
orow = cat['orig_row'][:, 0].astype('i4')
ocol = cat['orig_col'][:, 0].astype('i4')
ostart_row = orow - half_box_size + 1
ostart_col = ocol - half_box_size + 1
oend_row = orow + half_box_size + 1 # plus one for slices
oend_col = ocol + half_box_size + 1
ostart_row.clip(min=0, out=ostart_row)
ostart_col.clip(min=0, out=ostart_col)
oend_row.clip(max=maxrow, out=oend_row)
oend_col.clip(max=maxcol, out=oend_col)
# could result in smaller than box_size above
cat['orig_start_row'][:, 0] = ostart_row
cat['orig_start_col'][:, 0] = ostart_col
cat['orig_end_row'][:, 0] = oend_row
cat['orig_end_col'][:, 0] = oend_col
cat['cutout_row'][:, 0] = \
cat['orig_row'][:, 0] - cat['orig_start_row'][:, 0]
cat['cutout_col'][:, 0] = \
cat['orig_col'][:, 0] - cat['orig_start_col'][:, 0]
self.seg = seg
self.bmask = np.zeros(seg.shape, dtype='i4')
self.cat = cat
def process_file(filename, favor2=None, verbose=False, replace=False, dbname=None, dbhost=None, photodir='photometry'):
#### Some parameters
aper = 2.0
bkgann = None
order = 4
bg_order = 4
color_order = 2
sn = 5
if not posixpath.exists(filename):
return None
# Rough but fast checking of whether the file is already processed
if not replace and posixpath.exists(photodir + '/' + filename.split('/')[-2] + '/' + posixpath.splitext(posixpath.split(filename)[-1])[0] + '.cat'):
return
#### Preparation
header = fits.getheader(filename, -1)
if header['TYPE'] not in ['survey', 'imaging', 'widefield', 'Swift', 'Fermi', 'test']:
return
channel = header.get('CHANNEL ID')
fname = header.get('FILTER', 'unknown')
time = parse_time(header['TIME'])
shutter = header.get('SHUTTER', -1)
if fname not in ['Clear']:
return
if fname == 'Clear':
effective_fname = 'V'
else:
effective_fname = fname
night = get_night(time)
dirname = '%s/%s' % (photodir, night)
basename = posixpath.splitext(posixpath.split(filename)[-1])[0]
basename = dirname + '/' + basename
catname = basename + '.cat'
if not replace and posixpath.exists(catname):
return
if verbose:
print(filename, channel, night, fname, effective_fname)
image = fits.getdata(filename, -1).astype(np.double)
if favor2 is None:
favor2 = Favor2(dbname=options.db, dbhost=options.dbhost)
#### Basic calibration
darkname = favor2.find_image('masterdark', header=header, debug=False)
flatname = favor2.find_image('masterflat', header=header, debug=False)
if darkname:
dark = fits.getdata(darkname)
else:
dark = None
if flatname:
flat = fits.getdata(flatname)
else:
flat = None
if dark is None or flat is None:
survey.save_objects(catname, None)
return
image,header = calibrate.calibrate(image, header, dark=dark)
# Check whether the calibration failed
if 'SATURATE' not in header:
print('Calibration failed for', filename)
survey.save_objects(catname, None)
return
#### Basic masking
mask = image > 0.9*header['SATURATE']
fmask = ~np.isfinite(flat) | (flat < 0.5)
dmask = dark > 10.0*mad_std(dark) + np.nanmedian(dark)
if header.get('BLEMISHCORRECTION', 1):
# We have to mask blemished pixels
blemish = fits.getdata('calibrations/blemish_shutter_%d_channel_%d.fits' % (header['SHUTTER'], header['CHANNEL ID']))
if verbose:
print(100*np.sum(blemish>0)/blemish.shape[0]/blemish.shape[1], '% pixels blemished')
dmask |= blemish > 0
image[~fmask] *= np.median(flat[~fmask])/flat[~fmask]
#### WCS
wcs = WCS(header)
pixscale = np.hypot(wcs.pixel_scale_matrix[0,0], wcs.pixel_scale_matrix[0,1])
gain = 0.67 if header.get('SHUTTER') == 0 else 1.9
#### Background mask
mask_bg = np.zeros_like(mask)
mask_segm = np.zeros_like(mask)
# bg2 = sep.Background(image, mask=mask|mask_bg, bw=64, bh=64)
# for _ in xrange(3):
# bg1 = sep.Background(image, mask=mask|mask_bg, bw=256, bh=256)
# ibg = bg2.back() - bg1.back()
# tmp = np.abs(ibg - np.median(ibg)) > 5.0*mad_std(ibg)
# mask_bg |= survey.dilate(tmp, np.ones([50, 50]))
# mask_bg = survey.dilate(tmp, np.ones([50, 50]))
# Large objects?..
bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=128, bh=128)
image1 = image - bg.back()
obj0,segm = sep.extract(image1, err=bg.rms(), thresh=10, minarea=10, mask=mask|dmask|fmask|mask_bg, filter_kernel=None, clean=False, segmentation_map=True)
mask_segm = np.isin(segm, [_+1 for _,npix in enumerate(obj0['npix']) if npix > 500])
mask_segm = survey.dilate(mask_segm, np.ones([20, 20]))
if np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1] > 0.4:
print(100*np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1], '% of image masked, skipping', filename)
survey.save_objects(catname, None)
return
elif verbose:
print(100*np.sum(mask_bg|mask_segm|mask|fmask|dmask)/mask_bg.shape[0]/mask_bg.shape[1], '% of image masked')
# Frame footprint at +10 pixels from the edge
ra,dec = wcs.all_pix2world([10, 10, image.shape[1]-10, image.shape[1]-10], [10, image.shape[0]-10, image.shape[0]-10, 10], 0)
footprint = "(" + ",".join(["(%g,%g)" % (_,__) for _,__ in zip(ra, dec)]) + ")"
#### Catalogue
ra0,dec0,sr0 = survey.get_frame_center(header=header)
cat = favor2.get_stars(ra0, dec0, sr0, catalog='gaia', extra=['g<14', 'q3c_poly_query(ra, dec, \'%s\'::polygon)' % footprint], limit=1000000)
if verbose:
print(len(cat['ra']), 'star positions from Gaia down to g=%.1f mag' % np.max(cat['g']))
## Detection of blended and not really needed stars in the catalogue
h = htm.HTM(10)
m = h.match(cat['ra'], cat['dec'], cat['ra'], cat['dec'], 2.0*aper*pixscale, maxmatch=0)
m = [_[m[2]>1e-5] for _ in m]
blended = np.zeros_like(cat['ra'], dtype=np.bool)
notneeded = np.zeros_like(cat['ra'], dtype=np.bool)
for i1,i2,dist in zip(*m):
if dist*3600 > 0.5*aper*pixscale:
if cat['g'][i1] - cat['g'][i2] < 3:
blended[i1] = True
blended[i2] = True
else:
# i1 is fainter by more than 3 mag
notneeded[i1] = True
if dist*3600 < 0.5*aper*pixscale:
if cat['g'][i1] > cat['g'][i2]:
notneeded[i1] = True
cat,blended = [_[~notneeded] for _ in cat,blended]
#### Background subtraction
bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=128, bh=128)
# bg = sep.Background(image, mask=mask|dmask|fmask|mask_bg|mask_segm, bw=32, bh=32)
image1 = image - bg.back()
#### Detection of all objects on the frame
obj0,segm = sep.extract(image1, err=bg.rms(), thresh=2, minarea=3, mask=mask|dmask|fmask|mask_bg|mask_segm, filter_kernel=None, clean=False, segmentation_map=True)
obj0 = obj0[(obj0['x'] > 10) & (obj0['y'] > 10) & (obj0['x'] < image.shape[1]-10) & (obj0['y'] < image.shape[0]-10)]
obj0 = obj0[obj0['flag'] <= 1] # We keep only normal and blended oblects
fields = ['ra', 'dec', 'fluxerr', 'mag', 'magerr', 'flags', 'cat']
obj0 = np.lib.recfunctions.append_fields(obj0, fields, [np.zeros_like(obj0['x'], dtype=np.int if _ in ['flags', 'cat'] else np.double) for _ in fields], usemask=False)
obj0['ra'],obj0['dec'] = wcs.all_pix2world(obj0['x'], obj0['y'], 0)
obj0['flags'] = obj0['flag']
if verbose:
print(len(obj0['x']), 'objects detected on the frame')
## Filter out objects not coincident with catalogue positions
h = htm.HTM(10)
m = h.match(obj0['ra'], obj0['dec'], cat['ra'], cat['dec'], aper*pixscale)
nidx = np.isin(np.arange(len(obj0['ra'])), m[0], invert=True)
obj0 = obj0[nidx]
if verbose:
print(len(obj0['x']), 'are outside catalogue apertures')
# Catalogue stars
xc,yc = wcs.all_world2pix(cat['ra'], cat['dec'], 0)
obj = {'x':xc, 'y':yc, 'ra':cat['ra'], 'dec':cat['dec']}
obj['flags'] = np.zeros_like(xc, dtype=np.int)
obj['flags'][blended] |= FLAG_BLENDED
obj['cat'] = np.ones_like(xc, dtype=np.int)
for _ in ['mag', 'magerr', 'flux', 'fluxerr']:
obj[_] = np.zeros_like(xc)
# Merge detected objects
for _ in ['x', 'y', 'ra', 'dec', 'flags', 'mag', 'magerr', 'flux', 'fluxerr', 'cat']:
obj[_] = np.concatenate((obj[_], obj0[_]))
if verbose:
print(len(obj['x']), 'objects for photometry')
# Simple aperture photometry
obj['flux'],obj['fluxerr'],flag = sep.sum_circle(image1, obj['x'], obj['y'], aper, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm, bkgann=bkgann)
obj['flags'] |= flag
# Normalize flags
obj['flags'][obj['flags'] & sep.APER_TRUNC] |= FLAG_TRUNCATED
obj['flags'][obj['flags'] & sep.APER_ALLMASKED] |= FLAG_MASKED
obj['flags'] &= FLAG_NORMAL | FLAG_BLENDED | FLAG_TRUNCATED | FLAG_MASKED | FLAG_NO_BACKGROUND | FLAG_BAD_CALIBRATION
area,_,_ = sep.sum_circle(np.ones_like(image1), obj['x'], obj['y'], aper, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm, bkgann=bkgann)
# Simple local background estimation
bgflux,bgfluxerr,bgflag = sep.sum_circann(image1, obj['x'], obj['y'], 10, 15, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm|(segm>0))
bgarea,_,_ = sep.sum_circann(np.ones_like(image1), obj['x'], obj['y'], 10, 15, err=bg.rms(), gain=gain, mask=mask|dmask|fmask|mask_bg|mask_segm|(segm>0))
bgidx = np.isfinite(bgarea) & np.isfinite(area)
bgidx[bgidx] &= (bgarea[bgidx] > 10) & (area[bgidx] > 1)
obj['flux'][bgidx] -= bgflux[bgidx]*area[bgidx]/bgarea[bgidx]
obj['flags'][~bgidx] |= FLAG_NO_BACKGROUND # No local background
obj['deltabgflux'] = np.zeros_like(obj['x'])
obj['deltabgflux'][bgidx] = bgflux[bgidx]*area[bgidx]/bgarea[bgidx]
fidx = np.isfinite(obj['flux']) & np.isfinite(obj['fluxerr'])
fidx[fidx] &= (obj['flux'][fidx] > 0)
obj['mag'][fidx] = -2.5*np.log10(obj['flux'][fidx])
obj['magerr'][fidx] = 2.5/np.log(10)*obj['fluxerr'][fidx]/obj['flux'][fidx]
fidx[fidx] &= (obj['magerr'][fidx] > 0)
fidx[fidx] &= 1/obj['magerr'][fidx] > sn
for _ in obj.keys():
if hasattr(obj[_], '__len__'):
obj[_] = obj[_][fidx]
obj['aper'] = aper
if verbose:
print(len(obj['x']), 'objects with S/N >', sn)
if len(obj['x']) < 1000:
print('Only', len(obj['x']), 'objects on the frame, skipping', filename)
survey.save_objects(catname, None)
return
obj['fwhm'] = 2.0*sep.flux_radius(image1, obj['x'], obj['y'], 2.0*aper*np.ones_like(obj['x']), 0.5, mask=mask|dmask|fmask|mask_bg|mask_segm)[0]
#### Check FWHM of all objects and select only 'good' ones
idx = obj['flags'] == 0
idx &= obj['magerr'] < 1/20
fwhm0 = survey.fit_2d(obj['x'][idx], obj['y'][idx], obj['fwhm'][idx], obj['x'], obj['y'], weights=1/obj['magerr'][idx])
fwhm_idx = np.abs(obj['fwhm'] - fwhm0 - np.median((obj['fwhm'] - fwhm0)[idx])) < 3.0*mad_std((obj['fwhm'] - fwhm0)[idx])
obj['flags'][~fwhm_idx] |= FLAG_BLENDED
#### Catalogue matching
idx = obj['flags']
m = htm.HTM(10).match(obj['ra'], obj['dec'], cat['ra'], cat['dec'], 1e-5)
fidx = np.in1d(np.arange(len(cat['ra'])), m[1]) # Stars that got successfully measured and not blended
cidx = (cat['good'] == 1) & (cat['var'] == 0)
cidx &= np.isfinite(cat['B']) & np.isfinite(cat['V']) # & np.isfinite(cat['lum'])
cidx[cidx] &= ((cat['B'] - cat['V'])[cidx] > -0.5) & ((cat['B'] - cat['V'])[cidx] < 2.0)
# cidx[cidx] &= (cat['lum'][cidx] > 0.3) & (cat['lum'][cidx] < 30)
if np.sum(cidx & fidx & (cat['multi_70'] == 0)) > 2000:
cidx &= (cat['multi_70'] == 0)
obj['cat_multi'] = 70
elif np.sum(cidx & fidx & (cat['multi_45'] == 0)) > 1000:
cidx &= (cat['multi_45'] == 0)
obj['cat_multi'] = 45
else:
cidx &= (cat['multi_30'] == 0)
obj['cat_multi'] = 30
if verbose:
print(np.sum(obj['flags'] == 0), 'objects without flags')
print('Amount of good stars:',
np.sum(cidx & fidx & (cat['multi_70'] == 0)),
np.sum(cidx & fidx & (cat['multi_45'] == 0)),
np.sum(cidx & fidx & (cat['multi_30'] == 0)))
print('Using %d arcsec avoidance radius' % obj['cat_multi'])
# We match with very small SR to only account for manually placed apertures
if verbose:
print('Trying full fit:', len(obj['x']), 'objects,', np.sum(cidx), 'stars')
match = Match(width=image.shape[1], height=image.shape[0])
prev_ngoodstars = len(obj['x'])
for iter in range(10):
if not match.match(obj=obj, cat=cat[cidx], sr=1e-5, filter_name='V', order=order, bg_order=bg_order, color_order=color_order, verbose=False) or match.ngoodstars < 500:
if verbose:
print(match.ngoodstars, 'good matches, matching failed for', filename)
survey.save_objects(catname, None)
return
if verbose:
print(match.ngoodstars, 'good matches, std =', match.std)
if match.ngoodstars == prev_ngoodstars:
if verbose:
print('Converged on iteration', iter)
break
prev_ngoodstars = match.ngoodstars
# Match good objects with stars
oidx = obj['flags'] == 0
oidx1,cidx1,dist1 = htm.HTM(10).match(obj['ra'][oidx], obj['dec'][oidx], cat['ra'][cidx], cat['dec'][cidx], 1e-5)
x = obj['x'][oidx][oidx1]
y = obj['y'][oidx][oidx1]
cbv = match.color_term[oidx][oidx1]
cbv2 = match.color_term2[oidx][oidx1]
cbv3 = match.color_term3[oidx][oidx1]
bv = (cat['B'] - cat['V'])[cidx][cidx1]
cmag = cat[match.cat_filter_name][cidx][cidx1]
mag = match.mag[oidx][oidx1] + bv*cbv + bv**2*cbv2 + bv**3*cbv3
magerr = np.hypot(obj['magerr'][oidx][oidx1], 0.02)
dmag = mag-cmag
ndmag = ((mag-cmag)/magerr)
idx = cmag < match.mag_limit[oidx][oidx1]
x,y,cbv,cbv2,cbv3,bv,cmag,mag,magerr,dmag,ndmag = [_[idx] for _ in [x,y,cbv,cbv2,cbv3,bv,cmag,mag,magerr,dmag,ndmag]]
# Match all objects with good objects
xy = np.array([x,y]).T
xy0 = np.array([obj['x'], obj['y']]).T
kd = cKDTree(xy)
dist,m = kd.query(xy0, 101)
dist = dist[:,1:]
m = m[:,1:]
vchi2 = mad_std(ndmag[m]**2, axis=1)
# Mark regions of too sparse or too noisy matches as bad
obj['flags'][vchi2 > 5] |= FLAG_BAD_CALIBRATION
# obj['flags'][vchi2 > np.median(vchi2) + 5.0*mad_std(vchi2)] |= FLAG_BAD_CALIBRATION
obj['flags'][dist[:,10] > np.median(dist[:,10]) + 10.0*mad_std(dist[:,10])] |= FLAG_BAD_CALIBRATION
match.good_idx = (obj['flags'] & FLAG_BAD_CALIBRATION) == 0
if verbose:
print(np.sum(match.good_idx), 'of', len(match.good_idx), 'stars are good')
#### Store objects to file
try:
os.makedirs(dirname)
except:
pass
obj['mag_limit'] = match.mag_limit
obj['color_term'] = match.color_term
obj['color_term2'] = match.color_term2
obj['color_term3'] = match.color_term3
obj['filename'] = filename
obj['night'] = night
obj['channel'] = channel
obj['filter'] = fname
obj['cat_filter'] = match.cat_filter_name
obj['time'] = time
obj['mag_id'] = match.mag_id
obj['good_idx'] = match.good_idx
obj['calib_mag'] = match.mag
obj['calib_magerr'] = match.magerr
obj['std'] = match.std
obj['nstars'] = match.ngoodstars
survey.save_objects(catname, obj, header=header)
def do_stage(self, image):
try:
# Set the number of source pixels to be 5% of the total. This keeps us safe from
# satellites and airplanes.
sep.set_extract_pixstack(int(image.nx * image.ny * 0.05))
data = image.data.copy()
error = (np.abs(data) + image.readnoise**2.0)**0.5
mask = image.bpm > 0
# Fits can be backwards byte order, so fix that if need be and subtract
# the background
try:
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
except ValueError:
data = data.byteswap(True).newbyteorder()
bkg = sep.Background(data, mask=mask, bw=32, bh=32, fw=3, fh=3)
bkg.subfrom(data)
# Do an initial source detection
# TODO: Add back in masking after we are sure SEP works
sources = sep.extract(data,
self.threshold,
minarea=self.min_area,
err=error,
deblend_cont=0.005)
# Convert the detections into a table
sources = Table(sources)
# We remove anything with a detection flag >= 8
# This includes memory overflows and objects that are too close the edge
sources = sources[sources['flag'] < 8]
sources = array_utils.prune_nans_from_table(sources)
# Calculate the ellipticity
sources['ellipticity'] = 1.0 - (sources['b'] / sources['a'])
# Fix any value of theta that are invalid due to floating point rounding
# -pi / 2 < theta < pi / 2
sources['theta'][sources['theta'] > (np.pi / 2.0)] -= np.pi
sources['theta'][sources['theta'] < (-np.pi / 2.0)] += np.pi
# Calculate the kron radius
kronrad, krflag = sep.kron_radius(data, sources['x'], sources['y'],
sources['a'], sources['b'],
sources['theta'], 6.0)
sources['flag'] |= krflag
sources['kronrad'] = kronrad
# Calcuate the equivilent of flux_auto
flux, fluxerr, flag = sep.sum_ellipse(data,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * kronrad,
subpix=1,
err=error)
sources['flux'] = flux
sources['fluxerr'] = fluxerr
sources['flag'] |= flag
# Do circular aperture photometry for diameters of 1" to 6"
for diameter in [1, 2, 3, 4, 5, 6]:
flux, fluxerr, flag = sep.sum_circle(data,
sources['x'],
sources['y'],
diameter / 2.0 /
image.pixel_scale,
gain=1.0,
err=error)
sources['fluxaper{0}'.format(diameter)] = flux
sources['fluxerr{0}'.format(diameter)] = fluxerr
sources['flag'] |= flag
# Calculate the FWHMs of the stars:
fwhm = 2.0 * (np.log(2) *
(sources['a']**2.0 + sources['b']**2.0))**0.5
sources['fwhm'] = fwhm
# Cut individual bright pixels. Often cosmic rays
sources = sources[fwhm > 1.0]
# Measure the flux profile
flux_radii, flag = sep.flux_radius(data,
sources['x'],
sources['y'],
6.0 * sources['a'],
[0.25, 0.5, 0.75],
normflux=sources['flux'],
subpix=5)
sources['flag'] |= flag
sources['fluxrad25'] = flux_radii[:, 0]
sources['fluxrad50'] = flux_radii[:, 1]
sources['fluxrad75'] = flux_radii[:, 2]
# Calculate the windowed positions
sig = 2.0 / 2.35 * sources['fluxrad50']
xwin, ywin, flag = sep.winpos(data, sources['x'], sources['y'],
sig)
sources['flag'] |= flag
sources['xwin'] = xwin
sources['ywin'] = ywin
# Calculate the average background at each source
bkgflux, fluxerr, flag = sep.sum_ellipse(bkg.back(),
sources['x'],
sources['y'],
sources['a'],
sources['b'],
np.pi / 2.0,
2.5 * sources['kronrad'],
subpix=1)
# masksum, fluxerr, flag = sep.sum_ellipse(mask, sources['x'], sources['y'],
# sources['a'], sources['b'], np.pi / 2.0,
# 2.5 * kronrad, subpix=1)
background_area = (
2.5 * sources['kronrad']
)**2.0 * sources['a'] * sources['b'] * np.pi # - masksum
sources['background'] = bkgflux
sources['background'][background_area > 0] /= background_area[
background_area > 0]
# Update the catalog to match fits convention instead of python array convention
sources['x'] += 1.0
sources['y'] += 1.0
sources['xpeak'] += 1
sources['ypeak'] += 1
sources['xwin'] += 1.0
sources['ywin'] += 1.0
sources['theta'] = np.degrees(sources['theta'])
catalog = sources['x', 'y', 'xwin', 'ywin', 'xpeak', 'ypeak',
'flux', 'fluxerr', 'peak', 'fluxaper1',
'fluxerr1', 'fluxaper2', 'fluxerr2', 'fluxaper3',
'fluxerr3', 'fluxaper4', 'fluxerr4', 'fluxaper5',
'fluxerr5', 'fluxaper6', 'fluxerr6',
'background', 'fwhm', 'a', 'b', 'theta',
'kronrad', 'ellipticity', 'fluxrad25',
'fluxrad50', 'fluxrad75', 'x2', 'y2', 'xy',
'flag']
# Add the units and description to the catalogs
catalog['x'].unit = 'pixel'
catalog['x'].description = 'X coordinate of the object'
catalog['y'].unit = 'pixel'
catalog['y'].description = 'Y coordinate of the object'
catalog['xwin'].unit = 'pixel'
catalog['xwin'].description = 'Windowed X coordinate of the object'
catalog['ywin'].unit = 'pixel'
catalog['ywin'].description = 'Windowed Y coordinate of the object'
catalog['xpeak'].unit = 'pixel'
catalog['xpeak'].description = 'X coordinate of the peak'
catalog['ypeak'].unit = 'pixel'
catalog['ypeak'].description = 'Windowed Y coordinate of the peak'
catalog['flux'].unit = 'count'
catalog[
'flux'].description = 'Flux within a Kron-like elliptical aperture'
catalog['fluxerr'].unit = 'count'
catalog[
'fluxerr'].description = 'Error on the flux within Kron aperture'
catalog['peak'].unit = 'count'
catalog['peak'].description = 'Peak flux (flux at xpeak, ypeak)'
for diameter in [1, 2, 3, 4, 5, 6]:
catalog['fluxaper{0}'.format(diameter)].unit = 'count'
catalog['fluxaper{0}'.format(
diameter
)].description = 'Flux from fixed circular aperture: {0}" diameter'.format(
diameter)
catalog['fluxerr{0}'.format(diameter)].unit = 'count'
catalog['fluxerr{0}'.format(
diameter
)].description = 'Error on Flux from circular aperture: {0}"'.format(
diameter)
catalog['background'].unit = 'count'
catalog[
'background'].description = 'Average background value in the aperture'
catalog['fwhm'].unit = 'pixel'
catalog['fwhm'].description = 'FWHM of the object'
catalog['a'].unit = 'pixel'
catalog['a'].description = 'Semi-major axis of the object'
catalog['b'].unit = 'pixel'
catalog['b'].description = 'Semi-minor axis of the object'
catalog['theta'].unit = 'degree'
catalog['theta'].description = 'Position angle of the object'
catalog['kronrad'].unit = 'pixel'
catalog['kronrad'].description = 'Kron radius used for extraction'
catalog['ellipticity'].description = 'Ellipticity'
catalog['fluxrad25'].unit = 'pixel'
catalog[
'fluxrad25'].description = 'Radius containing 25% of the flux'
catalog['fluxrad50'].unit = 'pixel'
catalog[
'fluxrad50'].description = 'Radius containing 50% of the flux'
catalog['fluxrad75'].unit = 'pixel'
catalog[
'fluxrad75'].description = 'Radius containing 75% of the flux'
catalog['x2'].unit = 'pixel^2'
catalog[
'x2'].description = 'Variance on X coordinate of the object'
catalog['y2'].unit = 'pixel^2'
catalog[
'y2'].description = 'Variance on Y coordinate of the object'
catalog['xy'].unit = 'pixel^2'
catalog['xy'].description = 'XY covariance of the object'
catalog[
'flag'].description = 'Bit mask of extraction/photometry flags'
catalog.sort('flux')
catalog.reverse()
# Save some background statistics in the header
mean_background = stats.sigma_clipped_mean(bkg.back(), 5.0)
image.header['L1MEAN'] = (
mean_background,
'[counts] Sigma clipped mean of frame background')
median_background = np.median(bkg.back())
image.header['L1MEDIAN'] = (median_background,
'[counts] Median of frame background')
std_background = stats.robust_standard_deviation(bkg.back())
image.header['L1SIGMA'] = (
std_background, '[counts] Robust std dev of frame background')
# Save some image statistics to the header
good_objects = catalog['flag'] == 0
for quantity in ['fwhm', 'ellipticity', 'theta']:
good_objects = np.logical_and(
good_objects, np.logical_not(np.isnan(catalog[quantity])))
if good_objects.sum() == 0:
image.header['L1FWHM'] = ('NaN',
'[arcsec] Frame FWHM in arcsec')
image.header['L1ELLIP'] = ('NaN',
'Mean image ellipticity (1-B/A)')
image.header['L1ELLIPA'] = (
'NaN', '[deg] PA of mean image ellipticity')
else:
seeing = np.median(
catalog['fwhm'][good_objects]) * image.pixel_scale
image.header['L1FWHM'] = (seeing,
'[arcsec] Frame FWHM in arcsec')
mean_ellipticity = stats.sigma_clipped_mean(
catalog['ellipticity'][good_objects], 3.0)
image.header['L1ELLIP'] = (mean_ellipticity,
'Mean image ellipticity (1-B/A)')
mean_position_angle = stats.sigma_clipped_mean(
catalog['theta'][good_objects], 3.0)
image.header['L1ELLIPA'] = (
mean_position_angle, '[deg] PA of mean image ellipticity')
logging_tags = {
key: float(image.header[key])
for key in [
'L1MEAN', 'L1MEDIAN', 'L1SIGMA', 'L1FWHM', 'L1ELLIP',
'L1ELLIPA'
]
}
logger.info('Extracted sources',
image=image,
extra_tags=logging_tags)
# adding catalog (a data table) to the appropriate images attribute.
image.data_tables['catalog'] = DataTable(data_table=catalog,
name='CAT')
except Exception:
logger.error(logs.format_exception(), image=image)
return image
def AutoPhot(self, Kron_fact=2.5, min_diameter=3.5, write=True):
kronrad, krflag = sep.kron_radius(self.dat, self.src['x'],
self.src['y'], self.src['a'],
self.src['b'], self.src['theta'], 6.)
kronrad[np.isnan(kronrad) == True] = 0.
flux, fluxerr, flag = sep.sum_ellipse(self.dat,
self.src['x'],
self.src['y'],
self.src['a'],
self.src['b'],
self.src['theta'],
Kron_fact * kronrad,
err=self.skysigma,
gain=self.gain,
subpix=0)
flag |= krflag # Combining flags
r_min = 0.5 * min_diameter
use_circle = kronrad * np.sqrt(self.src['a'] * self.src['b']) < r_min
cflux, cfluxerr, cflag = sep.sum_circle(self.dat,
self.src['x'][use_circle],
self.src['y'][use_circle],
r_min,
err=self.skysigma,
gain=self.gain,
subpix=0)
flux[use_circle] = cflux
fluxerr[use_circle] = cfluxerr
flag[use_circle] = cflag
mag = self.zmag - 2.5 * np.log10(flux)
magerr = (2.5 / np.log(10.0)) * (fluxerr / flux)
r, flag = sep.flux_radius(self.dat,
self.src['x'],
self.src['y'],
6.0 * self.src['a'],
0.5,
normflux=flux,
subpix=5)
ra, dec = self.wcs.all_pix2world(self.src['x'] + 1, self.src['y'] + 1,
1)
df = pd.DataFrame(
data={
'x': self.src['x'],
'y': self.src['y'],
'ra': ra,
'dec': dec,
'a': self.src['a'],
'b': self.src['b'],
'theta': self.src['theta'],
'flux': flux,
'e_flux': fluxerr,
'mag': mag,
'e_mag': magerr,
'kronrad': kronrad,
'flxrad': r,
'flag': flag
})
if write:
df.to_csv(ip.tmp_dir + 'auto_' + self.img.split('.fits')[0] +
'.csv')
f = open(ip.tmp_dir + 'src_' + self.img.split('.fits')[0] + '.reg',
'w')
for i in np.arange(self.nsrc):
f.write('{0:.3f} {1:.3f}\n'.format(self.src['x'][i] + 1,
self.src['y'][i] + 1))
f.close()
return df
def _measure(self, img, sources, mask=None):
logger.info('measuring source parameters')
# HACK: issues with numerical precision
# must have pi/2 <= theta <= npi/2
sources[np.abs(np.abs(sources['theta']) -
np.pi / 2) < 1e-6] = np.pi / 2
for p in ['x', 'y', 'a', 'b', 'theta']:
sources = sources[~np.isnan(sources[p])]
# calculate "AUTO" parameters
kronrad, krflag = sep.kron_radius(img,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
sources['theta'],
6.0,
mask=mask)
flux, fluxerr, flag = sep.sum_ellipse(img,
sources['x'],
sources['y'],
sources['a'],
sources['b'],
sources['theta'],
2.5 * kronrad,
subpix=5,
mask=mask)
flag |= krflag # combine flags into 'flag'
sources = sources[~np.isnan(flux)]
flux = flux[~np.isnan(flux)]
sources = sources[flux > 0]
flux = flux[flux > 0]
mag_auto = self.zpt - 2.5 * np.log10(flux)
r, flag = sep.flux_radius(img,
sources['x'],
sources['y'],
6. * sources['a'],
0.5,
normflux=flux,
subpix=5,
mask=mask)
sources['mag_auto'] = mag_auto
sources['flux_auto'] = flux
sources['flux_radius'] = r * self.pixscale
# approximate fwhm
r_squared = sources['a']**2 + sources['b']**2
sources['fwhm'] = 2 * np.sqrt(np.log(2) * r_squared) * self.pixscale
q = sources['b'] / sources['a']
area = np.pi * q * sources['flux_radius']**2
sources['mu_ave_auto'] = sources['mag_auto'] + 2.5 * np.log10(2 * area)
area_arcsec = np.pi * (self.psf_fwhm / 2)**2 * self.pixscale**2
flux, fluxerr, flag = sep.sum_circle(img,
sources['x'],
sources['y'],
self.psf_fwhm / 2,
subpix=5,
mask=mask)
flux[flux <= 0] = np.nan
mu_0 = self.zpt - 2.5 * np.log10(flux / area_arcsec)
sources['mu_0_aper'] = mu_0
return sources
def test_masked_segmentation_measurements():
"""Test measurements with segmentation masking"""
NX = 100
data = np.zeros((NX * 2, NX * 2))
yp, xp = np.indices(data.shape)
####
# Make two 2D gaussians that slightly overlap
# width of the 2D objects
gsigma = 10.
# offset between two gaussians in sigmas
off = 4
for xy in [[NX, NX], [NX + off * gsigma, NX + off * gsigma]]:
R = np.sqrt((xp - xy[0])**2 + (yp - xy[1])**2)
g_i = np.exp(-R**2 / 2 / gsigma**2)
data += g_i
# Absolute total
total_exact = g_i.sum()
# Add some noise
rms = 0.02
np.random.seed(1)
data += np.random.normal(size=data.shape) * rms
# Run source detection
objs, segmap = sep.extract(data,
thresh=1.2,
err=rms,
mask=None,
segmentation_map=True)
seg_id = np.arange(1, len(objs) + 1, dtype=np.int32)
# Compute Kron/Auto parameters
x, y, a, b = objs['x'], objs['y'], objs['a'], objs['b']
theta = objs['theta']
kronrad, krflag = sep.kron_radius(data, x, y, a, b, theta, 6.0)
flux_auto, fluxerr, flag = sep.sum_ellipse(data,
x,
y,
a,
b,
theta,
2.5 * kronrad,
segmap=segmap,
seg_id=seg_id,
subpix=1)
# Test total flux
assert_allclose(flux_auto, total_exact, rtol=5.e-2)
# Flux_radius
for flux_fraction in [0.2, 0.5]:
# Exact solution
rhalf_exact = np.sqrt(-np.log(1 - flux_fraction) * gsigma**2 * 2)
# Masked measurement
flux_radius, flag = sep.flux_radius(data,
x,
y,
6. * a,
flux_fraction,
seg_id=seg_id,
segmap=segmap,
normflux=flux_auto,
subpix=5)
# Test flux fraction
assert_allclose(flux_radius, rhalf_exact, rtol=5.e-2)
if False:
print('test_masked_flux_radius')
print(total_exact, flux_auto)
print(rhalf_exact, flux_radius)
def evaluate_sky(img, sigma=1.5, radius=10, pixel_scale=0.168, central_mask_radius=7.0,
threshold=0.005, deblend_cont=0.001, deblend_nthresh=20,
clean_param=1.0, show_fig=True, show_hist=True, f_factor=None):
'''Evaluate the mean sky value.
Parameters:
----------
img: 2-D numpy array, the input image
show_fig: bool. If True, it will show you the masked sky image.
show_hist: bool. If True, it will show you the histogram of the sky value.
Returns:
-------
median: median of background pixels, in original unit
std: standard deviation, in original unit
'''
import sep
import copy
from slug.imutils import extract_obj, make_binary_mask
from astropy.convolution import convolve, Gaussian2DKernel
b = 35 # Box size
f = 5 # Filter width
bkg = sep.Background(img, maskthresh=0, bw=b, bh=b, fw=f, fh=f)
# first time
objects, segmap = extract_obj(img - bkg.globalback, b=35, f=5, sigma=sigma,
minarea=20, pixel_scale=pixel_scale,
deblend_nthresh=deblend_nthresh, deblend_cont=deblend_cont,
clean_param=clean_param, show_fig=False)
seg_sky = copy.deepcopy(segmap)
seg_sky[segmap > 0] = 1
seg_sky = seg_sky.astype(bool)
# Blow up the mask
for obj in objects:
sep.mask_ellipse(seg_sky, obj['x'], obj['y'], obj['a'], obj['b'], obj['theta'], r=radius)
bkg_mask_1 = seg_sky
data = copy.deepcopy(img - bkg.globalback)
data[bkg_mask_1 == 1] = 0
# Second time
obj_lthre, seg_lthre = extract_obj(data, b=35, f=5, sigma=sigma + 1,
minarea=5, pixel_scale=pixel_scale,
deblend_nthresh=deblend_nthresh, deblend_cont=deblend_cont,
clean_param=clean_param, show_fig=False)
seg_sky = copy.deepcopy(seg_lthre)
seg_sky[seg_lthre > 0] = 1
seg_sky = seg_sky.astype(bool)
# Blow up the mask
for obj in obj_lthre:
sep.mask_ellipse(seg_sky, obj['x'], obj['y'], obj['a'], obj['b'], obj['theta'], r=radius/2)
bkg_mask_2 = seg_sky
bkg_mask = (bkg_mask_1 + bkg_mask_2).astype(bool)
cen_obj = objects[segmap[int(bkg_mask.shape[0] / 2.), int(bkg_mask.shape[1] / 2.)] - 1]
fraction_radius = sep.flux_radius(img, cen_obj['x'], cen_obj['y'], 10*cen_obj['a'], 0.5)[0]
ba = np.divide(cen_obj['b'], cen_obj['a'])
if fraction_radius < int(bkg_mask.shape[0] / 8.):
sep.mask_ellipse(bkg_mask, cen_obj['x'], cen_obj['y'], fraction_radius, fraction_radius * ba,
cen_obj['theta'], r=central_mask_radius)
elif fraction_radius < int(bkg_mask.shape[0] / 4.):
sep.mask_ellipse(bkg_mask, cen_obj['x'], cen_obj['y'], fraction_radius, fraction_radius * ba,
cen_obj['theta'], r=1.2)
# Estimate sky from histogram of binned image
import copy
from scipy import stats
from astropy.stats import sigma_clip
from astropy.nddata import block_reduce
data = copy.deepcopy(img)
data[bkg_mask] = np.nan
if f_factor is None:
f_factor = round(6 / pixel_scale)
rebin = block_reduce(data, f_factor)
sample = rebin.flatten()
if show_fig:
display_single(rebin)
plt.savefig('./{}-bkg.png'.format(np.random.randint(1000)), dpi=100, bbox_inches='tight')
temp = sigma_clip(sample)
sample = temp.data[~temp.mask]
kde = stats.gaussian_kde(sample)
print(f_factor)
mean = np.nanmean(sample) / f_factor**2
median = np.nanmedian(sample) / f_factor**2
std = np.nanstd(sample, ddof=1) / f_factor / np.sqrt(len(sample))
xlim = np.std(sample, ddof=1) * 7
x = np.linspace(-xlim + np.median(sample), xlim + np.median(sample), 100)
offset = x[np.argmax(kde.evaluate(x))] / f_factor**2
print('mean', mean)
print('median', median)
print('std', std)
bkg_global = sep.Background(img,
mask=bkg_mask, maskthresh=0,
bw=f_factor, bh=f_factor,
fw=f_factor/2, fh=f_factor/2)
print("#SEP sky: Mean Sky / RMS Sky = %10.5f / %10.5f" % (bkg_global.globalback, bkg_global.globalrms))
if show_hist:
fig, ax = plt.subplots(figsize=(8,6))
ax.plot(x, kde.evaluate(x), linestyle='dashed', c='black', lw=2,
label='KDE')
ax.hist(sample, bins=x, normed=1);
ax.legend(loc='best', frameon=False, fontsize=20)
ax.set_xlabel('Pixel Value', fontsize=20)
ax.set_ylabel('Normed Number', fontsize=20)
ax.tick_params(labelsize=20)
ylim = ax.get_ylim()
ax.text(-0.1 * f_factor + np.median(sample), 0.9 * (ylim[1] - ylim[0]) + ylim[0],
r'$\mathrm{offset}='+str(round(offset, 6))+'$', fontsize=20)
ax.text(-0.1 * f_factor + np.median(sample), 0.8 * (ylim[1] - ylim[0]) + ylim[0],
r'$\mathrm{median}='+str(round(median, 6))+'$', fontsize=20)
ax.text(-0.1 * f_factor + np.median(sample), 0.7 * (ylim[1] - ylim[0]) + ylim[0],
r'$\mathrm{std}='+str(round(std, 6))+'$', fontsize=20)
plt.vlines(np.median(sample), 0, ylim[1], linestyle='--')
return median, std, sample
def test_masked_segmentation_measurements():
"""Test measurements with segmentation masking"""
NX = 100
data = np.zeros((NX*2,NX*2))
yp, xp = np.indices(data.shape)
####
# Make two 2D gaussians that slightly overlap
# width of the 2D objects
gsigma = 10.
# offset between two gaussians in sigmas
off = 4
for xy in [[NX,NX], [NX+off*gsigma, NX+off*gsigma]]:
R = np.sqrt((xp-xy[0])**2+(yp-xy[1])**2)
g_i = np.exp(-R**2/2/gsigma**2)
data += g_i
# Absolute total
total_exact = g_i.sum()
# Add some noise
rms = 0.02
np.random.seed(1)
data += np.random.normal(size=data.shape)*rms
# Run source detection
objs, segmap = sep.extract(data, thresh=1.2, err=rms, mask=None,
segmentation_map=True)
seg_id = np.arange(1, len(objs)+1, dtype=np.int32)
# Compute Kron/Auto parameters
x, y, a, b = objs['x'], objs['y'], objs['a'], objs['b']
theta = objs['theta']
kronrad, krflag = sep.kron_radius(data, x, y, a, b, theta, 6.0)
flux_auto, fluxerr, flag = sep.sum_ellipse(data, x, y, a, b, theta,
2.5*kronrad,
segmap=segmap, seg_id=seg_id,
subpix=1)
# Test total flux
assert_allclose(flux_auto, total_exact, rtol=5.e-2)
# Flux_radius
for flux_fraction in [0.2, 0.5]:
# Exact solution
rhalf_exact = np.sqrt(-np.log(1-flux_fraction)*gsigma**2*2)
# Masked measurement
flux_radius, flag = sep.flux_radius(data, x, y, 6.*a, flux_fraction,
seg_id=seg_id, segmap=segmap,
normflux=flux_auto, subpix=5)
# Test flux fraction
assert_allclose(flux_radius, rhalf_exact, rtol=5.e-2)
if False:
print('test_masked_flux_radius')
print(total_exact, flux_auto)
print(rhalf_exact, flux_radius)
