def test_bandwidth_zero(self):
kern = kernels.Gaussian()
for bw in ['scott', 'silverman', 'normal_reference']:
with pytest.raises(RuntimeError,
match="Selected KDE bandwidth is 0"):
select_bandwidth(self.xx, bw, kern)
def density_arr(dist_org_1, dist_org_2, col):
kernel_switch = dict(gau=kernels.Gaussian,
epa=kernels.Epanechnikov,
uni=kernels.Uniform,
tri=kernels.Triangular,
biw=kernels.Biweight,
triw=kernels.Triweight,
cos=kernels.Cosine,
cos2=kernels.Cosine2)
bw = bandwidths.select_bandwidth(dist_org_1[col], "normal_reference",
kernel_switch["gau"]())
def kde_func_sans(series, bw):
kde = sm.nonparametric.KDEUnivariate(series)
kde.fit(bw=bw) # Estimate the densities
return kde.support, kde.density, kde
support_org_1, density_org_1, _ = kde_func_sans(dist_org_1[col], bw)
max_target = dist_org_1[col].astype(int).max()
min_target = dist_org_1[col].astype(int).min()
change = dist_org_2[col]
scaled_array = np.interp(change, (change.min(), change.max()),
(min_target, max_target))
support_org_2, density_org_2, _ = kde_func_sans(scaled_array, bw)
return density_org_1, density_org_2, support_org_1, support_org_2
def make_kde(self, bw_fac=1.0):
""" Takes the prior data and constructs a KDE function
Computes the KDE based on the parameters of a previously fit sample of
targets.
The KDE bandwidth is by default computed automatically using the
cross-validated maximum liklihood method in the `statsmodels' package.
Notes
-----
If the bandwidth factor is != 1 the method for calculating the initial
bandwidth is the 'scott' method. Otherwise it is the cross-validated
maximum likelihood method, employed by the `statsmodels' package.
This is currently a limitation of imposed by the current version of
`statsmodels'.
Parameters
----------
bw_fac : float, optional
Factor for expanding the bandwidth of the KDE. If float-like the
scaling will be the same for all paramaters. If array-like it must
be of the same length as the number of fit parameters. Each scaling
will then be applied individually to each parameter.
"""
self.par_names = [
'dnu', 'numax', 'eps', 'd02', 'alpha', 'env_height', 'env_width',
'mode_width', 'teff', 'bp_rp'
]
self.select_prior_data(self._log_obs['numax'])
if self.verbose:
print(f'Selected data set length {len(self.prior_data)}')
if bw_fac != 1:
from statsmodels.nonparametric.bandwidths import select_bandwidth
bw = select_bandwidth(self.prior_data[self.par_names].values,
bw='scott',
kernel=None)
bw *= bw_fac
else:
if self.verbose:
print('Selecting sensible stars for kde')
print(f'Full data set length {len(self.prior_data)}')
bw = 'cv_ml'
self.kde = sm.nonparametric.KDEMultivariate(
data=self.prior_data[self.par_names].values,
var_type='c' * len(self.par_names),
bw=bw)
def test_calculate_bandwidth_gaussian(self):
bw_expected = [
0.29774853596742024, 0.25304408155871411, 0.29781147113698891
]
kern = kernels.Gaussian()
bw_calc = [0, 0, 0]
for ii, bw in enumerate(['scott', 'silverman', 'normal_reference']):
bw_calc[ii] = select_bandwidth(Xi, bw, kern)
assert_allclose(bw_expected, bw_calc)
def test_calculate_bandwidth_gaussian(self):
bw_expected = [0.29774853596742024,
0.25304408155871411,
0.29781147113698891]
kern = kernels.Gaussian()
bw_calc = [0, 0, 0]
for ii, bw in enumerate(['scott','silverman','normal_reference']):
bw_calc[ii] = select_bandwidth(Xi, bw, kern)
assert_allclose(bw_expected, bw_calc)
def k2p2FixFromSum(SumImage, thresh=1, output_folder=None, plot_folder=None, show_plot=True,
min_no_pixels_in_mask=8, min_for_cluster=4, cluster_radius=np.sqrt(2),
segmentation=True, ws_alg='flux', ws_blur=0.5, ws_thres=0.05, ws_footprint=3,
extend_overflow=True, catalog=None):
"""
Create pixel masks from Sum-image.
Parameters:
SumImage (ndarray): Sum-image.
thres (float, optional): Threshold for significant flux. The threshold is calculated as MODE+thres*MAD. Default=1.
output_folder (string, optional): Path to directory where output should be saved. Default=None.
plot_folder (string, optional): Path to directory where plots should be saved. Default=None.
show_plot (boolean, optional): Should plots be shown to the user? Default=True.
min_no_pixels_in_mask (integer, optional): Minimim number of pixels to constitute a mask.
min_for_cluster (integer, optional): Minimum number of pixels to be considered a cluster in DBSCAN clustering.
cluster_radius (float, optional): Radius around points to consider cluster in DBSCAN clustering.
segmentation (boolean, optional): Perform segmentation of clusters using Watershed segmentation.
ws_alg (string, optional): Watershed method to use. Default='flux'.
ws_thres (float, optional): Threshold for watershed segmentation.
ws_footprint (integer, optional): Footprint to use in watershed segmentation.
extend_overflow (boolean, optional): Enable extension of overflow columns for bright stars.
catalog (ndarray, optional): Catalog of stars as an array with three columns (column, row and magnitude). If this is provided
the results will only allow masks to be returned for stars in the catalog and the information is
also used in the extension of overflow columns.
Returns:
tuple: Tuple with two elements: A 3D boolean ndarray of masks and a float indicating the bandwidth used for the estimation background-levels.
.. codeauthor:: Rasmus Handberg <*****@*****.**>
.. codeauthor:: Mikkel Lund <*****@*****.**>
"""
# Get logger for printing messages:
logger = logging.getLogger(__name__)
logger.info("Creating masks from sum-image...")
NY, NX = np.shape(SumImage)
ori_mask = ~np.isnan(SumImage)
X, Y = np.meshgrid(np.arange(NX), np.arange(NY))
# Cut out pixels from sum image which were collected and contains flux
# and flatten the 2D image to 1D array:
Flux = SumImage[ori_mask].flatten()
Flux = Flux[Flux > 0]
# Check if there was actually any flux measured:
if len(Flux) == 0:
raise K2P2NoFlux("No measured flux in sum-image")
# Cut away the top 15% of the fluxes:
flux_cut = stats.trim1(np.sort(Flux), 0.15)
# Also do a cut on the absolute values of pixel - This helps in cases where
# the image is dominated by saturated pixels. The exact value is of course
# in principle dependent on the CCD, but we have found this value to be
# reasonable in TESS simulated data:
flux_cut = flux_cut[flux_cut < 70000]
# Estimate the bandwidth we are going to use for the background:
background_bandwidth = select_bandwidth(flux_cut, bw='scott', kernel='gau')
logger.debug(" Sum-image KDE bandwidth: %f", background_bandwidth)
# Make the Kernel Density Estimation of the fluxes:
kernel = KDE(flux_cut)
kernel.fit(kernel='gau', bw=background_bandwidth, fft=True, gridsize=100)
# MODE
def kernel_opt(x): return -1*kernel.evaluate(x)
max_guess = kernel.support[np.argmax(kernel.density)]
MODE = minimize(kernel_opt, max_guess, method='Powell').x
# MAD (around mode)
MAD1 = mad_to_sigma * nanmedian( np.abs( Flux[(Flux < MODE)] - MODE ) )
# Define the cutoff above which pixels are regarded significant:
CUT = MODE + thresh * MAD1
logger.debug(" Threshold used: %f", thresh)
logger.debug(" Flux cut is: %f", CUT)
if logger.isEnabledFor(logging.DEBUG) and plot_folder is not None:
fig = plt.figure()
ax = fig.add_subplot(111)
ax.fill_between(kernel.support, kernel.density, alpha=0.3)
ax.axvline(MODE, color='k')
ax.axvline(CUT, color='r')
ax.set_xlabel('Flux')
ax.set_ylabel('Distribution')
save_figure(os.path.join(plot_folder, 'flux_distribution'))
plt.close(fig)
#==========================================================================
# Find and seperate clusters of pixels
#==========================================================================
# Cut out pixels of sum image with flux above the cut-off:
idx = (SumImage > CUT)
X2 = X[idx]
Y2 = Y[idx]
if np.all(~idx):
raise K2P2NoStars("No flux above threshold")
logger.debug(" Min for cluster is: %f", min_for_cluster)
logger.debug(" Cluster radius is: %f", cluster_radius)
# Run clustering algorithm
XX, labels_ini, core_samples_mask = run_DBSCAN(X2, Y2, cluster_radius, min_for_cluster)
# Run watershed segmentation algorithm:
# Demand that there was any non-noise clusters found.
if segmentation and any(labels_ini != -1):
# Create a set of dummy-masks that are made up of the clusters
# that were found by DBSCAN, meaning that there could be masks
# with several stars in them:
DUMMY_MASKS = np.zeros((0, NY, NX), dtype='bool')
DUMMY_MASKS_LABELS = []
m = np.zeros_like(SumImage, dtype='bool')
for lab in set(labels_ini):
if lab == -1: continue
# Create "image" of this mask:
m[:,:] = False
for x,y in XX[labels_ini == lab]:
m[y, x] = True
# Append them to lists:
DUMMY_MASKS = np.append(DUMMY_MASKS, [m], axis=0)
DUMMY_MASKS_LABELS.append(lab)
# Run the dummy masks through the detection of saturated columns:
logger.debug("Detecting saturated columns in non-segmentated masks...")
smask, _ = k2p2_saturated(SumImage, DUMMY_MASKS, idx)
# Create dictionary that will map a label to the mask of saturated pixels:
if np.any(smask):
saturated_masks = {}
for u,sm in enumerate(smask):
saturated_masks[DUMMY_MASKS_LABELS[u]] = sm
else:
saturated_masks = None
# Run the mask segmentaion algorithm on the found clusters:
labels, unique_labels, NoCluster = k2p2WS(X, Y, X2, Y2, SumImage, XX, labels_ini, core_samples_mask, saturated_masks=saturated_masks, ws_thres=ws_thres,
ws_footprint=ws_footprint, ws_blur=ws_blur, ws_alg=ws_alg, output_folder=plot_folder, catalog=catalog)
else:
labels = labels_ini
unique_labels = set(labels)
#NoCluster = len(unique_labels) - (1 if -1 in labels else 0)
# Make sure it is a tuple and not a set - much easier to work with:
unique_labels = tuple(unique_labels)
# Create list of clusters and their number of pixels:
No_pix_sort = np.zeros([len(unique_labels), 2])
for u,lab in enumerate(unique_labels):
No_pix_sort[u, 0] = np.sum(labels == lab)
No_pix_sort[u, 1] = lab
# Only select the clusters that have enough pixels and are not noise:
cluster_select = (No_pix_sort[:, 0] >= min_no_pixels_in_mask) & (No_pix_sort[:, 1] != -1)
no_masks = sum(cluster_select)
No_pix_sort = No_pix_sort[cluster_select, :]
# No masks were found, so return None:
if no_masks == 0:
MASKS = None
else:
# Sort the clusters by the number of pixels:
cluster_sort = np.argsort(No_pix_sort[:, 0])
No_pix_sort = No_pix_sort[cluster_sort[::-1], :]
# Create 3D array that will hold masks for each target:
MASKS = np.zeros((no_masks, NY, NX))
for u in range(no_masks):
lab = No_pix_sort[u, 1]
class_member_mask = (labels == lab)
xy = XX[class_member_mask ,:]
MASKS[u, xy[:,1], xy[:,0]] = 1
#==========================================================================
# Fill holes in masks
#==========================================================================
pattern = np.array([[[0, 0.25, 0],[0.25, 0, 0.25],[0, 0.25, 0]]]) # 3D array - shape=(1, 3, 3)
mask_holes_indx = ndimage.convolve(MASKS, pattern, mode='constant', cval=0.0)
mask_holes_indx = (mask_holes_indx > 0.95) & (MASKS == 0) # Should be exactly 1.0, but let's assume some round-off errors
if np.any(mask_holes_indx):
logger.info("Filling %d holes in the masks", np.sum(mask_holes_indx))
MASKS[mask_holes_indx] = 1
if not plot_folder is None:
# Create image showing all masks at different levels:
img = np.zeros((NY,NX))
for r in np.transpose(np.where(MASKS > 0)):
img[r[1], r[2]] = r[0]+1
# Plot everything together:
fig = plt.figure()
ax = fig.add_subplot(111)
plot_image(img, ax=ax, scale='linear', percentile=100, cmap='nipy_spectral', title='Holes in mask filled')
# Create outline of filled holes:
for hole in np.transpose(np.where(mask_holes_indx)):
cen = (hole[2]-0.5, hole[1]-0.5)
ax.add_patch(mpl.patches.Rectangle(cen, 1, 1, color='k', lw=2, fill=False, hatch='//'))
#fig.savefig(os.path.join(plot_folder, 'mask_filled_holes.png'), format='png', bbox_inches='tight')
save_figure(os.path.join(plot_folder, 'mask_filled_holes'))
plt.close(fig)
#==========================================================================
# Entend overflow lanes
#==========================================================================
if extend_overflow:
logger.debug("Detecting saturated columns in masks...")
# Find pixels that are saturated in each mask and find out if they should
# be added to the mask:
saturated_mask, pixels_added = k2p2_saturated(SumImage, MASKS, idx)
logger.info("Overflow will add %d pixels in total to the masks.", pixels_added)
# If we have a catalog of stars, we will only allow stars above the saturation
# limit to get their masks extended:
if catalog is not None:
# Filter that catalog, only keeping stars actully inside current image:
c = np.asarray(np.round(catalog[:, 0]), dtype='int32')
r = np.asarray(np.round(catalog[:, 1]), dtype='int32')
tmag = catalog[:, 2]
indx = (c >= 0) & (c < SumImage.shape[1]) & (r >= 0) & (r < SumImage.shape[0])
c = c[indx]
r = r[indx]
tmag = tmag[indx]
# Loop through the masks:
for u in range(no_masks):
if np.any(saturated_mask[u, :, :]):
# Find out which stars fall inside this mask:
which_stars = np.asarray(MASKS[u, :, :][r, c], dtype='bool')
if np.any(which_stars):
# Only allow extension of columns if the combined light of
# the targts in the mask exceeds the saturation limit:
mags_in_mask = tmag[which_stars]
mags_total = -2.5*np.log10(np.nansum(10**(-0.4*mags_in_mask)))
if mags_total > saturation_limit:
# The combined magnitude of the targets is now
# above saturation
saturated_mask[u, :, :] = False
else:
# Do not add saturation columns if no stars were found:
saturated_mask[u, :, :] = False
# If we are going to plot later on, make a note
# of how the outline of the masks looked before
# changing anything:
if plot_folder is not None and logger.isEnabledFor(logging.DEBUG):
outline_before = []
for u in range(no_masks):
outline_before.append( k2p2maks(MASKS[u,:,:], 1, 0.5) )
# Add the saturated pixels to the masks:
MASKS[saturated_mask] = 1
# If we are running as DEBUG, output some plots as well:
if plot_folder is not None and logger.isEnabledFor(logging.DEBUG):
logger.debug("Plotting overflow figures...")
Ypixel = np.arange(NY)
for u in range(no_masks):
mask = np.asarray(MASKS[u, :, :], dtype='bool')
mask_rows, mask_columns = np.where(mask)
mask_max = np.nanmax(SumImage[mask])
# The outline of the mask after saturated columns have been
# corrected for:
outline = k2p2maks(mask, 1, 0.5)
with PdfPages(os.path.join(plot_folder, 'overflow_mask' + str(u) + '.pdf')) as pdf:
for c in sorted(set(mask_columns)):
column_rows = mask_rows[mask_columns == c]
title = "Mask %d - Column %d" % (u, c)
if np.any(saturated_mask[u,:,c]):
title += " - Saturated"
fig = plt.figure(figsize=(14,6))
ax1 = fig.add_subplot(121)
ax1.axvspan(np.min(column_rows)-0.5, np.max(column_rows)+0.5, color='0.7')
ax1.plot(Ypixel, SumImage[:, c], 'ro-', drawstyle='steps-mid')
ax1.set_title(title)
ax1.set_xlabel('Y pixels')
ax1.set_ylabel('Sum-image counts')
ax1.set_ylim(0, mask_max)
ax1.set_xlim(-0.5, NY-0.5)
ax2 = fig.add_subplot(122)
plot_image(SumImage, ax=ax2, scale='log')
ax2.plot(outline_before[u][:,0], outline_before[u][:,1], 'r:')
ax2.plot(outline[:,0], outline[:,1], 'r-')
ax2.axvline(c, color='r', ls='--')
pdf.savefig(fig)
plt.close(fig)
#==============================================================================
# Create plots
#==============================================================================
if plot_folder is not None:
# Colors to use for each cluster label:
colors = plt.cm.gist_rainbow(np.linspace(0, 1, len(unique_labels)))
# Colormap to use for clusters:
# https://stackoverflow.com/questions/9707676/defining-a-discrete-colormap-for-imshow-in-matplotlib/9708079#9708079
#cmap = mpl.colors.ListedColormap(np.append([[1, 1, 1, 1]], colors, axis=0))
#cmap_norm = mpl.colors.BoundaryNorm(np.arange(-1, len(unique_labels)-1)+0.5, cmap.N)
# Set up figure to hold subplots:
if NY/NX > 5:
aspect = 0.5
else:
aspect = 0.2
fig0 = plt.figure(figsize=(2*plt.figaspect(aspect)))
fig0.subplots_adjust(wspace=0.12)
# ---------------
# PLOT 1
ax0 = fig0.add_subplot(151)
plot_image(SumImage, ax=ax0, scale='log', title='Sum-image', xlabel=None, ylabel=None)
# ---------------
# PLOT 2
Flux_mat2 = np.zeros_like(SumImage)
Flux_mat2[SumImage < CUT] = 1
Flux_mat2[SumImage > CUT] = 2
Flux_mat2[ori_mask == 0] = 0
ax2 = fig0.add_subplot(152)
plot_image(Flux_mat2, ax=ax2, scale='linear', percentile=100, cmap='nipy_spectral', title='Significant flux', xlabel=None, ylabel=None)
# ---------------
# PLOT 3
ax2 = fig0.add_subplot(153)
Flux_mat4 = np.zeros_like(SumImage)
for u,lab in enumerate(unique_labels):
class_member_mask = (labels == lab)
xy = XX[class_member_mask,:]
if lab == -1:
# Black used for noise.
ax2.plot(xy[:, 0], xy[:, 1], '+', markerfacecolor='k',
markeredgecolor='k', markersize=5)
else:
Flux_mat4[xy[:,1], xy[:,0]] = u+1
ax2.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(colors[u]),
markeredgecolor='k', markersize=5)
ax2.set_title("Clustering + Watershed")
ax2.set_xlim([-0.5, SumImage.shape[1]-0.5])
ax2.set_ylim([-0.5, SumImage.shape[0]-0.5])
ax2.set_aspect('equal')
# ---------------
# PLOT 4
ax4 = fig0.add_subplot(154)
plot_image(Flux_mat4, ax=ax4, scale='linear', percentile=100, cmap='nipy_spectral', title='Extracted clusters', xlabel=None, ylabel=None)
# ---------------
# PLOT 5
ax5 = fig0.add_subplot(155)
plot_image(SumImage, ax=ax5, scale='log', title='Final masks', xlabel=None, ylabel=None)
# Plot outlines of selected masks:
for u in range(no_masks):
# Get the color associated with this label:
col = colors[ int(np.where(unique_labels == No_pix_sort[u, 1])[0]) ]
# Make mask outline:
outline = k2p2maks(MASKS[u, :, :], 1, threshold=0.5)
# Plot outlines:
ax5.plot(outline[:, 0], outline[:, 1], color=col, zorder=10, lw=2.5)
ax4.plot(outline[:, 0], outline[:, 1], color='k', zorder=10, lw=1.5)
# Save the figure and close it:
save_figure(os.path.join(plot_folder, 'masks_'+ws_alg))
if show_plot:
plt.show()
else:
plt.close('all')
return MASKS, background_bandwidth
def remove_stars(tpf):
sumimage = np.nansum(tpf, axis=0, dtype='float64')
ny, nx = np.shape(sumimage)
ori_mask = ~np.isnan(sumimage)
X, Y = np.meshgrid(np.arange(nx), np.arange(ny))
Flux = sumimage[ori_mask].flatten()
Flux = Flux[Flux > 0]
flux_cut = stats.trim1(np.sort(Flux), 0.15)
background_bandwidth = select_bandwidth(flux_cut, bw='scott', kernel='gau')
kernel = KDE(flux_cut)
kernel.fit(kernel='gau', bw=background_bandwidth, fft=True, gridsize=100)
def kernel_opt(x):
return -1 * kernel.evaluate(x)
max_guess = kernel.support[np.argmax(kernel.density)]
MODE = optimize.fmin_powell(kernel_opt, max_guess, disp=0)
mad_to_sigma = 1.482602218505602
MAD1 = mad_to_sigma * nanmedian(np.abs(Flux[(Flux < MODE)] - MODE))
thresh = 2.
CUT = MODE + thresh * MAD1
idx = (sumimage > CUT)
X2 = X[idx]
Y2 = Y[idx]
cluster_radius = np.sqrt(2)
min_for_cluster = 4
XX, labels_ini, core_samples_mask = run_DBSCAN(X2, Y2, cluster_radius,
min_for_cluster)
DUMMY_MASKS = np.zeros((0, ny, nx), dtype='bool')
DUMMY_MASKS_LABELS = []
m = np.zeros_like(sumimage, dtype='bool')
for lab in set(labels_ini):
if lab == -1: continue
# Create "image" of this mask:
m[:, :] = False
for x, y in XX[labels_ini == lab]:
m[y, x] = True
# Append them to lists:
DUMMY_MASKS = np.append(DUMMY_MASKS, [m], axis=0)
DUMMY_MASKS_LABELS.append(lab)
smask, _ = k2p2_saturated(sumimage, DUMMY_MASKS, idx)
if np.any(smask):
saturated_masks = {}
for u, sm in enumerate(smask):
saturated_masks[DUMMY_MASKS_LABELS[u]] = sm
else:
saturated_masks = None
ws_thres = 0.02
ws_footprint = 3
ws_blur = 0.2
ws_alg = 'flux'
plot_folder = None
catalog = None
labels, unique_labels, NoCluster = k2p2WS(
X,
Y,
X2,
Y2,
sumimage,
XX,
labels_ini,
core_samples_mask,
saturated_masks=saturated_masks,
ws_thres=ws_thres,
ws_footprint=ws_footprint,
ws_blur=ws_blur,
ws_alg=ws_alg,
output_folder=plot_folder,
catalog=catalog)
# Make sure it is a tuple and not a set - much easier to work with:
unique_labels = tuple(unique_labels)
# Create list of clusters and their number of pixels:
No_pix_sort = np.zeros([len(unique_labels), 2])
for u, lab in enumerate(unique_labels):
No_pix_sort[u, 0] = np.sum(labels == lab)
No_pix_sort[u, 1] = lab
# Only select the clusters that are not the largest or noise:
cluster_select = (No_pix_sort[:, 0] < np.max(
No_pix_sort.T[0])) & (No_pix_sort[:, 1] != -1)
# cluster_select = (No_pix_sort[:, 0] < np.max(No_pix_sort.T[0]))
no_masks = sum(cluster_select)
No_pix_sort = No_pix_sort[cluster_select, :]
MASKS = np.zeros((no_masks, ny, nx))
for u in range(no_masks):
lab = No_pix_sort[u, 1]
class_member_mask = (labels == lab)
xy = XX[class_member_mask, :]
MASKS[u, xy[:, 1], xy[:, 0]] = 1
maskimg = np.sum(MASKS, axis=0)
invmaskimg = np.abs(maskimg - 1)
return invmaskimg * tpf
