def manual_bg(data):
#directly from photutils example
mean, median, std = sigma_clipped_stats(data, sigma=3.0, iters=5)
threshold = median + (std * 2.)
segm_img = detect_sources(data, threshold, npixels=5)
#%% morphological
mask = segm_img.data.astype(bool)
strel = ones((3,3))
dilated = binary_dilation(mask,strel)
mean, median, std = sigma_clipped_stats(data, sigma=3.0, mask=dilated)
#%% use mean or median for background? Let's do what sextractor does
if (mean - median) / std > 0.3:
bgval = median
else:
bgval = mean
#%% background subtract
dinf = iinfo(data.dtype)
imgmask = masked_where(~dilated,100+data).clip(0,dinf.max).astype(data.dtype)
datashift=(data-bgval).clip(0,dinf.max).astype(data.dtype)
#%%
fg,axs = subplots(2,2)
ax = axs[0,0]
hi=ax.imshow(data,interpolation='none',cmap='gray')
ax.imshow(imgmask,interpolation='none',cmap='gray')
ax.set_title('original image with stars highlighted')
fg.colorbar(hi,ax=ax)
ax.grid(False)
ax=axs[0,1]
ax.imshow(dilated,interpolation='none')
ax.set_title('dilated')
ax.grid(False)
ax = axs[1,0]
hi=ax.imshow(datashift,interpolation='none',cmap='gray')
ax.set_title('background subtracted image {:.3f}'.format(bgval))
fg.colorbar(hi,ax=ax)
ax.grid(False)
fg.tight_layout()
return datashift
def manual_bg(data):
#directly from photutils example
mean, median, std = sigma_clipped_stats(data, sigma=3.0, iters=5)
threshold = median + (std * 2.)
segm_img = detect_sources(data, threshold, npixels=5)
#%% morphological
mask = segm_img.data.astype(bool)
strel = ones((3,3))
dilated = binary_dilation(mask,strel)
mean, median, std = sigma_clipped_stats(data, sigma=3.0, mask=dilated)
#%% use mean or median for background? Let's do what sextractor does
if (mean - median) / std > 0.3:
bgval = median
else:
bgval = mean
#%% background subtract
dinf = iinfo(data.dtype)
imgmask = masked_where(~dilated,100+data).clip(0,dinf.max).astype(data.dtype)
datashift=(data-bgval).clip(0,dinf.max).astype(data.dtype)
#%%
fg,axs = subplots(2,2)
ax = axs[0,0]
hi=ax.imshow(data,interpolation='none',cmap='gray')
ax.imshow(imgmask,interpolation='none',cmap='gray')
ax.set_title('original image with stars highlighted')
fg.colorbar(hi,ax=ax)
ax.grid(False)
ax=axs[0,1]
ax.imshow(dilated,interpolation='none')
ax.set_title('dilated')
ax.grid(False)
ax = axs[1,0]
hi=ax.imshow(datashift,interpolation='none',cmap='gray')
ax.set_title('background subtracted image {:.3f}'.format(bgval))
fg.colorbar(hi,ax=ax)
ax.grid(False)
fg.tight_layout()
return datashift
def source_find(img,ota,inst):
"""
This function will find sources on an OTA
using the detect_sources module from photutils.
This will return of csv file of the sources found
with the x,y,Ra,Dec,source_sum,max_value, and
elongation of the source. The elongation parameter is
semimajor_axis / semiminor_axis. This output is needed
for the source_xy function.
"""
image = odi.reprojpath+'reproj_'+ota+'.'+str(img[16:])
QR_raw = odi.fits.open(image)
hdu_ota = QR_raw[0]
if inst == 'podi':
pvlist = hdu_ota.header['PV*']
for pv in pvlist:
tpv = 'T'+pv
hdu_ota.header.rename_keyword(pv, tpv, force=False)
w = odi.WCS(hdu_ota.header)
#w.wcs.ctype = ["RA---TPV", "DEC--TPV"]
bg_mean,bg_median,bg_std = odi.mask_ota(img,ota,reproj=True)
threshold = bg_median + (bg_std * 5.)
print bg_mean,bg_median,bg_std
segm_img = detect_sources(hdu_ota.data, threshold, npixels=20)
source_props = source_properties(hdu_ota.data,segm_img,wcs=w)
columns = ['id', 'xcentroid', 'ycentroid', 'ra_icrs_centroid',
'dec_icrs_centroid','source_sum','max_value','elongation']
source_tbl = properties_table(source_props,columns=columns)
source_tbl_df = source_tbl.to_pandas()
outputfile = odi.sourcepath+'source_'+ota+'.'+str(img[16:-5])+'.csv'
source_tbl_df.to_csv(outputfile,index=False)
QR_raw.close()
def segmentation_photometry(path_file_abs,
bkg_sigma=3.0,
source_snr=3.0,
fwhm_kernel=2.0,
x_size_kernel=3,
y_size_kernel=3,
clobber=False):
"""
aperture photometry from source segmentation
make_source_mask not yet available in photutils v0.2.2, this version
manually creates a source mask for determining background
"""
import os
import copy
import glob
import pickle
import numpy as np
from scipy import ndimage
import matplotlib
#matplotlib.rcParams['text.usetex'] = True
#matplotlib.rcParams['text.latex.unicode'] = True
#from matplotlib.backends.backend_pdf import PdfPages
import matplotlib.pyplot as plt
from astropy.io import fits, ascii
from astropy.convolution import Gaussian2DKernel
from astropy.stats import sigma_clipped_stats, gaussian_fwhm_to_sigma
# from astropy.table import Table
from astropy.visualization import (LogStretch, mpl_normalize)
# from astropy.extern.six.moves import StringIO
from photutils import (detect_threshold, EllipticalAperture,
source_properties, properties_table)
from photutils.detection import detect_sources
from photutils.utils import random_cmap
# create preliminary mask
#from photutils import make_source_mask
#masterMask = make_source_mask(master, snr=2, npixels=5, dilate_size=11)
# if LEDoff was used, get threshold from LEDoff/background
# path_dataset = os.path.dirname(path_file_abs) + os.path.sep
# filenameCombined = '\t'.join(
# os.listdir(os.path.join(datasetDirLocal, 'master')))
# if 'master_ledoff_subtracted' in filename:
# print('Using master_ledoff')
# # path_file_abs = os.path.join(datasetDir, 'master', filename)
# hdu = fits.open(path_file_abs)[0]
# data_subtracted = hdu.data
# # calculate threadhold
# ledoff_pred = np.mean(data_subtracted) * \
# np.ones(data_subtracted.shape)
# mse = mean_squared_error(data_subtracted, ledoff_pred)
# rmse = np.sqrt(mse)
# threshold = 7.0 * rmse
# threshold_value = threshold
# if no LEDoff was used, background subtraction is needed
# there should exist no file named "subtracted"
# if 'master.fit' in filenameCombined \
# or 'master_normalised.fit' in filenameCombined:
#filenamedir = os.path.basename(path_file_abs)
#print(filenamedir)
#New stuff made by Parker
f_dir, filename = os.path.split(path_file_abs)
ff = os.path.splitext(filename)[0]
new_dir = f_dir + '/' + ff
if not os.path.exists(new_dir):
os.makedirs(new_dir)
dir_save = new_dir
print("The photometry files will be saved in ", dir_save)
filenames_combined = '\t'.join(os.listdir(dir_save))
if clobber == False \
and 'segm.obj' in filenames_combined \
and 'props.obj' in filenames_combined \
and 'props.csv' in filenames_combined\
and 'props.ecsv' in filenames_combined:
print('Photometry properties table already exists. Reading objects...')
segm = pickle.load(
open(glob.glob(os.path.join(dir_save, '*segm.obj*'))[0], 'rb'))
props = pickle.load(
open(glob.glob(os.path.join(dir_save, '*props.obj*'))[0], 'rb'))
return [segm, props]
if 'master' in path_file_abs:
if 'normalised' in path_file_abs:
print('Performing photometry to ' +
'normalised master object image {}...'.format(path_file_abs))
else:
print('Performing photometry to ' +
'un-normalised master image {}...'.format(path_file_abs))
else:
print('Warning: Photometry being performed to ' +
'a single exposure {}...'.format(path_file_abs))
hdu = fits.open(path_file_abs)[0]
data = hdu.data
header = hdu.header
if 'EXPREQ' in header:
exptime = header['EXPREQ']
elif 'EXPTIME' in header:
exptime = header['EXPTIME']
else:
print('Exposure time not found in header. Cannot determine magnitude.')
exptime = np.nan
# === Iteratively determine background level ===
# assuming background is homogenous, estimate background by sigma clipping
# if background noise varies across image, generate 2D background instead
print('Determining background noise level...' '')
[mean, median, std] = sigma_clipped_stats(data, sigma=bkg_sigma, iters=5)
threshold = median + (std * 2.0)
segm = detect_sources(data, threshold, npixels=5)
# turn segm into a mask
mask = segm.data.astype(np.bool)
# dilate the source mask to ensure complete masking of detected sources
dilate_structure = np.ones((5, 5))
mask_dilated = ndimage.binary_dilation(mask, structure=dilate_structure)
# get sigma clipping stats of background, without sources that are masekd
[bkg_mean, bkg_median, bkg_std] = sigma_clipped_stats(data,
sigma=bkg_sigma,
mask=mask_dilated,
iters=3)
# === Detect sources by segmentation ===
print('Determining threshold for source detection...')
# determine threshold for source detection
# in current implementation, if all inputs are present, the formula is
# threshold = background + (background_error * snr)
threshold = detect_threshold(data,
background=bkg_median,
error=bkg_std,
snr=source_snr)
print('Preparing 2D Gaussian kernal...')
sigma_kernel = fwhm_kernel * gaussian_fwhm_to_sigma
kernel = Gaussian2DKernel(sigma_kernel,
x_size=x_size_kernel,
y_size=y_size_kernel)
# normalise kernel
# The kernel models are normalized per default, ∫∞−∞f(x)dx=1∫−∞∞f(x)dx=1.
# But because of the limited kernel array size, the normalization
# for kernels with an infinite response can differ from one.
kernel.normalize()
# obtain a SegmentationImage object with the same shape as the data,
# where sources are labeled by different positive integer values.
# A value of zero is always reserved for the background.
# if the threshold includes the background level as above, then the image
# input into detect_sources() should not be background subtracted.
print('Segmentation processing...')
segm = detect_sources(data, threshold, npixels=5, filter_kernel=kernel)
print('Segmentation labels are: ', repr(segm.labels))
# === Measure regional source properties ===
# source_properties() assumes that the data have been background-subtracted.
# Background is the background level that was previously present
# in the input data.
# The input background does not get subtracted from the input data,
# which should already be background-subtracted.
print('Extracting source properties...')
props = source_properties(data - bkg_median, segm, background=bkg_median)
# add flux and instrumental magnitude to properties
# flux = source_sum / exptime
# instrumental magnitude = -2.5 * log10(flux)
for i in range(len(props)):
# source_sum is by definition background-subtracted already
props[i].flux = props[i].source_sum / exptime
props[i].mag_instr = -2.5 * np.log10(props[i].flux)
# make plots and save to images
# define approximate isophotal ellipses for each object
apertures = []
r = 2.8 # approximate isophotal extent
for prop in props:
position = (prop.xcentroid.value, prop.ycentroid.value)
a = prop.semimajor_axis_sigma.value * r
b = prop.semiminor_axis_sigma.value * r
theta = prop.orientation.value
apertures.append(EllipticalAperture(position, a, b, theta=theta))
# create a table of properties
try:
props_table = properties_table(props)
except:
print('No source detected in {}'.format(path_file_abs))
return [None, None]
props_table['flux'] = [props[i].flux for i in range(len(props))]
props_table['mag_instr'] = [props[i].mag_instr for i in range(len(props))]
# add custom columns to the table: mag_instru and flux
# plot centroid and segmentation using approximate elliptical apertures
norm = mpl_normalize.ImageNormalize(stretch=LogStretch())
rand_cmap = random_cmap(segm.max + 1, random_state=12345)
#[fig1, (ax1, ax2)] = plt.subplots(1, 2, figsize = (12, 6))
#ax1.imshow(data, origin='lower', cmap=plt.cm.gray, norm=norm)
#ax1.plot(
# props_table['xcentroid'], props_table['ycentroid'],
# ls='none', color='blue', marker='+', ms=10, lw=1.5)
#ax2.imshow(segm, origin='lower', cmap=rand_cmap)
#for aperture in apertures:
# aperture.plot(ax=ax1, lw=1.0, alpha=1.0, color='red')
# aperture.plot(ax=ax2, lw=1.0, alpha=1.0, color='red')
# plot using actual segmentation outlines (to be improved)
#[fig2, ax3] = plt.subplots(figsize = (6, 6))
#ax3.imshow(data, origin='lower', cmap=plt.cm.gray, norm=norm)
#segm_outline = np.array(segm.outline_segments(), dtype=float)
#segm_outline[segm_outline<1] = np.nan
# get a copy of the gray color map
#segm_outline_cmap = copy.copy(plt.cm.get_cmap('autumn'))
# set how the colormap handles 'bad' values
#segm_outline_cmap.set_bad(alpha=0)
#ax3.imshow(segm_outline, origin='lower', cmap=segm_outline_cmap)
# === save ===
# Save segm, porps to object files, and also save props to table file.
print('Saving segmentation and source propdderties to {}...'.format(
dir_save))
try:
# if filename ends with fits, remove it in the filename
if filename[-5:] == '.fits':
dir_save_prefix = os.path.join(dir_save, filename[0:-5])
else:
dir_save_prefix = os.path.join(dir_save, filename)
# Enhanced CSV allows preserving table meta-data such as
# column data types and units.
# In this way a data table can be stored and read back as ASCII
# with no loss of information.
ascii.write(props_table,
dir_save_prefix + '-phot_props.ecsv',
format='ecsv')
# csv for readability in MS excel
ascii.write(props_table,
dir_save_prefix + '-phot_props.csv',
format='csv')
# dump segmentation and properties to object files in binary mode
file_segm = open(dir_save_prefix + '-phot_segm.obj', 'wb')
pickle.dump(segm, file_segm)
file_props = open(dir_save_prefix + '-phot_props.obj', 'wb')
pickle.dump(props, file_props)
# save figures
#fig1.savefig(dir_save_prefix + '-phot_segm_fig1.png', dpi=600)
#pp1 = PdfPages(dir_save_prefix + '-phot_segm_fig1.pdf')
#pp1.savefig(fig1)
#pp1.close()
#fig2.savefig(dir_save_prefix + '-phot_segm_fig2.png', dpi=600)
#pp2 = PdfPages(dir_save_prefix + '-phot_segm_fig2.pdf')
#pp2.savefig(fig2)
#pp2.close()
print('Segmentation, properties objects, tables, and images saved to',
dir_save)
except:
print('Unable to write to disk, check permissions.')
return [segm, props]
""" We have run through all of the images now. """
allSources = []
sourceList = ultraspecClasses.sourceList()
for index, w in enumerate(allWindows):
xll = w.xll/w.xbin - xmin
yll = w.yll/w.ybin - ymin
image = w.stackedData
mean, median, std = sigma_clipped_stats(image, sigma=3.0)
maximum = numpy.max(image)
minimum = numpy.min(image)
debug.write("Mean: %f, Median: %f, Std (clipped 3sigma):%f"%(mean, median, std) , 2)
debug.write("Minimum: %f, Maximum: %f"%(minimum, maximum), 2)
threshold = median + (std * 2.)
segm_img = detect_sources(image, threshold, npixels=5)
mask = segm_img.astype(numpy.bool)
mean, median, std = sigma_clipped_stats(image, sigma=3.0, mask=mask)
debug.write("After source masking", 2)
debug.write("Mean: %f, Median: %f, Std (clipped 3sigma): %f"%(mean, median, std), 2)
selem = numpy.ones((5, 5)) # dilate using a 5x5 box
mask2 = binary_dilation(mask, selem)
mean, median, std = sigma_clipped_stats(image, sigma=3.0, mask=mask2)
debug.write("After dilation", 2)
debug.write("Mean: %f, Median: %f, Std (clipped 3sigma): %f"%(mean, median, std), 2)
# Check the window image for any areas that should be masked...
lowerLimitBkg = median - std*5.
debug.write("5 sigma below the median is the lowerLimitBkg for the mask: %f"%(lowerLimitBkg), 2)
mask = (image < lowerLimitBkg)
maskBitmap = numpy.zeros(numpy.shape(mask))
def bkg_boxes(hdu,nboxes,length,sources):
"""
Function to calculate the sigma clipped statistics
of a number of randomly generated boxes
Variables:
frame: fits image
nboxes: number of boxes to generate
length: length of side of box in pixels
sources: if sources = True, the sources in each box will be detected and masked
if sources = False, no masking is done
"""
#hdu = pyfits.open(frame,memmap=True)
image = hdu.data
#Get length of image in each axis
naxis1 = hdu.header['NAXIS1']
naxis2 = hdu.header['NAXIS2']
#generate the centers of 1000 random boxes.
#np.random.seed(1234)
# box_centers = np.random.random_integers(0,np.min([naxis1,naxis2]),size=(nboxes,2))
# use np.random.randint() instead due to deprecation error on wopr
box_centers = np.random.randint(0,np.min([naxis1,naxis2]),size=(nboxes,2))
#divide length by 2
# side = float(length)/2.0
# another numpy error on wopr (can't convert to integer), so don't worry about integer arithmetic
side = length/2
bg_stats = []
centers = []
for center in range(len(box_centers)):
x1 = box_centers[center][0]-side
x2 = box_centers[center][0]+side
y1 = box_centers[center][1]-side
y2 = box_centers[center][1]+side
#Check to ensure that box is within image
if (x1 > side and x2 < naxis1-side) and (y1 > side and y2 < naxis2-side):
centers.append(box_centers[center])
"""
The centers that are within the image bounds are returned
in case you need to examine the regions used.
"""
box = image[x1:x2,y1:y2]
# Mask gaps
#gaps_mask = np.isnan(box).astype(int)
# Mask hot pixels count greater than 58000
#hot_pix_mask = (box > 58000.0).astype(int)
# Mask dead pixels
#dead_pix_mask = (box < 1.0).astype(int)
#mask_first_pass = hot_pix_mask + dead_pix_mask + gaps_mask
#if (box >= 0).all() == True:
if np.isnan(box).any() == False:
"""
Only boxes with non-negative values are kept.
This should help deal with cell gaps
The sigma and iter values might need some tuning.
"""
mean, median, std = sigma_clipped_stats(box, sigma=3.0)
if sources == False:
bg_stats.append((mean, median, std))
if sources == True:
threshold = median + (std * 2.)
segm_img = detect_sources(box, threshold, npixels=20)
mask = segm_img.data.astype(np.bool)# turn segm_img into a mask
selem = np.ones((10, 10)) # dilate using a 25x25 box
mask2 = binary_dilation(mask, selem)
#new_mask = mask_first_pass + mask2
new_mask = mask2
mean_mask, median_mask, std_mask = sigma_clipped_stats(box, sigma=3.0, mask=new_mask)
bg_stats.append((mean_mask, median_mask, std_mask))
bg_stats = np.reshape(np.array(bg_stats),(len(bg_stats),3))
centers = np.reshape(np.array(centers),(len(centers),2))
#Calculate median std of Background
med_std = np.median(bg_stats[:,2])
#calculate standard deviation of the std values
std_std = np.std(bg_stats[:,2])
#median
bg_median = np.median(bg_stats[:,1])
#Locate the box that had the largest std
#Array will be returned for plotting if wanted
max_std = np.argmax(bg_stats[:,2])
max_center = centers[max_std]
max_box = image[max_center[0]-side:max_center[0]+side,max_center[1]-side:max_center[1]+side]
#plt.imshow(max_box,origin='lower', cmap='Greys_r')
#plt.show()
return bg_stats,bg_median,med_std,std_std,centers,max_box