def find_ellipse(img, mask, background):
"""
Compute properties of an image (basically fit a 2D Gaussian) and
determine the corresponding elliptical aperture.
inputs:
-------
filename: string
r: float
isophotal extent (multiplied by semi-major axes of fitted
gaussian to determine the elliptical aperture)
extents: array-like, optional
xmin, xmax, ymin, ymax of sub-image
"""
cprops = morphology.data_properties(img - background, mask=(mask==0),
background = background)
tbl = photutils.properties_table(cprops, columns=columns)
#print tbl
position = (cprops.xcentroid.value, cprops.ycentroid.value)
a = cprops.semimajor_axis_sigma.value
b = cprops.semiminor_axis_sigma.value
theta = cprops.orientation.value
return position, a, b, theta
def _find_star_properties(self, data, median, mask, star_coo):
self.info('Finding star properties started')
sigma = self.config_section.get('fwhm') * gaussian_fwhm_to_sigma
kernel = Gaussian2DKernel(sigma,
x_size=self.config_section.get('kernel_x'),
y_size=self.config_section.get('kernel_y'))
kernel.normalize()
data[mask] = 0
segm = detect_sources(data,
median *
self.config_section.get('detect_threshold'),
npixels=self.config_section.get('npixels'),
filter_kernel=kernel)
properties = properties_table(
source_properties(data - np.uint64(median), segm),
columns=['id', 'xcentroid', 'ycentroid', 'source_sum',
'semimajor_axis_sigma', 'semiminor_axis_sigma',
'orientation'])
self.info('Found star properties')
self.info(properties)
if len(properties) > 1:
self.warning('More than one object has been found')
properties = self._find_nearest_object(
star_coo, properties, data.shape)
return properties
else:
self.info('Finding star properties finished')
return properties[0]
def source_find(img, ota, inst, nbg_std=10.0):
"""
This function will find sources on an OTA using the detect_sources module
from photutils. This will return of csv file of the sources found with the
x,y,Ra,Dec,source_sum,max_value, and elongation of the source. The
elongation parameter is semimajor_axis / semiminor_axis.
This output is needed for the source_xy function. This function is set
to work on the reprojected otas.
Parameters
----------
img : str
Name of image
ota : str
Name of OTA
int : str
Version of ODI used, ``podi`` or ``5odi``
nbg_std : float
Multiplier to the standard deviation of the background. It has a default
value of ``10`` to only detect bright sources
Note
----
This function produces a ``csv`` file in ``odi.sourcepath`` with the
following naming convention ``'source_'+ota+'.'+img.base()+'.csv'``.
"""
image = odi.reprojpath + 'reproj_' + ota + '.' + img.stem()
QR_raw = odi.fits.open(image)
# hdu_ota = QR_raw[0]
hdu_ota = odi.tan_header_fix(QR_raw[0])
w = odi.WCS(hdu_ota.header)
# needed to remind astropy that the header says RADESYS=ICRS
# your mileage may vary (logic probably needed here to handle cases)
w.wcs.radesys = 'ICRS'
# if inst == '5odi':
# w.wcs.ctype = ["RA---TPV", "DEC--TPV"]
bg_mean, bg_median, bg_std = odi.mask_ota(img, ota, reproj=True)
threshold = bg_median + (bg_std * nbg_std)
print bg_mean, bg_median, bg_std
segm_img = detect_sources(hdu_ota.data, threshold, npixels=20)
source_props = source_properties(hdu_ota.data, segm_img, wcs=w)
columns = [
'id', 'xcentroid', 'ycentroid', 'ra_icrs_centroid',
'dec_icrs_centroid', 'source_sum', 'max_value', 'elongation'
]
source_tbl = properties_table(source_props, columns=columns)
source_tbl_df = source_tbl.to_pandas()
outputfile = odi.sourcepath + 'source_' + ota + '.' + img.base() + '.csv'
source_tbl_df.to_csv(outputfile, index=False)
QR_raw.close()
def find_centroid(data):
"""
find the centroid again after the image was rotated
"""
sigma_clip = SigmaClip(sigma=3., iters=10)
bkg_estimator = MedianBackground()
bkg = Background2D(data, (25, 25),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
threshold = bkg.background + (3. * bkg.background_rms)
sigma = 2.0 * gaussian_fwhm_to_sigma # FWHM = 2.
kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
kernel.normalize()
segm = detect_sources(data, threshold, npixels=5, filter_kernel=kernel)
props = source_properties(data, segm)
tbl = properties_table(props)
my_min = 100000.
x_shape = np.float(data.shape[0])
y_shape = np.float(data.shape[1])
r = 3. # approximate isophotal extent
apertures = []
for prop in props:
position = (prop.xcentroid.value, prop.ycentroid.value)
#print(position)
a = prop.semimajor_axis_sigma.value * r
b = prop.semiminor_axis_sigma.value * r
theta = prop.orientation.value
apertures.append(EllipticalAperture(position, a, b, theta=theta))
my_dist = np.sqrt((prop.xcentroid.value - x_shape / 2.)**2 +
(prop.ycentroid.value - y_shape / 2.)**2)
if (my_dist < my_min):
my_label = prop.id - 1
my_min = my_dist
mytheta = props[my_label].orientation.value
mysize = np.int(np.round(r * props[my_label].semimajor_axis_sigma.value))
my_x = props[my_label].xcentroid.value
my_y = props[my_label].ycentroid.value
return my_x, my_y, mytheta, mysize
def grid_smooth(i_ra_f, i_dec_f, fwhm, width, height):
# bin the filtered stars into a grid with pixel size XXX
# print "Binning for m-M =",dm
# bins = 165
# width = 30
bins_h = int(height * 60. / 8.)
bins_w = int(width * 60. / 8.)
grid, xedges, yedges = np.histogram2d(i_dec_f, i_ra_f, bins=[bins_h,bins_w], range=[[0,height],[0,width]])
hist_points = zip(xedges,yedges)
sig = ((bins_w/width)*fwhm)/2.355
pltsig = fwhm/2.0
# convolve the grid with a gaussian
grid_gaus = ndimage.filters.gaussian_filter(grid, sig, mode='constant', cval=0)
S = np.array(grid_gaus*0)
S_th = 3.0
grid_mean = np.mean(grid_gaus)
grid_sigma = np.std(grid_gaus)
S = (grid_gaus-grid_mean)/grid_sigma
above_th = [(int(i),int(j)) for i in range(len(S)) for j in range(len(S[i])) if (S[i][j] >= S_th)]
segm = detect_sources(S, 2.0, npixels=5)
props = source_properties(S, segm)
columns = ['id', 'maxval_xpos', 'maxval_ypos', 'max_value', 'area']
tbl = properties_table(props, columns=columns)
# print tbl
# rand_cmap = random_cmap(segm.max + 1, random_state=12345)
# find the maximum point in the grid and center the circle there
x_cent, y_cent = np.unravel_index(grid_gaus.argmax(),grid_gaus.shape)
x_cent_S, y_cent_S = np.unravel_index(S.argmax(),S.shape)
# print 'Max of S located at:','('+'{0:6.3f}'.format(y_cent_S)+','+'{0:6.3f}'.format(x_cent_S)+')'
# print 'Value of S at above:','{0:6.3f}'.format(S[x_cent_S][y_cent_S])
# print 'Number of bins above S_th: {0:4d}'.format(len(above_th))
return xedges, x_cent, yedges, y_cent, S, x_cent_S, y_cent_S, pltsig, tbl, segm
def calc_fwhm_on_bright_star(image_file, print=True, fwhm_init=2.0):
"""Calculate the FWHM on a single bright star (either open or closed loops).
image_file -- either FITS or PGM
fwhm_init -- (def=2) pixels for FWHM initial guess
"""
img = load_image(image_file)
# Calculate the bacgkround
bkg = photutils.Background(img,
img.shape,
filter_shape=(1, 1),
method='median')
threshold = bkg.background + (30.0 * bkg.background_rms)
sigma = 2.0 * gaussian_fwhm_to_sigma # FWHM = 2. pixels
kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
kernel.normalize()
segm = detect_sources(img, threshold, npixels=5, filter_kernel=kernel)
props = source_properties(img, segm)
tbl = properties_table(props)
# Check for junk stars (cosmic rays)
idx = np.where((tbl['semimajor_axis_sigma'] > 1)
& (tbl['semiminor_axis_sigma'] > 1))[0]
tbl = tbl[idx]
tbl['image_name'] = image_file
if print == True:
reformat_source_table(tbl)
print_source_table(tbl)
return tbl
def source_find(img,ota,inst):
"""
This function will find sources on an OTA
using the detect_sources module from photutils.
This will return of csv file of the sources found
with the x,y,Ra,Dec,source_sum,max_value, and
elongation of the source. The elongation parameter is
semimajor_axis / semiminor_axis. This output is needed
for the source_xy function.
"""
image = odi.reprojpath+'reproj_'+ota+'.'+str(img[16:])
QR_raw = odi.fits.open(image)
hdu_ota = QR_raw[0]
if inst == 'podi':
pvlist = hdu_ota.header['PV*']
for pv in pvlist:
tpv = 'T'+pv
hdu_ota.header.rename_keyword(pv, tpv, force=False)
w = odi.WCS(hdu_ota.header)
#w.wcs.ctype = ["RA---TPV", "DEC--TPV"]
bg_mean,bg_median,bg_std = odi.mask_ota(img,ota,reproj=True)
threshold = bg_median + (bg_std * 5.)
print bg_mean,bg_median,bg_std
segm_img = detect_sources(hdu_ota.data, threshold, npixels=20)
source_props = source_properties(hdu_ota.data,segm_img,wcs=w)
columns = ['id', 'xcentroid', 'ycentroid', 'ra_icrs_centroid',
'dec_icrs_centroid','source_sum','max_value','elongation']
source_tbl = properties_table(source_props,columns=columns)
source_tbl_df = source_tbl.to_pandas()
outputfile = odi.sourcepath+'source_'+ota+'.'+str(img[16:-5])+'.csv'
source_tbl_df.to_csv(outputfile,index=False)
QR_raw.close()
def make_source_catalog(model, kernel_fwhm, kernel_xsize, kernel_ysize,
snr_threshold, npixels, deblend_nlevels=32,
deblend_contrast=0.001, deblend_mode='exponential',
connectivity=8, deblend=False):
"""
Create a final catalog of source photometry and morphologies.
Parameters
----------
model : `DrizProductModel`
The input `DrizProductModel` of a single drizzled image. The
input image is assumed to be background subtracted.
kernel_fwhm : float
The full-width at half-maximum (FWHM) of the 2D Gaussian kernel
used to filter the image before thresholding. Filtering the
image will smooth the noise and maximize detectability of
objects with a shape similar to the kernel.
kernel_xsize : odd int
The size in the x dimension (columns) of the kernel array.
kernel_ysize : odd int
The size in the y dimension (row) of the kernel array.
snr_threshold : float
The signal-to-noise ratio per pixel above the ``background`` for
which to consider a pixel as possibly being part of a source.
npixels : int
The number of connected pixels, each greater than the threshold
that an object must have to be detected. ``npixels`` must be a
positive integer.
deblend_nlevels : int, optional
The number of multi-thresholding levels to use for deblending
sources. Each source will be re-thresholded at
``deblend_nlevels``, spaced exponentially or linearly (see the
``deblend_mode`` keyword), between its minimum and maximum
values within the source segment.
deblend_contrast : float, optional
The fraction of the total (blended) source flux that a local
peak must have to be considered as a separate object.
``deblend_contrast`` must be between 0 and 1, inclusive. If
``deblend_contrast = 0`` then every local peak will be made a
separate object (maximum deblending). If ``deblend_contrast =
1`` then no deblending will occur. The default is 0.001, which
will deblend sources with a magnitude differences of about 7.5.
deblend_mode : {'exponential', 'linear'}, optional
The mode used in defining the spacing between the
multi-thresholding levels (see the ``deblend_nlevels`` keyword)
when deblending sources.
connectivity : {4, 8}, optional
The type of pixel connectivity used in determining how pixels
are grouped into a detected source. The options are 4 or 8
(default). 4-connected pixels touch along their edges.
8-connected pixels touch along their edges or corners. For
reference, SExtractor uses 8-connected pixels.
deblend : bool, optional
Whether to deblend overlapping sources. Source deblending
requires scikit-image.
Returns
-------
catalog : `~astropy.Table`
An astropy Table containing the source photometry and
morphologies.
"""
if not isinstance(model, DrizProductModel):
raise ValueError('The input model must be an DrizProductModel.')
# Remove until model.wht contains an IVM map
# Calculate "background-only" error assuming the weight image is an
# inverse-variance map (IVM). The weight image is clipped because it
# may contain zeros.
# bkg_error = np.sqrt(1.0 / np.clip(model.wht, 1.0e-20, 1.0e20))
# threshold = snr_threshold * bkg_error
# Estimate the 1-sigma noise in the image empirically because model.wht
# does not yet contain an IVM map
mask = (model.wht == 0)
data_mean, data_median, data_std = sigma_clipped_stats(
model.data, mask=mask, sigma=3.0, iters=10)
threshold = data_median + (data_std * snr_threshold)
sigma = kernel_fwhm * gaussian_fwhm_to_sigma
kernel = Gaussian2DKernel(sigma, x_size=kernel_xsize, y_size=kernel_ysize)
kernel.normalize()
segm = photutils.detect_sources(model.data, threshold, npixels=npixels,
filter_kernel=kernel,
connectivity=connectivity)
# source deblending requires scikit-image
if deblend:
segm = photutils.deblend_sources(model.data, segm, npixels=npixels,
filter_kernel=kernel,
nlevels=deblend_nlevels,
contrast=deblend_contrast,
mode=deblend_mode,
connectivity=connectivity,
relabel=True)
# Calculate total error, including source Poisson noise.
# This calculation assumes that the data and bkg_error images are in
# units of electron/s. Poisson noise is not included for pixels
# where data < 0.
exptime = model.meta.resample.product_exposure_time # total exptime
#total_error = np.sqrt(bkg_error**2 + np.maximum(model.data / exptime, 0))
total_error = np.sqrt(data_std**2 + np.maximum(model.data / exptime, 0))
wcs = model.get_fits_wcs()
source_props = photutils.source_properties(
model.data, segm, error=total_error, filter_kernel=kernel, wcs=wcs)
columns = ['id', 'xcentroid', 'ycentroid', 'ra_icrs_centroid',
'dec_icrs_centroid', 'area', 'source_sum',
'source_sum_err', 'semimajor_axis_sigma',
'semiminor_axis_sigma', 'orientation']
catalog = photutils.properties_table(source_props, columns=columns)
# convert orientation to degrees
catalog = QTable(catalog)
orient_deg = catalog['orientation'].to(u.deg)
catalog.replace_column('orientation', orient_deg)
# define orientation position angle
rot = _get_rotation(wcs)
catalog['orientation_sky'] = ((270. - rot +
catalog['orientation'].value) * u.deg)
# define AB mag and AB mag error
pixelarea = model.meta.photometry.pixelarea_arcsecsq
if pixelarea is None:
micro_Jy = 0.0
else:
micro_Jy = (catalog['source_sum'] *
model.meta.photometry.conversion_microjanskys *
model.meta.photometry.pixelarea_arcsecsq)
if micro_Jy > 0.0:
abmag = -2.5 * np.log10(micro_Jy) + 23.9
else:
abmag = 0.
catalog['abmag'] = abmag
# assuming SNR >> 1 (otherwise abmag_error is asymmetric):
catalog['abmag_error'] = (2.5 * np.log10(np.e) *
catalog['source_sum_err'] /
catalog['source_sum'])
return catalog
def simple_photometry(lst_of_fits_files,dither):
"""
Takes list of FPC exposure images, makes a folder to hold the segmentation results, and returns a list of results from the segmentation photometry.
INPUT: list of exposure IDs that identify the FPC images
OUTPUT: python dictionary
KEY: exposure ID
ITEM: list of information about the photometry
- path file
- area:
- Sum: sum of flux in pixels
- exptime: from FITS header (requested exposure time)
- background: mean background of image
- x: x_centroid pixel value
- y: x_centroid pixel value
- flux: sum /exptime
"""
d = {}
print(lst_of_fits_files)
list = np.array(lst_of_fits_files)
print(len(list))
for x in list:
if dither == 'tel':
id = x
d[id] = {}
filename = 'PROTODESI_FPC_%08d.fits' % x
if dither == 'pos':
filename = x
id = os.path.splitext(x)[0][-3:]
d[id] = {}
print(filename)
path_file_abs = directory+'/'+filename
d[id]['path'] = filename
[segm, props_list_return] = segmentation_photometry(path_file_abs,
bkg_sigma = 3.0,
source_snr = 3.0,
fwhm_kernel = 2.0,
x_size_kernel = 3,
y_size_kernel = 3,
clobber = False)
FIBRE_IDS = {0:(1407, 1208), 1:(1209, 1072), 2: (1093, 1864)} #fibre names
DIST_THRESHOLD = 20
expreq = fits.open(path_file_abs)[0].header['EXPREQ']
data = np.array(properties_table(props_list_return,columns=('area','source_sum','background_mean','xcentroid','ycentroid')))
area = data['area']
sume = data['source_sum']
i = np.where(area == max(area))
flux = sume[i]/expreq
d[id]['area'] = area[i]
d[id]['sum'] = sume[i]
d[id]['exptime'] = expreq
d[id]['background'] = data['background_mean'][i]
d[id]['x'] = data['xcentroid'][i]
d[id]['y'] = data['ycentroid'][i]
d[id]['flux'] = flux
return d
def make_source_catalog(model,
kernel_fwhm,
kernel_xsize,
kernel_ysize,
snr_threshold,
npixels,
deblend_nlevels=32,
deblend_contrast=0.001,
deblend_mode='exponential',
connectivity=8,
deblend=False):
"""
Create a final catalog of source photometry and morphologies.
Parameters
----------
model : `DrizProductModel`
The input `DrizProductModel` of a single drizzled image. The
input image is assumed to be background subtracted.
kernel_fwhm : float
The full-width at half-maximum (FWHM) of the 2D Gaussian kernel
used to filter the image before thresholding. Filtering the
image will smooth the noise and maximize detectability of
objects with a shape similar to the kernel.
kernel_xsize : odd int
The size in the x dimension (columns) of the kernel array.
kernel_ysize : odd int
The size in the y dimension (row) of the kernel array.
snr_threshold : float
The signal-to-noise ratio per pixel above the ``background`` for
which to consider a pixel as possibly being part of a source.
npixels : int
The number of connected pixels, each greater than the threshold
that an object must have to be detected. ``npixels`` must be a
positive integer.
deblend_nlevels : int, optional
The number of multi-thresholding levels to use for deblending
sources. Each source will be re-thresholded at
``deblend_nlevels``, spaced exponentially or linearly (see the
``deblend_mode`` keyword), between its minimum and maximum
values within the source segment.
deblend_contrast : float, optional
The fraction of the total (blended) source flux that a local
peak must have to be considered as a separate object.
``deblend_contrast`` must be between 0 and 1, inclusive. If
``deblend_contrast = 0`` then every local peak will be made a
separate object (maximum deblending). If ``deblend_contrast =
1`` then no deblending will occur. The default is 0.001, which
will deblend sources with a magnitude differences of about 7.5.
deblend_mode : {'exponential', 'linear'}, optional
The mode used in defining the spacing between the
multi-thresholding levels (see the ``deblend_nlevels`` keyword)
when deblending sources.
connectivity : {4, 8}, optional
The type of pixel connectivity used in determining how pixels
are grouped into a detected source. The options are 4 or 8
(default). 4-connected pixels touch along their edges.
8-connected pixels touch along their edges or corners. For
reference, SExtractor uses 8-connected pixels.
deblend : bool, optional
Whether to deblend overlapping sources. Source deblending
requires scikit-image.
Returns
-------
catalog : `~astropy.Table`
An astropy Table containing the source photometry and
morphologies.
"""
if not isinstance(model, DrizProductModel):
raise ValueError('The input model must be an DrizProductModel.')
# Remove until model.wht contains an IVM map
# Calculate "background-only" error assuming the weight image is an
# inverse-variance map (IVM). The weight image is clipped because it
# may contain zeros.
# bkg_error = np.sqrt(1.0 / np.clip(model.wht, 1.0e-20, 1.0e20))
# threshold = snr_threshold * bkg_error
# Estimate the 1-sigma noise in the image empirically because model.wht
# does not yet contain an IVM map
mask = (model.wht == 0)
data_mean, data_median, data_std = sigma_clipped_stats(model.data,
mask=mask,
sigma=3.0,
iters=10)
threshold = data_median + (data_std * snr_threshold)
sigma = kernel_fwhm * gaussian_fwhm_to_sigma
kernel = Gaussian2DKernel(sigma, x_size=kernel_xsize, y_size=kernel_ysize)
kernel.normalize()
segm = photutils.detect_sources(model.data,
threshold,
npixels=npixels,
filter_kernel=kernel,
connectivity=connectivity)
# source deblending requires scikit-image
if deblend:
segm = photutils.deblend_sources(model.data,
segm,
npixels=npixels,
filter_kernel=kernel,
nlevels=deblend_nlevels,
contrast=deblend_contrast,
mode=deblend_mode,
connectivity=connectivity,
relabel=True)
# Calculate total error, including source Poisson noise.
# This calculation assumes that the data and bkg_error images are in
# units of electron/s. Poisson noise is not included for pixels
# where data < 0.
exptime = model.meta.resample.product_exposure_time # total exptime
#total_error = np.sqrt(bkg_error**2 + np.maximum(model.data / exptime, 0))
total_error = np.sqrt(data_std**2 + np.maximum(model.data / exptime, 0))
wcs = model.get_fits_wcs()
source_props = photutils.source_properties(model.data,
segm,
error=total_error,
filter_kernel=kernel,
wcs=wcs)
columns = [
'id', 'xcentroid', 'ycentroid', 'ra_icrs_centroid',
'dec_icrs_centroid', 'area', 'source_sum', 'source_sum_err',
'semimajor_axis_sigma', 'semiminor_axis_sigma', 'orientation'
]
catalog = photutils.properties_table(source_props, columns=columns)
# convert orientation to degrees
catalog = QTable(catalog)
orient_deg = catalog['orientation'].to(u.deg)
catalog.replace_column('orientation', orient_deg)
# define orientation position angle
rot = _get_rotation(wcs)
catalog['orientation_sky'] = ((270. - rot + catalog['orientation'].value) *
u.deg)
# define AB mag and AB mag error
pixelarea = model.meta.photometry.pixelarea_arcsecsq
if pixelarea is None:
micro_Jy = 0.0
else:
micro_Jy = (catalog['source_sum'] *
model.meta.photometry.conversion_microjanskys *
model.meta.photometry.pixelarea_arcsecsq)
if micro_Jy > 0.0:
abmag = -2.5 * np.log10(micro_Jy) + 23.9
else:
abmag = 0.
catalog['abmag'] = abmag
# assuming SNR >> 1 (otherwise abmag_error is asymmetric):
catalog['abmag_error'] = (2.5 * np.log10(np.e) *
catalog['source_sum_err'] /
catalog['source_sum'])
return catalog
def simple_photometry(self):
"""
Takes list of FPC exposure images, makes a folder to hold the segmentation results, and returns a list of results from the segmentation photometry.
INPUT: list of exposure IDs that identify the FPC images
OUTPUT: python dictionary
KEY: exposure ID
ITEM: list of information about the photometry
- path file
- area:
- Sum: sum of flux in pixels
- exptime: from FITS header (requested exposure time)
- background: mean background of image
- x: x_centroid pixel value
- y: x_centroid pixel value
- flux: sum /exptime
"""
self.list_of_files()
for i, values in self.d.items():
path_file_abs = values['path']
#Get info from fits header
exptime = fits.open(path_file_abs)[0].header['EXPREQ']
date = fits.open(path_file_abs)[0].header['DATE-OBS']
time = fits.open(path_file_abs)[0].header['MJD-OBS']
try:
delta_ra = fits.open(path_file_abs)[0].header['DELTARA']
delta_dec = fits.open(path_file_abs)[0].header['DELTADEC']
except:
delta_ra = None
delta_dec = None
self.d[i]['exptime'] = exptime
self.d[i]['delta_ra'] = delta_ra
self.d[i]['delta_dec'] = delta_dec
self.d[i]['date'] = date
self.d[i]['time'] = time
#Do photometry
[segm, props_list_return] = segmentation_photometry(path_file_abs,
bkg_sigma = 3.0,
source_snr = 3.0,
fwhm_kernel = 2.0,
x_size_kernel = 3,
y_size_kernel = 3,
clobber = False)
#Identify individual fibers
# there may be more than 3 sources returned, depending on parameters
# for each of the fibres, look for exactly one match in sources
fibers_found = []
if self.match_type == 'location':
for fiber_num, location in self.fiber_ids.items():
for props in props_list_return:
# check distance to known fibre position
# prop.centroid is in (y, x) format
dist = np.linalg.norm(
np.fliplr([props.centroid.value])[0]
- location)
if dist < self.DIST_THRESHOLD:
fibers_found.append(fiber_num)
print("Fiber %d was identified" % fiber_num)
self.d[i][fiber_num] = {}
self.d[i][fiber_num]['seg_object'] = props
# remove the props of identified source from props list
#props_list_return.remove(props)
# stop searching for current fibre
#break
for fiber_num in fibers_found:
props = self.d[i][fiber_num]['seg_object']
data = np.array(properties_table(props,columns=('area','source_sum','background_mean','centroid','max_value')))
self.d[i][fiber_num]['area'] = data['area']
self.d[i][fiber_num]['sum'] = data['source_sum']
self.d[i][fiber_num]['background'] = data['background_mean']
self.d[i][fiber_num]['centroid'] = data['centroid']
self.d[i][fiber_num]['flux'] = data['source_sum']/exptime
self.d[i][fiber_num]['max_value'] = data['max_value']
#else:
# # this is not the fibre propwe are looking for
# print('Cannot match fiber %d' % fiber_num)
elif self.match_type == 'max':
if self.max_fiber_num == 'all':
print("Can only run max for single fibers")
else:
try:
max_fiber = int(self.max_fiber_num)
self.d[i][max_fiber] = {}
except:
print("fiber number has to be 3001, 3002, 3003")
try:
all_data = np.array(properties_table(props_list_return,columns=('area')))
area = all_data['area']
x = np.where(area == max(area))
if len(x[0]) > 1:
print("Couldn't find a maximum")
elif len(x[0]) == 1:
prop = props_list_return[x[0]]
self.d[i][max_fiber]['seg_object'] = prop
#Put info in dictionary
props = self.d[i][max_fiber]['seg_object']
data = np.array(properties_table(props,columns=('area','source_sum','background_mean','centroid','max_value')))
self.d[i][max_fiber]['area'] = data['area']
self.d[i][max_fiber]['sum'] = data['source_sum']
self.d[i][max_fiber]['background'] = data['background_mean']
self.d[i][max_fiber]['centroid'] = data['centroid']
self.d[i][max_fiber]['flux'] = data['source_sum']/exptime
self.d[i][max_fiber]['max_value'] = data['max_value']
except:
print("no area to speak of")
def segmentation_photometry(path_file_abs,
bkg_sigma=3.0,
source_snr=3.0,
fwhm_kernel=2.0,
x_size_kernel=3,
y_size_kernel=3,
clobber=False):
"""
aperture photometry from source segmentation
make_source_mask not yet available in photutils v0.2.2, this version
manually creates a source mask for determining background
"""
import os
import copy
import glob
import pickle
import numpy as np
from scipy import ndimage
import matplotlib
#matplotlib.rcParams['text.usetex'] = True
#matplotlib.rcParams['text.latex.unicode'] = True
#from matplotlib.backends.backend_pdf import PdfPages
import matplotlib.pyplot as plt
from astropy.io import fits, ascii
from astropy.convolution import Gaussian2DKernel
from astropy.stats import sigma_clipped_stats, gaussian_fwhm_to_sigma
# from astropy.table import Table
from astropy.visualization import (LogStretch, mpl_normalize)
# from astropy.extern.six.moves import StringIO
from photutils import (detect_threshold, EllipticalAperture,
source_properties, properties_table)
from photutils.detection import detect_sources
from photutils.utils import random_cmap
# create preliminary mask
#from photutils import make_source_mask
#masterMask = make_source_mask(master, snr=2, npixels=5, dilate_size=11)
# if LEDoff was used, get threshold from LEDoff/background
# path_dataset = os.path.dirname(path_file_abs) + os.path.sep
# filenameCombined = '\t'.join(
# os.listdir(os.path.join(datasetDirLocal, 'master')))
# if 'master_ledoff_subtracted' in filename:
# print('Using master_ledoff')
# # path_file_abs = os.path.join(datasetDir, 'master', filename)
# hdu = fits.open(path_file_abs)[0]
# data_subtracted = hdu.data
# # calculate threadhold
# ledoff_pred = np.mean(data_subtracted) * \
# np.ones(data_subtracted.shape)
# mse = mean_squared_error(data_subtracted, ledoff_pred)
# rmse = np.sqrt(mse)
# threshold = 7.0 * rmse
# threshold_value = threshold
# if no LEDoff was used, background subtraction is needed
# there should exist no file named "subtracted"
# if 'master.fit' in filenameCombined \
# or 'master_normalised.fit' in filenameCombined:
#filenamedir = os.path.basename(path_file_abs)
#print(filenamedir)
#New stuff made by Parker
f_dir, filename = os.path.split(path_file_abs)
ff = os.path.splitext(filename)[0]
new_dir = f_dir + '/' + ff
if not os.path.exists(new_dir):
os.makedirs(new_dir)
dir_save = new_dir
print("The photometry files will be saved in ", dir_save)
filenames_combined = '\t'.join(os.listdir(dir_save))
if clobber == False \
and 'segm.obj' in filenames_combined \
and 'props.obj' in filenames_combined \
and 'props.csv' in filenames_combined\
and 'props.ecsv' in filenames_combined:
print('Photometry properties table already exists. Reading objects...')
segm = pickle.load(
open(glob.glob(os.path.join(dir_save, '*segm.obj*'))[0], 'rb'))
props = pickle.load(
open(glob.glob(os.path.join(dir_save, '*props.obj*'))[0], 'rb'))
return [segm, props]
if 'master' in path_file_abs:
if 'normalised' in path_file_abs:
print('Performing photometry to ' +
'normalised master object image {}...'.format(path_file_abs))
else:
print('Performing photometry to ' +
'un-normalised master image {}...'.format(path_file_abs))
else:
print('Warning: Photometry being performed to ' +
'a single exposure {}...'.format(path_file_abs))
hdu = fits.open(path_file_abs)[0]
data = hdu.data
header = hdu.header
if 'EXPREQ' in header:
exptime = header['EXPREQ']
elif 'EXPTIME' in header:
exptime = header['EXPTIME']
else:
print('Exposure time not found in header. Cannot determine magnitude.')
exptime = np.nan
# === Iteratively determine background level ===
# assuming background is homogenous, estimate background by sigma clipping
# if background noise varies across image, generate 2D background instead
print('Determining background noise level...' '')
[mean, median, std] = sigma_clipped_stats(data, sigma=bkg_sigma, iters=5)
threshold = median + (std * 2.0)
segm = detect_sources(data, threshold, npixels=5)
# turn segm into a mask
mask = segm.data.astype(np.bool)
# dilate the source mask to ensure complete masking of detected sources
dilate_structure = np.ones((5, 5))
mask_dilated = ndimage.binary_dilation(mask, structure=dilate_structure)
# get sigma clipping stats of background, without sources that are masekd
[bkg_mean, bkg_median, bkg_std] = sigma_clipped_stats(data,
sigma=bkg_sigma,
mask=mask_dilated,
iters=3)
# === Detect sources by segmentation ===
print('Determining threshold for source detection...')
# determine threshold for source detection
# in current implementation, if all inputs are present, the formula is
# threshold = background + (background_error * snr)
threshold = detect_threshold(data,
background=bkg_median,
error=bkg_std,
snr=source_snr)
print('Preparing 2D Gaussian kernal...')
sigma_kernel = fwhm_kernel * gaussian_fwhm_to_sigma
kernel = Gaussian2DKernel(sigma_kernel,
x_size=x_size_kernel,
y_size=y_size_kernel)
# normalise kernel
# The kernel models are normalized per default, ∫∞−∞f(x)dx=1∫−∞∞f(x)dx=1.
# But because of the limited kernel array size, the normalization
# for kernels with an infinite response can differ from one.
kernel.normalize()
# obtain a SegmentationImage object with the same shape as the data,
# where sources are labeled by different positive integer values.
# A value of zero is always reserved for the background.
# if the threshold includes the background level as above, then the image
# input into detect_sources() should not be background subtracted.
print('Segmentation processing...')
segm = detect_sources(data, threshold, npixels=5, filter_kernel=kernel)
print('Segmentation labels are: ', repr(segm.labels))
# === Measure regional source properties ===
# source_properties() assumes that the data have been background-subtracted.
# Background is the background level that was previously present
# in the input data.
# The input background does not get subtracted from the input data,
# which should already be background-subtracted.
print('Extracting source properties...')
props = source_properties(data - bkg_median, segm, background=bkg_median)
# add flux and instrumental magnitude to properties
# flux = source_sum / exptime
# instrumental magnitude = -2.5 * log10(flux)
for i in range(len(props)):
# source_sum is by definition background-subtracted already
props[i].flux = props[i].source_sum / exptime
props[i].mag_instr = -2.5 * np.log10(props[i].flux)
# make plots and save to images
# define approximate isophotal ellipses for each object
apertures = []
r = 2.8 # approximate isophotal extent
for prop in props:
position = (prop.xcentroid.value, prop.ycentroid.value)
a = prop.semimajor_axis_sigma.value * r
b = prop.semiminor_axis_sigma.value * r
theta = prop.orientation.value
apertures.append(EllipticalAperture(position, a, b, theta=theta))
# create a table of properties
try:
props_table = properties_table(props)
except:
print('No source detected in {}'.format(path_file_abs))
return [None, None]
props_table['flux'] = [props[i].flux for i in range(len(props))]
props_table['mag_instr'] = [props[i].mag_instr for i in range(len(props))]
# add custom columns to the table: mag_instru and flux
# plot centroid and segmentation using approximate elliptical apertures
norm = mpl_normalize.ImageNormalize(stretch=LogStretch())
rand_cmap = random_cmap(segm.max + 1, random_state=12345)
#[fig1, (ax1, ax2)] = plt.subplots(1, 2, figsize = (12, 6))
#ax1.imshow(data, origin='lower', cmap=plt.cm.gray, norm=norm)
#ax1.plot(
# props_table['xcentroid'], props_table['ycentroid'],
# ls='none', color='blue', marker='+', ms=10, lw=1.5)
#ax2.imshow(segm, origin='lower', cmap=rand_cmap)
#for aperture in apertures:
# aperture.plot(ax=ax1, lw=1.0, alpha=1.0, color='red')
# aperture.plot(ax=ax2, lw=1.0, alpha=1.0, color='red')
# plot using actual segmentation outlines (to be improved)
#[fig2, ax3] = plt.subplots(figsize = (6, 6))
#ax3.imshow(data, origin='lower', cmap=plt.cm.gray, norm=norm)
#segm_outline = np.array(segm.outline_segments(), dtype=float)
#segm_outline[segm_outline<1] = np.nan
# get a copy of the gray color map
#segm_outline_cmap = copy.copy(plt.cm.get_cmap('autumn'))
# set how the colormap handles 'bad' values
#segm_outline_cmap.set_bad(alpha=0)
#ax3.imshow(segm_outline, origin='lower', cmap=segm_outline_cmap)
# === save ===
# Save segm, porps to object files, and also save props to table file.
print('Saving segmentation and source propdderties to {}...'.format(
dir_save))
try:
# if filename ends with fits, remove it in the filename
if filename[-5:] == '.fits':
dir_save_prefix = os.path.join(dir_save, filename[0:-5])
else:
dir_save_prefix = os.path.join(dir_save, filename)
# Enhanced CSV allows preserving table meta-data such as
# column data types and units.
# In this way a data table can be stored and read back as ASCII
# with no loss of information.
ascii.write(props_table,
dir_save_prefix + '-phot_props.ecsv',
format='ecsv')
# csv for readability in MS excel
ascii.write(props_table,
dir_save_prefix + '-phot_props.csv',
format='csv')
# dump segmentation and properties to object files in binary mode
file_segm = open(dir_save_prefix + '-phot_segm.obj', 'wb')
pickle.dump(segm, file_segm)
file_props = open(dir_save_prefix + '-phot_props.obj', 'wb')
pickle.dump(props, file_props)
# save figures
#fig1.savefig(dir_save_prefix + '-phot_segm_fig1.png', dpi=600)
#pp1 = PdfPages(dir_save_prefix + '-phot_segm_fig1.pdf')
#pp1.savefig(fig1)
#pp1.close()
#fig2.savefig(dir_save_prefix + '-phot_segm_fig2.png', dpi=600)
#pp2 = PdfPages(dir_save_prefix + '-phot_segm_fig2.pdf')
#pp2.savefig(fig2)
#pp2.close()
print('Segmentation, properties objects, tables, and images saved to',
dir_save)
except:
print('Unable to write to disk, check permissions.')
return [segm, props]
def segmentation_photometry(path_image_abs,
path_error_abs = None,
logger = None,
bkg_sigma = 1.5,
source_snr = 1.05,
fwhm_kernel = 25,
x_size_kernel = 100,
y_size_kernel = 80,
dump_pickle = False,
clobber = True):
"""
given a fits file (master image), this function calculates
photometry by source segmentation.
make_source_mask not yet available in photutils v0.2.2, this version
manually creates a source mask for determining background.
"""
def msg(string, msgtype = None):
if logger == None:
print(string)
else:
print(string)
if msgtype == 'info':
logger.info(string)
if msgtype == 'error':
logger.error(string)
if msgtype == 'warning':
logger.warning(string)
filename = os.path.basename(path_image_abs)
dir_save = os.path.dirname(path_image_abs)
filenames_combined = '\t'.join(os.listdir(dir_save))
if clobber == False \
and filename[0:-5]+'-segm.obj' in filenames_combined \
and filename[0:-5]+'-props.obj' in filenames_combined \
and filename[0:-5]+'-centroid_outline.png' in filenames_combined \
and filename[0:-5]+'-centroid_outline.pdf' in filenames_combined \
and filename[0:-5]+'-segmentation.png' in filenames_combined \
and filename[0:-5]+'-segmentation.pdf' in filenames_combined:
msg('Photometry properties table already exists. '
+ 'Reading pickles...',
msgtype='info')
try:
segm = pickle.load(open(glob.glob(os.path.join(
dir_save, filename[0:-5]+'-segm.obj*'))[0],
'rb'))
props_list = pickle.load(open(glob.glob(os.path.join(
dir_save, filename[0:-5]+'-props.obj*'))[0],
'rb'))
return [segm, props_list]
except:
# pickle file corrupt or empty, proceed
pass
elif clobber == False \
and filename[0:-5]+'-logstretch.png' in filenames_combined \
and filename[0:-5]+'-logstretch.pdf' in filenames_combined:
msg('Non-detection from previous results.',
msgtype='info')
return [None, []]
# image type notifications
if 'master' in path_image_abs:
if 'normalised' in path_image_abs:
msg('Performing photometry to '
+ 'normalised master object image {}...'.format(path_image_abs),
msgtype='info')
else:
msg('Performing photometry to '
+ 'un-normalised master image {}...'.format(path_image_abs),
msgtype='info')
elif 'reduced' in path_image_abs:
msg('Performing photometry to '
+ 'reduced image frame {}...'.format(path_image_abs),
msgtype='info')
else:
msg('Warning: Photometry being performed to '
+ 'a single exposure {}...'.format(path_image_abs),
msgtype='warning')
# read in data
try:
hdu = fits.open(path_image_abs)['FPC']
data = hdu.data
except:
hdu = fits.open(path_image_abs)[0]
data = hdu.data
# read in error in data
msg('Reading master error image {}...'
.format(path_error_abs))
try:
hdu_error = fits.open(path_error_abs)[0]
data_error = hdu_error.data
except:
data_error = np.zeros(data.shape)
msg('No master error image available for {}'
.format(path_image_abs))
header = hdu.header
if 'EXPREQ' in header:
exptime = header['EXPREQ']
elif 'EXPTIME' in header:
exptime = header['EXPTIME']
else:
msg('Exposure time not found in header. '
+ 'Cannot determine magnitude.',
msgtype='error')
exptime = np.nan
# === Iteratively determine background level ===
# assuming backcground is homogenous, estimate background by sigma clipping
# if background noise varies across image, generate 2D background instead
# using the Background function
msg('Determining background noise level...',
msgtype='info')
[mean, median, std] = sigma_clipped_stats(data, sigma=bkg_sigma, iters=3)
threshold = median + (std * 4)
segm = detect_sources(data, threshold, npixels=5)
# turn segm into a mask
mask = segm.data.astype(np.bool)
# dilate the source mask to ensure complete masking of detected sources
dilate_structure = np.ones((5, 5))
mask_dilated = ndimage.binary_dilation(mask, structure=dilate_structure)
# get sigma clipping stats of background, without sources that are masekd
[bkg_mean, bkg_median, bkg_std] = sigma_clipped_stats(
data, sigma=bkg_sigma, mask=mask_dilated, iters = 3)
# === Detect sources by segmentation ===
msg('Determining threshold for source detection...',
msgtype='info')
# determine threshold for source detection
# in current implementation, if all inputs are present, the formula is
# threshold = background + (background_error * snr)
threshold = detect_threshold(data,
background = bkg_median,
error = data_error+bkg_std,
snr = source_snr)
# calculate total error including poisson statistics
try:
# this is for v0.3 and above
msg('Calculating total errors including background and Poisson...',
msgtype='info')
err_tot = calc_total_error(data,
bkg_error= data_error+bkg_std,
effective_gain=0.37)
gain = None
# in version earlier than 0.3, this function is not available
except:
# error must be of the same shape as the data array
# this is for v0.2.2
err_tot = data_error + bkg_std
gain = 0.37
msg('Preparing 2D Gaussian kernal...',
msgtype='info')
sigma_kernel = fwhm_kernel * gaussian_fwhm_to_sigma
kernel = Gaussian2DKernel(sigma_kernel,
x_size = x_size_kernel,
y_size = y_size_kernel)
# normalise kernel
# The kernel models are normalized per default, ∫∞−∞f(x)dx=1∫−∞∞f(x)dx=1.
# But because of the limited kernel array size, the normalization
# for kernels with an infinite response can differ from one.
kernel.normalize()
# obtain a SegmentationImage object with the same shape as the data,
# where sources are labeled by different positive integer values.
# A value of zero is always reserved for the background.
# if the threshold includes the background level as above, then the image
# input into detect_sources() should not be background subtracted.
msg('Segmentation processing...',
msgtype='info')
segm = detect_sources(data, threshold, npixels=5, filter_kernel=kernel)
msg('Segmentation labels are: ' + repr(segm.labels),
msgtype='info')
# === Measure regional source properties ===
# source_properties() assumes that the data have been background-subtracted.
# Background is the background level that was previously present
# in the input data.
# The input background does not get subtracted from the input data,
# which should already be background-subtracted.
msg('Extracting source properties...',
msgtype='info')
if gain is None:
# gain is no longer supported in v0.3 and included in total error array
props_list = source_properties(data-bkg_median, segm,
background = bkg_median,
error = err_tot)
else: # still in v0.2.2
props_list = source_properties(data-bkg_median, segm,
background = bkg_median,
error = err_tot,
effective_gain = gain)
# add more properties that are not automatically calculated
for i in range(len(props_list)):
# source_sum is by definition background-subtracted already
props_list[i].flux = props_list[i].source_sum/exptime
props_list[i].flux_err = props_list[i].source_sum_err/exptime
# flux = source_sum / exptime
# instrumental magnitude = -2.5 * log10(flux)
props_list[i].mag_instr = -2.5 * np.log10(props_list[i].flux)
props_list[i].mag_instr_err = -2.5 / props_list[i].flux / np.log(10) \
* props_list[i].flux_err
# assuming fwhm of a circule gaussian of the same cross section area
props_list[i].fwhm = gaussian_sigma_to_fwhm * np.sqrt(
props_list[i].semimajor_axis_sigma.value
* props_list[i].semiminor_axis_sigma.value)
# make plots and save to images
# define approximate isophotal ellipses for each object
apertures = []
r = 5 # approximate isophotal extent
for props in props_list:
position = (props.xcentroid.value, props.ycentroid.value)
a = props.semimajor_axis_sigma.value * r
b = props.semiminor_axis_sigma.value * r
theta = props.orientation.value
apertures.append(EllipticalAperture(position, a, b, theta=theta))
# === plot and save ===
# if filename ends with fits, remove it in the filename
if filename[-5:] == '.fits':
path_save_prefix = os.path.join(dir_save, filename[0:-5])
else:
path_save_prefix = os.path.join(dir_save, filename)
norm_log = mpl_normalize.ImageNormalize(vmin=0, vmax=2000,
stretch = LogStretch())
if len(props_list) > 0:
# Save segm, porps to object files, and also save props to table file.
msg('Saving segmentation and source properties to {}...'
.format(dir_save),
msgtype='info')
# at least one source was detected
# create a table of properties
props_table = properties_table(props_list)
# add custom columns to the table: mag_instru and flux
props_table['flux'] = [props_list[i].flux
for i in range(len(props_list))]
props_table['flux_err'] = [props_list[i].flux_err
for i in range(len(props_list))]
props_table['mag_instr'] = [props_list[i].mag_instr
for i in range(len(props_list))]
props_table['mag_instr_err'] = [props_list[i].mag_instr_err
for i in range(len(props_list))]
props_table['fwhm'] = [props_list[i].fwhm
for i in range(len(props_list))]
# plot centroid and segmentation outline
[fig1, ax1] = plt.subplots(figsize=(4, 3))
ax1.imshow(data, origin='lower', cmap=plt.cm.gray, norm=norm_log)
ax1.plot(props_table['xcentroid'], props_table['ycentroid'],
linestyle='none', color='red',
marker='+', markersize=2, markeredgewidth=0.1, alpha=1)
segm_outline = np.array(segm.outline_segments(), dtype=float)
segm_outline[segm_outline<1] = np.nan
# get a copy of the gray color map
segm_outline_cmap = plt.cm.winter
# set how the colormap handles 'bad' values
segm_outline_cmap.set_bad(alpha=0)
ax1.imshow(segm_outline,
origin='lower', cmap=segm_outline_cmap, alpha=1)
ax1.get_xaxis().set_visible(False)
ax1.get_yaxis().set_visible(False)
fig1.tight_layout()
# segmentation image and aperture using approximate elliptical apertures
[fig2, ax2] = plt.subplots(figsize=(4, 3))
rand_cmap = random_cmap(segm.max + 1, random_state=8)
ax2.imshow(segm, origin='lower', cmap=rand_cmap)
ax2.plot(props_table['xcentroid'], props_table['ycentroid'],
linestyle='none', color='red',
marker='+', markersize=2, markeredgewidth=0.1, alpha=1)
for aperture in apertures:
aperture.plot(ax=ax2, lw=0.1, alpha=1, color='lime')
ax2.axis('off')
ax2.get_xaxis().set_visible(False)
ax2.get_yaxis().set_visible(False)
fig2.tight_layout()
try:
# Enhanced CSV allows preserving table meta-data such as
# column data types and units.
# In this way a data table can be stored and read back as ASCII
# with no loss of information.
ascii.write(props_table, path_save_prefix + '-props.ecsv',
format = 'ecsv')
# csv for readability in MS excel
ascii.write(props_table, path_save_prefix + '-props.csv',
format = 'csv')
# save figures
fig1.savefig(path_save_prefix + '-centroid_outline.png',
bbox_inches='tight', pad_inches=0, dpi=1200)
fig2.savefig(path_save_prefix + '-segmentation.png',
bbox_inches='tight', pad_inches=0, dpi=2000)
pp1 = PdfPages(path_save_prefix + '-centroid_outline.pdf')
pp1.savefig(fig1, dpi=1200)
pp1.close()
pp2 = PdfPages(path_save_prefix + '-segmentation.pdf')
pp2.savefig(fig2, dpi=2000)
pp2.close()
if dump_pickle:
# dump segmentation and properties to objects in binary mode
file_segm = open(path_save_prefix + '-segm.obj', 'wb')
pickle.dump(segm, file_segm)
file_props = open(path_save_prefix + '-props.obj', 'wb')
pickle.dump(props_list, file_props)
msg('Segmentation, properties objects, tables, and images '
+ 'saved to {}'.format(dir_save),
msgtype='info')
except:
msg('Unable to write to disk, check permissions.',
msgtype='error')
# memory leak?
try:
plt.close('all')
del (hdu, hdu_error, data, data_error, header, mask, mask_dilated,
err_tot, kernel, apertures, norm_log, props_table,
segm_outline, segm_outline_cmap, rand_cmap,
fig1, ax1, fig2, ax2, pp1, pp2, file_segm, file_props)
except:
pass
return [segm, props_list]
else:
msg('No source detected in {}'.format(path_image_abs),
msgtype='warning')
# save log scale stretched image, if no source was detected
[fig0, ax0] = plt.subplots(figsize=(4, 3))
ax0.imshow(data, origin='lower', cmap=plt.cm.gray, norm=norm_log)
ax0.get_xaxis().set_visible(False)
ax0.get_yaxis().set_visible(False)
try:
fig0.savefig(path_save_prefix + '-logstretch.png',
bbox_inches='tight', pad_inches=0, dpi=1200)
pp0 = PdfPages(path_save_prefix + '-logstretch.pdf')
pp0.savefig(fig0, dpi=1200)
pp0.close()
except:
msg('Unable to write to disk, check permissions.',
msgtype='error')
return [None, []]
def aperture_photometry(self, filename):
# aperture photometry from source segmentation
# determine threshold for background detection
# if LEDoff was used, get threshold from LEDoff/background
filepath = os.path.join(datasetDirLocal, 'master', filename)
filenameCombined = '\t'.join(
os.listdir(os.path.join(datasetDirLocal, 'master')))
if 'master_ledoff_subtracted' in filename:
self.msg('Using master_ledoff')
# filepath = os.path.join(datasetDir, 'master', filename)
hdu = fits.open(filepath)[0]
data_subtracted = hdu.data
# calculate threadhold
ledoff_pred = np.mean(data_subtracted) * np.ones(
data_subtracted.shape)
mse = mean_squared_error(data_subtracted, ledoff_pred)
rmse = np.sqrt(mse)
threshold = 7.0 * rmse
threshold_value = threshold
# if no LEDoff was used, background subtraction is needed
# there should exist no file named "subtracted"
elif 'master.fit' in filenameCombined \
or 'master_normalised.fit' in filenameCombined:
self.ui.statusbar.showMessage('Using master or master_normalised')
# create preliminary mask
""" make_source_mask not yet available in photutils v0.2.1
wait for v0.3 release
"""
#from photutils import make_source_mask
#masterMask = make_source_mask(master, snr=2, npixels=5, dilate_size=11)
# background subtraction
""" create 2D image of background and background rms and
apply sigma-clipping to each region in the low-res
background map to get mean, median, and std/rms.
sigma-clipping is the most widely used method though not as
good as using mask; still superior to robust standard
deviation using median absolute deviation (MAD-STD)
"""
hdu = fits.open(filepath)[0]
data = hdu.data
if 'EXPTIME' in hdu.header:
exptime = hdu.header['EXPTIME']
else:
exptime = hdu.header['EXPREQ']
self.msg('Determining threshold for target detection...')
# calculate threashold
# [mean, median, std] = sigma_clipped_stats(master, sigma=3.0, iters=5)
bkg = Background(data, (100, 100),
filter_shape=(3, 3),
method='median')
# bkg = Background(master, (50, 50), filter_size=(3, 3), method='median')
# plt.imshow(bkg.background, norm=normalisation, origin='lower', cmap=plt.cm.gray)
plt.imshow(bkg.background, origin='lower', cmap=plt.cm.gray)
[fig, ax] = plt.subplots(figsize=(8, 8))
# make background-substracted image
data_subtracted = data - bkg.background
# plot
plt.imshow(data_subtracted, origin='lower', cmap=plt.cm.gray)
# save background subtracted image
if 'master.fit' in filename:
hdu_subtracted = fits.PrimaryHDU(data_subtracted)
hdu_subtracted.writeto('master_subtracted.fits', clobber=True)
elif 'master_normalised.fit' in filename:
hdu_normalised_subtracted = fits.PrimaryHDU(data_subtracted)
hdu_normalised_subtracted.writeto(
'master_normalised_subtracted.fits', clobber=True)
# segmentation at a given sigma level, for regional properties
threshold = 5.0 * bkg.background_rms # since data is background-subtracted
threshold_value = threshold.flat[0]
self.msg('Threshold for target detection is: ' + repr(threshold_value))
# perform segmentation whether flat was available or not
self.msg('Performing segmentation...')
segm = detect_sources(data_subtracted, threshold, npixels=5)
self.msg('Segmentation labels are:')
self.msg((str(segm.labels)))
# measure regional source properties from segmentation
# the centroid is from image moments, already intensity-weighted
self.msg('Measuring source properties')
if 'bkg' in locals():
props = source_properties(data_subtracted,
segm,
error=bkg.background_rms,
background=bkg.background)
elif 'master_ledoff_subtracted' in filenameCombined:
filepath = os.path.join(datasetDirLocal, 'master',
'master_ledoff_subtracted.fits')
hdu = fits.open(filepath)[0]
master_ledoff_subtracted = hdu.data
props = source_properties(data_subtracted,
segm,
error=master_ledoff_subtracted -
np.mean(master_ledoff_subtracted),
background=master_ledoff_subtracted)
# instrumental magnitude = -2.5 * log10(flux)
for i in range(len(props)):
props[i].mag_instr = -2.5 * np.log10(props[i].source_sum / exptime)
# source_sum are by definition background-subtracted already
propsTableColumns = [
'id', 'xcentroid', 'ycentroid', 'area', 'max_value', 'source_sum',
'mag_instr'
]
# there are other properties available, see list of SourceProperties
# http://photutils.readthedocs.io/en/latest/api/photutils.segmentation.SourceProperties.html#photutils.segmentation.SourceProperties
propsTable = properties_table(props, columns=propsTableColumns)
self.ui.statusbar.showMessage(repr(propsTable))
# plot segmentated image
self.rmmpl()
segFig = Figure()
cmapRand = random_cmap(segm.max + 1, random_state=12345)
axes = segFig.add_subplot(111)
axes.imshow(segm, origin='lower', cmap=cmapRand)
axes.plot(propsTable['xcentroid'],
propsTable['ycentroid'],
ls='none',
color='red',
marker='+',
ms=10,
lw=1.5)
self.addmpl(segFig)
# set properties table font and font size
self.ui.tablePhot.setCurrentFont(QtGui.QFont('Courier'))
self.ui.tablePhot.setFontPointSize(9)
self.ui.tablePhot.setPlainText(repr(propsTable))
self.msg('Photometry completed')
# seg_flux=np.bincount(np.ravel(seg_data), weights=np.ravel(shared_im))
# for i in range(seg_nid):
# cat_data[i]=(i, seg_npix[i], seg_flux[i], 0., 0.)
# cat_data=cat_data[1:]
sys.exit()
cat_properties = segment_properties(shared_im,
seg_data,
background=0.,
mask=im_mask_nan)
cat_columns = [
'id', 'xcentroid', 'ycentroid', 'segment_sum', 'area',
'semimajor_axis_sigma', 'semiminor_axis_sigma', 'elongation'
]
cat_data = properties_table(cat_properties, columns=cat_columns)
gv_finite = (np.isfinite(cat_data['xcentroid'])
& np.isfinite(cat_data['ycentroid']))
cat_data = cat_data[gv_finite]
gv_sort = np.argsort(cat_data['area'])[::-1]
cat_data = cat_data[gv_sort]
gv_mask = (cat_data['area'] > mask_area_min)
fig = plt.figure()
ax = plt.gca()
ax.plot(np.sqrt(cat_data['area']),
cat_data['segment_sum'],
linestyle='',
marker='o',
markersize=5,
# cat_data=np.recarray(seg_nid, dtype={'names':['id','npix','flux','mag','fwhm'], 'formats':['i4','i4','f4','f4','f4']})
# cat_data.fill(0)
# print 'Computing the flux and size for the detected sources'
# seg_npix=np.bincount(np.ravel(seg_data))
# seg_flux=np.bincount(np.ravel(seg_data), weights=np.ravel(shared_im))
# for i in range(seg_nid):
# cat_data[i]=(i, seg_npix[i], seg_flux[i], 0., 0.)
# cat_data=cat_data[1:]
sys.exit()
cat_properties = segment_properties(shared_im, seg_data, background=0., mask=im_mask_nan)
cat_columns = ['id', 'xcentroid', 'ycentroid', 'segment_sum', 'area', 'semimajor_axis_sigma', 'semiminor_axis_sigma', 'elongation']
cat_data = properties_table(cat_properties, columns=cat_columns)
gv_finite=(np.isfinite(cat_data['xcentroid']) & np.isfinite(cat_data['ycentroid']))
cat_data=cat_data[gv_finite]
gv_sort=np.argsort(cat_data['area'])[::-1]
cat_data=cat_data[gv_sort]
gv_mask=(cat_data['area'] > mask_area_min)
fig = plt.figure()
ax = plt.gca()
ax.plot(np.sqrt(cat_data['area']), cat_data['segment_sum'], linestyle='', marker='o', markersize=5, c='blue', alpha=0.5, markeredgecolor='none')
ax.plot(np.sqrt(cat_data['area'][gv_mask]), cat_data['segment_sum'][gv_mask], linestyle='', marker='o', markersize=5, c='red', alpha=0.5, markeredgecolor='none')
ax.set_yscale('log')
ax.set_xlabel('Sqrt(Area) (pix)')
ax.set_ylabel('Flux (ADU)')
fig.savefig('plot_flux_area_tile1_g.pdf', format='pdf')
def prepare_data(file, data_dir, folder):
"""
prepare_data picks a file with the image of the galaxy, detect the central object, rotate it to the major axis, and returns
the data and errors ready to fit a warp curve, along with the maximum distance from the center
"""
# check if there is a folder for the figures, if not create it
if not os.path.isdir('../figs/' + str(folder)):
os.mkdir('../figs/' + str(folder))
# check if there is a folder for the text output, if not create it
if not os.path.isdir('../output/' + str(folder)):
os.mkdir('../output/' + str(folder))
print(data_dir + '/' + str(file))
hdu = fits.open(data_dir + '/' + str(file[:-1]))[
0] #fits.open(data_dir+'/'+str(file[:-1]))[0]
wcs = WCS(hdu.header)
data = hdu.data
sigma_clip = SigmaClip(sigma=3., iters=10)
bkg_estimator = MedianBackground()
bkg = Background2D(data, (25, 25),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
if (cfg.DATA_TYPE == 'REAL'):
weight = fits.open(data_dir + '/' + str(file[:-1]))[1].data
else:
weight = bkg.background_rms
threshold = bkg.background + (3. * bkg.background_rms)
sigma = 2.0 * gaussian_fwhm_to_sigma # FWHM = 2.
kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
kernel.normalize()
segm = detect_sources(data, threshold, npixels=5, filter_kernel=kernel)
rand_cmap = random_cmap(segm.max + 1, random_state=12345)
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 8))
ax1.imshow(data, origin='lower', cmap='Greys_r')
ax2.imshow(segm, origin='lower', cmap=rand_cmap)
plt.savefig('../figs/' + str(folder) + '/' + str(file[:-5]) + 'fig2.png')
plt.close()
props = source_properties(data, segm)
tbl = properties_table(props)
my_min = 100000.
x_shape = np.float(data.shape[0])
y_shape = np.float(data.shape[1])
r = 3. # approximate isophotal extent
apertures = []
for prop in props:
position = (prop.xcentroid.value, prop.ycentroid.value)
a = prop.semimajor_axis_sigma.value * r
b = prop.semiminor_axis_sigma.value * r
theta = prop.orientation.value
apertures.append(EllipticalAperture(position, a, b, theta=theta))
my_dist = np.sqrt((prop.xcentroid.value - x_shape / 2.)**2 +
(prop.ycentroid.value - y_shape / 2.)**2)
if (my_dist < my_min):
my_label = prop.id - 1
my_min = my_dist
mytheta = props[my_label].orientation.value
mysize = np.int(np.round(r * props[my_label].semimajor_axis_sigma.value))
my_x = props[my_label].xcentroid.value
my_y = props[my_label].ycentroid.value
mask_obj = np.ones(data.shape, dtype='bool')
mask_obj[(segm.data != 0) * (segm.data != props[my_label].id)] = 0
weigth = weight[mask_obj]
rand_cmap = random_cmap(segm.max + 1, random_state=12345)
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8, 8))
ax1.imshow(data, origin='lower', cmap='Greys_r')
ax2.imshow(segm, origin='lower', cmap=rand_cmap)
for aperture in apertures:
aperture.plot(color='blue', lw=1.5, alpha=0.5, ax=ax1)
aperture.plot(color='white', lw=1.5, alpha=1.0, ax=ax2)
plt.savefig('../figs/' + str(folder) + '/' + str(file[:-5]) + 'fig3.png')
plt.close()
data_rot = rotate(data, np.rad2deg(mytheta))
data_rot = data_rot[data_rot.shape[0] / 2 - 100:data_rot.shape[0] / 2 +
100, data_rot.shape[1] / 2 -
100:data_rot.shape[1] / 2 + 100]
w_rot = rotate(weight, np.rad2deg(mytheta))
w = w_rot[w_rot.shape[0] / 2 - 100:w_rot.shape[0] / 2 + 100,
w_rot.shape[1] / 2 - 100:w_rot.shape[1] / 2 + 100]
plt.figure()
plt.imshow(data_rot, origin='lower', cmap='Greys_r')
plt.savefig('../figs/' + str(folder) + '/' + str(file[:-5]) + 'fig4.png')
plt.close()
newx, newy, newtheta, newsize = find_centroid(data_rot)
print('old center = ', my_x, my_y, mysize)
print('new center = ', newx, newy, np.rad2deg(newtheta), newsize)
x_shape2 = np.float(data_rot.shape[0])
y_shape2 = np.float(data_rot.shape[1])
np.savetxt(
'../output/' + str(folder) + '/' + str(file[:-5]) +
'_size_xcent_ycent_xy_shape.txt',
np.array([newsize, newx, newy, x_shape2, y_shape2]))
return data_rot, w, newsize, newx, newy, x_shape2, y_shape2
segm,
error=master_ledoff_subtracted -
np.mean(master_ledoff_subtracted),
background=master_ledoff_subtracted)
# instrumental magnitude = -2.5 * log10(flux)
for i in range(len(props)):
props[i].mag_instr = -2.5 * np.log10(props[i].source_sum)
propsTableColumns = [
'id', 'xcentroid', 'ycentroid', 'area', 'max_value', 'source_sum',
'mag_instr'
]
# there are other properties available, see list of SourceProperties
# http://photutils.readthedocs.io/en/latest/api/photutils.segmentation.SourceProperties.html#photutils.segmentation.SourceProperties
propsTable = properties_table(props, columns=propsTableColumns)
print(propsTable)
#%% plots for visualisation
apertures = []
for prop in props:
position = (prop.xcentroid.value, prop.ycentroid.value)
a = prop.semimajor_axis_sigma.value * 3.0
b = prop.semiminor_axis_sigma.value * 3.0
theta = prop.orientation.value
apertures.append(EllipticalAperture(position, a, b, theta=theta))
norm = ImageNormalize(stretch=SqrtStretch())
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(18, 18))
if 'bkg' in locals():
def aperture_photometry(self, path_file_abs):
"""
aperture photometry from source segmentation
make_source_mask not yet available in photutils v0.2.1
wait for v0.3 release
aperture_photometry() assumes that the data have been
background-subtracted.
"""
# create preliminary mask
#from photutils import make_source_mask
#masterMask = make_source_mask(master, snr=2, npixels=5, dilate_size=11)
# if LEDoff was used, get threshold from LEDoff/background
# path_dataset = os.path.dirname(path_file_abs) + os.path.sep
# filenameCombined = '\t'.join(
# os.listdir(os.path.join(datasetDirLocal, 'master')))
# if 'master_ledoff_subtracted' in filename:
# self.msg('Using master_ledoff')
# # path_file_abs = os.path.join(datasetDir, 'master', filename)
# hdu = fits.open(path_file_abs)[0]
# data_subtracted = hdu.data
# # calculate threadhold
# ledoff_pred = np.mean(data_subtracted) * \
# np.ones(data_subtracted.shape)
# mse = mean_squared_error(data_subtracted, ledoff_pred)
# rmse = np.sqrt(mse)
# threshold = 7.0 * rmse
# threshold_value = threshold
# if no LEDoff was used, background subtraction is needed
# there should exist no file named "subtracted"
# if 'master.fit' in filenameCombined \
# or 'master_normalised.fit' in filenameCombined:
if 'master.fit' in path_file_abs:
self.msg('Photometry using un-normalised master image')
elif 'master_normalised.fit' in path_file_abs:
self.msg('Photometry using normalised master image')
hdu = fits.open(path_file_abs)[0]
data = hdu.data
if 'EXPTIME' in hdu.header:
exptime = hdu.header['EXPTIME']
else:
exptime = hdu.header['EXPREQ']
# === background subtraction ===
"""
if no LEDoff was used, background subtraction is needed.
there should exist no file named "subtracted".
create 2D image of background and background rms and
apply sigma-clipping to each region in the low-res
background map to get mean, median, and std/rms.
sigma-clipping is the most widely used method though not as
good as using mask; still superior to robust standard
deviation using median absolute deviation (MAD-STD).
"""
# create background
# [mean, median, std] = sigma_clipped_stats(master, sigma=3.0, iters=5)
# bkg = Background(master, (50, 50), filter_size=(3, 3), method='median')
bkg = Background(data, (100, 100),
filter_shape=(3, 3),
method='median')
# plot background image
# plt.imshow(bkg.background, norm=normalisation, origin='lower', cmap=plt.cm.gray)
plt.imshow(bkg.background, origin='lower', cmap=plt.cm.gray)
[fig, ax] = plt.subplots(figsize=(8, 8))
# make background-substracted image
data_subtracted = data - bkg.background
# plot subtracted image
plt.imshow(data_subtracted, origin='lower', cmap=plt.cm.gray)
# === segmentation at a given sigma level ===
# perform segmentation whether flat is available or not
self.msg('Determining threshold for target detection...')
# because data is background-subtracted
threshold_array = 5.0 * bkg.background_rms
# print out threshold value
threshold_value = threshold_array.flat[0]
self.msg('Threshold for target detection is: ' + repr(threshold_value))
self.msg('Detecting sources and performing segmentation...')
segm = detect_sources(data_subtracted, threshold_array, npixels=5)
self.msg('Segmentation labels are:')
self.msg((repr(segm.labels)))
# === regional properties ===
# measure regional source properties from segmentation
# the centroid is from image moments, already intensity-weighted
self.msg('Measuring source properties...')
if 'bkg' in locals():
# use the background determined from master_subtracted
props = source_properties(data_subtracted,
segm,
error=bkg.background_rms,
background=bkg.background)
# elif 'master_ledoff_subtracted' in filenameCombined:
# path_file_abs = os.path.join(
# datasetDirLocal, 'master', 'master_ledoff_subtracted.fits')
# hdu = fits.open(path_file_abs)[0]
# master_ledoff_subtracted = hdu.data
# props = source_properties(data_subtracted, segm,
# error = master_ledoff_subtracted \
# - np.mean(master_ledoff_subtracted),
# background = master_ledoff_subtracted)
# add instrumental magnitude to properties
# instrumental magnitude = -2.5 * log10(flux)
for i in range(len(props)):
# source_sum is by definition background-subtracted already
props[i].mag_instr = -2.5 * np.log10(props[i].source_sum / exptime)
# create table from props object
# there are other properties available, see list of SourceProperties:
# http://goo.gl/rkfQ9V
props_table_columns = [
'id', 'xcentroid', 'ycentroid', 'area', 'max_value', 'source_sum',
'mag_instr'
]
props_table_display = properties_table(props,
columns=props_table_columns)
props_table_save = properties_table(props)
# self.msg(repr(props_table_display))
print(repr(props_table_display))
# check and create analysis folder if it doesn't exist
path_dataset = os.path.dirname(path_file_abs)
path_analysis = path_dataset.replace('/data/images/fpc/',
'/data/images/fpc_analysis/')
if not os.path.exists(path_analysis):
os.makedirs(path_analysis)
# save background subtracted image
if 'master.fit' in path_file_abs:
path_save = os.path.join(path_dataset, 'master_subtracted.fits')
elif 'master_object.fit' in path_file_abs:
path_save = os.path.join(path_dataset,
'master_object_subtracted.fits')
elif 'master_normalised.fit' in path_file_abs:
path_save = os.path.join(path_dataset,
'master_normalised_subtracted.fits')
hdu_subtracted = fits.PrimaryHDU(data_subtracted)
hdu_subtracted.writeto(path_save, clobber=True)
# save properties to table file
path_save = os.path.join(path_dataset, 'props_table.csv')
ascii.write(props_table_save, path_save, format='csv')
# === update UI ===
# plot segmentated image
self.rmmpl()
figure_photometry = Figure()
cmap_rand = random_cmap(segm.max + 1, random_state=12345)
axes = figure_photometry.add_subplot(111)
axes.imshow(segm, origin='lower', cmap=cmap_rand)
axes.plot(props_table_save['xcentroid'],
props_table_save['ycentroid'],
ls='none',
color='red',
marker='+',
ms=10,
lw=1.5)
self.addmpl(figure_photometry)
# set properties table font and font size
self.ui.tablePhot.setCurrentFont(QtGui.QFont(TEXT_BROWSER_FONT))
self.ui.tablePhot.setFontPointSize(TEXT_BROWSER_FONT_SIZE)
self.ui.tablePhot.setPlainText(repr(props_table_display))
self.msg('Photometry completed for {}.'.format(path_file_abs))
# get labels of hot pixels from master bias and master dark
# pixel label, x coord, y coord, master bias, sd, master dark, sd, master object, sd,
os.chdir(r'C:\Users\givoltage\Google Drive\DESI\protoDESI\ssl\20160524')
filepath_bias = r'C:\Users\givoltage\Downloads\20160718_-15c\1.0.fit'
filepath_dark = r'C:\Users\givoltage\Google Drive\DESI\protoDESI\ssl\20160524\1sec_dark_009.FIT'
m_bias = fits.open(filepath_bias)[0].data
m_dark = fits.open(filepath_dark)[0].data
segm_bias = detect_sources(m_bias, 1300, npixels=1)
segm_dark = detect_sources(m_dark, 1300, npixels=1)
props_bias = source_properties(m_bias, segm_bias)
props_dark = source_properties(m_dark, segm_dark)
# propsTableColumns = ['id', 'xcentroid', 'ycentroid', 'max_value', 'min_value','source_sum']
table_bias = properties_table(props_bias)
table_dark = properties_table(props_dark)
ascii.write(table_bias, '1.0.csv', format = 'csv')
ascii.write(table_dark, 'protodesi-FPC_00000009(1).csv', format = 'csv')
#log_bias = open('labels_bias.log', 'w')
#log_dark = open('labels_dark.log', 'w')
#
#log_bias.write(log_bias_text)
#log_dark.write(log_dark_text)
#
#log_bias.close()
#log_dark.close()