def cutouts(image, stars, size=15):
"""Custom version to extract stars cutouts
Parameters
----------
Parameters
----------
image: np.ndarray or path
stars: np.ndarray
stars positions with shape (n,2)
size: int
size of the cuts around stars (in pixels), by default 15
Returns
-------
np.ndarray of shape (size, size)
"""
if isinstance(image, str):
image = fits.getdata(image)
stars_in = np.logical_and(
np.all(stars < np.array(image.shape) - size - 2, axis=1),
np.all(stars > np.ones(2) * size + 2, axis=1))
stars = stars[stars_in]
# with warnings.catch_warnings():
warnings.simplefilter("ignore")
stars_tbl = Table([stars[:, 0], stars[:, 1]], names=["x", "y"])
stars = extract_stars(NDData(data=image), stars_tbl, size=size)
return np.argwhere(stars_in).flatten(), stars
def make_stars_guess(
image: np.ndarray,
star_finder: photutils.StarFinderBase,
cutout_size: int = config.cutout_size) -> photutils.psf.EPSFStars:
"""
Given an image, extract stars as EPSFStars for psf fitting
:param image: yes
:param cutout_size: how big should the regions around each star used for fitting be?
:param star_finder: which starfinder to use?
:return: instance of exctracted EPSFStars
"""
# The idea here is to run a "greedy" starfinder that finds a lot more candidates than we need and then
# to filter out the bright and isolated stars
peaks_tbl = star_finder(image)
peaks_tbl.rename_columns(['xcentroid', 'ycentroid'], ['x', 'y'])
peaks_tbl = cut_edges(peaks_tbl, cutout_size, image.shape[0])
# TODO this gets medianed away with the image combine approach, so more star more good?
# stars_tbl = cut_close_stars(peaks_tbl, cutoff_dist=3)
stars_tbl = peaks_tbl
image_no_background = image - np.median(image)
stars = extract_stars(NDData(image_no_background),
stars_tbl,
size=cutout_size)
return stars
def extract(self):
epsf_builder = EPSFBuilder(oversampling=4,
maxiters=3,
progress_bar=True)
for file in self.params.inFiles:
logging.info("Extracting PSFs from file {}".format(file))
psf_file = PSFfile(file,
self.params.tmpDir,
frame_shape=(self.box_size, self.box_size))
frame_number = fits.getheader(file)['NAXIS3']
for frame_index in range(frame_number):
print("\rExtracting PSF from frame {}/{}".format(
frame_index + 1, frame_number),
end='')
with fits.open(file) as hdulist:
frame = hdulist[0].data[frame_index]
stars = extract_stars(NDData(data=frame),
self.star_table,
size=self.box_size)
# Compute instantaneous PSF
# epsf, fitted_stars = epsf_builder(stars)
epsf = np.zeros(stars[0].data.shape)
for star in stars:
epsf += star.data
psf_file.update_frame(frame_index, epsf)
print('\r')
def _extractStars(self):
self._findStars()
self._starsTab = Table()
self._starsTab['x'] = self._selectedStars['xcentroid']
self._starsTab['y'] = self._selectedStars['ycentroid']
self._starsCut = extract_stars(NDData(data=self._image),
self._starsTab,
self._size)
def extract_stars(self):
# just in case sky was not properly subtracted
mean_val, median_val, std_val = sigma_clipped_stats(self.data,
sigma=2.)
self.data -= median_val
nddata = NDData(data=self.data)
self.stars = extract_stars(nddata, self.stars_tbl, size=self.size)
# check to make sure stars don't have zeros
# Virgo 2017 pointing-4_R.coadd.fits is giving me trouble b/c a lot of stars have zeros
keepflag = np.ones(len(self.stars), 'bool')
for i, s in enumerate(self.stars):
if len(np.where(s.data == 0)[0]) > 1:
keepflag[i] = False
self.keepflag2 = keepflag
self.stars = extract_stars(nddata,
self.stars_tbl[keepflag],
size=self.size)
#self.stars = self.stars[keepflag]
if len(self.stars) < self.nstars:
self.nstars = len(self.stars)
def cutouts(image, stars, size=15):
"""Custom version to extract stars cutouts
Parameters
----------
Parameters
----------
image: np.ndarray or path
stars: np.ndarray
stars positions with shape (n,2)
size: int
size of the cuts around stars (in pixels), by default 15
Returns
-------
np.ndarray of shape (size, size)
"""
if isinstance(image, str):
image = fits.getdata(image)
warnings.simplefilter("ignore")
if np.shape(stars) > (1,2):
stars_tbl = Table(
[stars[:, 0], stars[:, 1], np.arange(len(stars))],
names=["x", "y", "id"])
_stars = [None]*len(stars)
stars = extract_stars(NDData(data=image), stars_tbl, size=size)
idxs = np.array([s.id_label for s in stars])
for i, s in enumerate(stars):
_stars[idxs[i]] = s
return idxs, _stars
else:
stars_tbl = Table(
data=np.array([stars[0][0], stars[0][1]]),
names=["x", "y"])
stars = ([0], [extract_stars(NDData(data=image), stars_tbl, size=size)])
return stars
def construct_epsf(imdata, mean):
peaks_tbl = find_peaks(imdata, threshold=20.0)
peaks_tbl['peak_value'].info.format = '%.8g'
for i in range(len(peaks_tbl)):
print(i, peaks_tbl['x_peak'][i], peaks_tbl['y_peak'][i])
#rem_indA = [0,1,3,4,5,6,8,9,11,12,13,16,84,85,86,87,91,92,99,102]
#rem_indB = [0,1,3,4,5,6,8,9,11,12,13,15,16,17,24,25,61,62,73,78,84,85,86,87,91,92,99,102]
#rem_indC = [0,1,2,3,4,5,6,8,9,10,11,12,13,15,16,17,22,23,24,25,61,62,71,73,77,78,84,85,86,87,91,92,94,96,99,102]
#rem_indVIS = [2,3,4,5,6,7,8,9,10,12,13,14]
#rem_indD = [3,5]
#rem_indVIS2 = [0,1,2,7]
#rem_indE = [22,23,24,25,27,28,29]
#rem_indF = [11,12,13,15,16,18,19,20,21,22,23,24,25,26]
#rem_indG = [9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28]
#rem_indH=[4,5,6,7,8,9,10,11,12,13]
#rem_indI = [4,5,6,7,8,9,10,11,12,13,14,15,16,17,18]
#rem_indJ = [0,1,2,3,4,5,7,8,9,10,11,12,13,14,15,16,22,25,27,28,29,30,33,34,36,37,38,39]
#rem_indK = [0,1,2,3,4,5,6,7,8,9,10,11,13,14,15,23,24,25,26,29,30,31,33,34,35]
#rem_indL = [4,5,6,7,8,10,11,12,13]
#rem_indM = [5,6,7,8,9,10,11,13,14,15,16,18,27,29]
#rem_indN = [4,5,6,7,8,9,10,11,13,14,15,16,17,19,20,21]
#peaks_tbl.remove_rows([rem_indM])
plt.figure()
plt.scatter(peaks_tbl['x_peak'], peaks_tbl['y_peak'], c='k')
plt.imshow(imdata, vmin=-0.2, vmax=2., origin='lowerleft')
plt.show()
stars_tbl = Table()
stars_tbl['x'] = peaks_tbl['x_peak']
stars_tbl['y'] = peaks_tbl['y_peak']
mean_val, median_val, std_val = sigma_clipped_stats(imdata, sigma=2.0)
imdata -= median_val
nddata = NDData(data=imdata)
stars = extract_stars(nddata, stars_tbl, size=20)
epsf_builder = EPSFBuilder(oversampling=4, maxiters=3, progress_bar=False)
epsf, fitted_stars = epsf_builder(stars)
hdu = fits.PrimaryHDU(epsf.data)
hdul = fits.HDUList([hdu])
hdul.writeto('Serpens3/epsf.fits')
return stars, epsf
def make_psf(data, catalog, show=False, boxsize=25.):
catalog = catalog.copy()
catalog['x'], catalog['y'] = data.wcs.all_world2pix(catalog['raMean'], catalog['decMean'], 0)
bkg = np.nanmedian(data)
nddata = NDData(data - bkg)
stars = psf.extract_stars(nddata, catalog, size=boxsize)
epsf_builder = EPSFBuilder(oversampling=1.)
epsf, fitted_stars = epsf_builder(stars)
if show:
plt.figure()
plt.imshow(epsf.data)
plot_stars(fitted_stars)
return epsf, fitted_stars
def make_psf(filepath, filename, filt_name, show, datamax, nstars, ncores,
catalog_filepath):
# TODO: add stage checking generally to stop errors
t_start = time.time()
if not nstars:
nstars = 12
size = 25
print(f'Working on {filename}')
full_filepath = filepath + filename
data = getdata(full_filepath, 0)
banzai = getdata(full_filepath, 1)
metadata = load_pickle(filename)
# TODO: be smarter about saving useful stars, add field option
banzai_coords = _filter_banzai(banzai, datamax)
stars_without_cosmics = _filter_cosmics(metadata, size, banzai_coords)
all_stars = _filter_to_catalog(stars_without_cosmics, catalog_filepath,
filt_name, filepath, filename)
psf_stars = all_stars[:nstars]
extracted_stars = extract_stars(NDData(data=data), psf_stars, size=size)
print('\tBuilding psf . . .')
epsf_builder = EPSFBuilder(oversampling=2, maxiters=3, progress_bar=False)
epsf, fitted_stars = epsf_builder(extracted_stars)
# can only pickle python built-ins?
create_or_update_pickle(filename=filename,
key='epsf',
val=epsf.data.tolist())
create_or_update_pickle(filename=filename,
key='psf_fitted_stars',
val=np.array(all_stars).tolist())
#create_or_update_pickle(filename=filename, key='psf_fitted_stars', val=fitted_stars)
t_end = time.time()
print(f'Time to generate psf (s): {t_end-t_start:.2f}')
if show and ncores == 1:
check_psf(data, filename, epsf.data, all_stars, psf_stars, size)
elif show and ncores != 1:
print("Can't display psfs while using multiple cores")
print()
def generate_epsf(img_file, x, y, size=51, oversampling=2, maxiters=5):
# Construct stars table from bright
stars_tbl = Table()
stars_tbl['x'] = x
stars_tbl['y'] = y
print(stars_tbl)
img_hdu = fits.open(img_file)
ndimage = NDData(data=img_hdu[0].data)
stars = extract_stars(ndimage, stars_tbl, size=size)
print('Extracted {0} stars. Building EPSF...'.format(len(stars)))
epsf_builder = EPSFBuilder(oversampling=oversampling,
maxiters=maxiters, progress_bar=True)
epsf, fitted_stars = epsf_builder(stars)
return(epsf)
def calculate_psf_template(image, xx, yy, size, oversampling_rate,
maxiters=11, mask=None, plot_candidates=False) :
hsize = (size+2)/2
# remove stars close to the sample edges
pos_mask = ((xx > hsize) & (xx < (image.shape[1] - 1 - hsize)) &
(yy > hsize) & (yy < (image.shape[0] - 1 - hsize)))
# Table of good star positions
stars_tbl = Table()
stars_tbl['x'] = xx[pos_mask]
stars_tbl['y'] = yy[pos_mask]
# Subtract background
# print("image.shape ", image.shape)
mean_val, median_val, std_val = sigma_clipped_stats(image, mask=mask, sigma=2.)
image -= median_val
nddata = NDData(data=image)
stars = extract_stars(nddata, stars_tbl, size=size)
print("number of stars ", len(stars))
#
# Plot candidates (use with care!)
#
if(plot_candidates == True) :
nrows = 1+int(np.sqrt(float(len(stars))))
ncols = 1+ int(float(len(stars))/float(nrows))
# print("nrows, ncols, len(stars) ",nrows, ncols, len(stars))
# nrows = 7
# ncols = 8
fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20),squeeze=True)
ax = ax.ravel()
for i in range(nrows*ncols):
norm = simple_norm(stars[i], 'log', percent=99.)
ax[i].imshow(stars[i], norm=norm, origin='lower', cmap='viridis')
plt.show()
#
# build average PSF
#
epsf_builder = EPSFBuilder(oversampling=oversampling_rate, maxiters=3,progress_bar=False)
epsf, fitted_stars = epsf_builder(stars)
return epsf, fitted_stars
def make_psf(data, catalog, show=False):
catalog = catalog.copy()
catalog['x'], catalog['y'] = data.wcs.all_world2pix(catalog['raMean'], catalog['decMean'], 0)
bkg = np.nanmedian(data)
nddata = NDData(data - bkg)
stars = psf.extract_stars(nddata, catalog, size=25.)
epsf_builder = EPSFBuilder(oversampling=1.)
epsf, fitted_stars = epsf_builder(stars)
if show:
plt.figure()
plt.imshow(epsf.data)
nrows = int(np.ceil(len(stars)**0.5))
fig, axarr = plt.subplots(nrows, nrows, figsize=(20, 20), squeeze=True)
for ax, star in zip(axarr.ravel(), fitted_stars):
ax.imshow(star)
ax.plot(star.cutout_center[0], star.cutout_center[1], 'r+')
return epsf, fitted_stars
def run_photometry(img_file, epsf, fwhm, x, y, subtract_back=False,
forced=False):
img_hdu = fits.open(img_file)
if subtract_back:
bkg = Background2D(img_hdu[0].data, (21,21), filter_size=(3,3))
image = img_hdu[0].data - bkg.background
ndimage = NDData(data=backsub)
else:
image = img_hdu[0].data
ndimage = NDData(data=img_hdu[0].data)
psf = copy.copy(epsf)
stars_tbl = Table()
stars_tbl['x'] = x
stars_tbl['y'] = y
stars = extract_stars(ndimage, stars_tbl, size=51)
stars_tbl['flux'] = np.array([stars[0].estimate_flux()])
targets = Table()
targets['x_0'] = stars_tbl['x']
targets['y_0'] = stars_tbl['y']
targets['flux_0'] = stars_tbl['flux']
if forced:
psf.x_0.fixed = True
psf.y_0.fixed = True
daogroup = DAOGroup(fwhm)
photometry = BasicPSFPhotometry(group_maker=daogroup,
bkg_estimator=mmm_bkg,
psf_model=psf,
fitter=fitter,
fitshape=(51,51))
result_tab = photometry(image=image, init_guesses=targets)
return(result_tab)
def build_epsf(image_num):
size = 7
hsize = (size - 1) / 2
peaks_tbl = find_peaks(Reduced_Image_Data[image_num], threshold=750.)
x = peaks_tbl['x_peak']
y = peaks_tbl['y_peak']
mask = ((x > hsize) &
(x < (Reduced_Image_Data[image_num].shape[1] - 1 - hsize)) &
(y > hsize) &
(y < (Reduced_Image_Data[image_num].shape[0] - 1 - hsize)))
stars_tbl = Table()
stars_tbl['x'] = x[mask]
stars_tbl['y'] = y[mask]
nddata = NDData(data=Reduced_Image_Data[image_num])
stars = extract_stars(nddata, stars_tbl, size=10)
#nrows = 5
#ncols = 5
#fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize = (15, 15), squeeze = True)
#ax = ax.ravel()
#for i in range(nrows*ncols):
#norm = simple_norm(stars[i], 'log', percent = 99.)
#ax[i].imshow(stars[i], norm = norm, origin = 'lower', cmap = 'gray')
epsf_builder = EPSFBuilder(oversampling=8, maxiters=20, progress_bar=False)
epsf, fitted_stars = epsf_builder(stars)
#norm = simple_norm(epsf.data, 'log', percent = 99.)
#plt.imshow(epsf.data, norm = norm, origin = 'lower', cmap = 'viridis')
#plt.colorbar()
return epsf
def cutouts(image, stars, size):
stars_tbl = Table(stars.T, names=["x", "y"])
stars = extract_stars(NDData(data=image), stars_tbl, size=size)
return stars
def removeStarsPSF(img, starPixX, starPixY, starR, mags, gX, gY, gR, p1, p2, radius, showProgress, inverted = False):
remove = []
for i in range(0,len(starPixX)):
if i%100 == 0:
print(i)
for j in range(i+1,len(starPixX)):
if np.abs(starPixX[i]-starPixX[j])<2.5*radius and np.abs(starPixY[i]-starPixY[j])<2.5*radius:
remove.append(i)
remove.append(j)
mask = np.ones(len(starPixX), dtype = bool)
mask[np.array(remove)] = 0
starPixX = starPixX[mask]
starPixY = starPixY[mask]
starR = starR[mask]
mags = mags[mask]
print(radius)
mask = np.ones(starPixX.shape[0],dtype=bool)
for i in range(gX.shape[0]):
mask = np.logical_and(mask, np.sqrt((starPixX-gX[i])**2+(starPixY-gY[i])**2)>gR[i]+2*radius)
starPixX = starPixX[mask]
starPixY = starPixY[mask]
starR = starR[mask]
mags = mags[mask]
mask = (2*radius<starPixX) & (starPixX<img.shape[1]-2*radius) & (2*radius<starPixY) & (starPixY<img.shape[0]-2*radius)
starPixX = starPixX[mask]
starPixY = starPixY[mask]
starR = starR[mask]
mags = mags[mask]
if p1[0]==p2[0]:
psfMask = np.abs(starPixX-p1[0])>2.5*radius
trailMask = np.abs(starPixX-p1[0])<10
psfX = starPixX[psfMask]
psfY = starPixY[psfMask]
psfR = starR[psfMask]
psfMags = mags[psfMask]
trailX = starPixX[trailMask]
trailY = starPixY[trailMask]
trailMags = mags[trailMask]
else:
m,b = TF.getLineParams(p1,p2)
par,perp = GT.findDistances(starPixX,starPixY,m,b)
psfMask = np.abs(perp)>2.5*radius
trailMask = np.abs(perp)<10
psfX = starPixX[psfMask]
psfY = starPixY[psfMask]
psfR = starR[psfMask]
psfMags = mags[psfMask]
trailX = starPixX[trailMask]
trailY = starPixY[trailMask]
trailMags = mags[trailMask]
mask = psfR>10
psfX = psfX[mask]
psfY = psfY[mask]
psfMags = psfMags[mask]
print(psfX.shape[0])
showImageWithDetectedHotPixels(img, "Image with Star Picks", psfX, psfY, "b", inverted)
psfX,psfY = centroid_sources(img, psfX, psfY, box_size=21, centroid_func=centroid_com)
showImageWithDetectedHotPixels(img, "Image with Star Picks Centroid", psfX, psfY, "b", inverted)
stars = Table()
stars["x"] = psfX
stars["y"] = psfY
#stars2 = Table()
#stars2["x_0"] = psfX
#stars2["y_0"] = psfY
stars = extract_stars(NDData(data=img), stars, size = 2*int(2*radius)+1)
nrows = 5
ncols = 5
fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True)
ax = ax.ravel()
for i in range(nrows*ncols):
norm = simple_norm(stars[i], 'log', percent=99.)
ax[i].imshow(stars[i], norm=norm, origin='lower', cmap='viridis')
plt.show()
#sigma_psf = 2.0
#daogroup = DAOGroup(2.0)
#mmm_bkg = MMMBackground()
#fitter = LevMarLSQFitter()
#psf_model = PRF(sigma=sigma_psf)
#photometry = BasicPSFPhotometry(group_maker=daogroup,bkg_estimator=mmm_bkg,psf_model=psf_model,fitter=LevMarLSQFitter(),fitshape=(11,11))
#result_tab = photometry(image=img, init_guesses=stars2)
#residual_image = photometry.get_residual_image()
epsfBuilder = EPSFBuilder(oversampling=2, maxiters=20, smoothing_kernel = "quadratic")
epsf, starFits = epsfBuilder(stars)
#showImage(residual_image,"Image No Stars",True)
norm = simple_norm(epsf.data, 'log', percent=99.)
plt.imshow(epsf.data, norm=norm, origin='lower', cmap='viridis')
plt.colorbar()
plt.show()
nrows = 5
ncols = 5
fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True)
ax = ax.ravel()
for i in range(nrows*ncols):
norm = simple_norm(starFits[i], 'log', percent=99.)
ax[i].imshow(starFits[i], norm=norm, origin='lower', cmap='viridis')
plt.show()
nrows = 5
ncols = 5
fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20), squeeze=True)
ax = ax.ravel()
for i in range(nrows*ncols):
norm = simple_norm(stars[i].data-starFits[i].data, 'log', percent=99.)
ax[i].imshow(stars[i].data-starFits[i].data, norm=norm, origin='lower', cmap='viridis')
plt.show()
showImage(finalImg, "Image With No Stars", inverted)
def photometry(fileid):
"""
Run photometry.
.. codeauthor:: Rasmus Handberg <*****@*****.**>
"""
# Settings:
ref_mag_limit = 17 # Lower limit on reference target brightness
ref_target_dist_limit = 30 # Reference star must be further than this away to be included
logger = logging.getLogger(__name__)
tic = default_timer()
# Use local copy of archive if configured to do so:
config = load_config()
# Get datafile dict from API:
datafile = api.get_datafile(fileid)
logger.debug("Datafile: %s", datafile)
targetid = datafile['targetid']
photfilter = datafile['photfilter']
archive_local = config.get('photometry', 'archive_local', fallback=None)
if archive_local is not None:
datafile['archive_path'] = archive_local
if not os.path.isdir(datafile['archive_path']):
raise FileNotFoundError("ARCHIVE is not available")
# Get the catalog containing the target and reference stars:
# TODO: Include proper-motion to the time of observation
catalog = api.get_catalog(targetid, output='table')
target = catalog['target'][0]
# Extract information about target:
target_name = str(target['target_name'])
target_coord = coords.SkyCoord(ra=target['ra'],
dec=target['decl'],
unit='deg',
frame='icrs')
# Folder to save output:
# TODO: Change this!
output_folder_root = config.get('photometry', 'output', fallback='.')
output_folder = os.path.join(output_folder_root, target_name,
'%04d' % fileid)
os.makedirs(output_folder, exist_ok=True)
# Also write any logging output to the
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
_filehandler = logging.FileHandler(os.path.join(output_folder,
'photometry.log'),
mode='w')
_filehandler.setFormatter(formatter)
_filehandler.setLevel(logging.INFO)
logger.addHandler(_filehandler)
# The paths to the science image:
filepath = os.path.join(datafile['archive_path'], datafile['path'])
# TODO: Download datafile using API to local drive:
# TODO: Is this a security concern?
#if archive_local:
# api.download_datafile(datafile, archive_local)
# Translate photometric filter into table column:
if photfilter == 'gp':
ref_filter = 'g_mag'
elif photfilter == 'rp':
ref_filter = 'r_mag'
elif photfilter == 'ip':
ref_filter = 'i_mag'
elif photfilter == 'zp':
ref_filter = 'z_mag'
elif photfilter == 'B':
ref_filter = 'B_mag'
elif photfilter == 'V':
ref_filter = 'V_mag'
else:
logger.warning(
"Could not find filter '%s' in catalogs. Using default gp filter.",
photfilter)
ref_filter = 'g_mag'
references = catalog['references']
references.sort(ref_filter)
# Load the image from the FITS file:
image = load_image(filepath)
# Calculate pixel-coordinates of references:
row_col_coords = image.wcs.all_world2pix(
np.array([[ref['ra'], ref['decl']] for ref in references]), 0)
references['pixel_column'] = row_col_coords[:, 0]
references['pixel_row'] = row_col_coords[:, 1]
# Calculate the targets position in the image:
target_pixel_pos = image.wcs.all_world2pix(
[[target['ra'], target['decl']]], 0)[0]
# Clean out the references:
hsize = 10
x = references['pixel_column']
y = references['pixel_row']
references = references[
(np.sqrt((x - target_pixel_pos[0])**2 +
(y - target_pixel_pos[1])**2) > ref_target_dist_limit)
& (references[ref_filter] < ref_mag_limit)
& (x > hsize) & (x < (image.shape[1] - 1 - hsize))
& (y > hsize) & (y < (image.shape[0] - 1 - hsize))]
#==============================================================================================
# BARYCENTRIC CORRECTION OF TIME
#==============================================================================================
ltt_bary = image.obstime.light_travel_time(target_coord, ephemeris='jpl')
image.obstime = image.obstime.tdb + ltt_bary
#==============================================================================================
# BACKGROUND ESITMATION
#==============================================================================================
fig, ax = plt.subplots(1, 2, figsize=(20, 18))
plot_image(image.clean, ax=ax[0], scale='log', cbar='right', title='Image')
plot_image(image.mask,
ax=ax[1],
scale='linear',
cbar='right',
title='Mask')
fig.savefig(os.path.join(output_folder, 'original.png'),
bbox_inches='tight')
plt.close(fig)
# Estimate image background:
# Not using image.clean here, since we are redefining the mask anyway
bkg = Background2D(
image.clean,
(128, 128),
filter_size=(5, 5),
#mask=image.mask | (image.clean > background_cutoff),
sigma_clip=SigmaClip(sigma=3.0),
bkg_estimator=SExtractorBackground(),
exclude_percentile=50.0)
image.background = bkg.background
# Create background-subtracted image:
image.subclean = image.clean - image.background
# Plot background estimation:
fig, ax = plt.subplots(1, 3, figsize=(20, 6))
plot_image(image.clean, ax=ax[0], scale='log', title='Original')
plot_image(image.background, ax=ax[1], scale='log', title='Background')
plot_image(image.subclean,
ax=ax[2],
scale='log',
title='Background subtracted')
fig.savefig(os.path.join(output_folder, 'background.png'),
bbox_inches='tight')
plt.close(fig)
# TODO: Is this correct?!
image.error = calc_total_error(image.clean, bkg.background_rms, 1.0)
#==============================================================================================
# DETECTION OF STARS AND MATCHING WITH CATALOG
#==============================================================================================
logger.info("References:\n%s", references)
radius = 10
fwhm_guess = 6.0
fwhm_min = 3.5
fwhm_max = 13.5
# Extract stars sub-images:
#stars = extract_stars(
# NDData(data=image.subclean, mask=image.mask),
# stars_for_epsf,
# size=size
#)
# Set up 2D Gaussian model for fitting to reference stars:
g2d = models.Gaussian2D(amplitude=1.0,
x_mean=radius,
y_mean=radius,
x_stddev=fwhm_guess * gaussian_fwhm_to_sigma)
g2d.amplitude.bounds = (0.1, 2.0)
g2d.x_mean.bounds = (0.5 * radius, 1.5 * radius)
g2d.y_mean.bounds = (0.5 * radius, 1.5 * radius)
g2d.x_stddev.bounds = (fwhm_min * gaussian_fwhm_to_sigma,
fwhm_max * gaussian_fwhm_to_sigma)
g2d.y_stddev.tied = lambda model: model.x_stddev
g2d.theta.fixed = True
gfitter = fitting.LevMarLSQFitter()
fwhms = np.full(len(references), np.NaN)
for i, (x, y) in enumerate(
zip(references['pixel_column'], references['pixel_row'])):
x = int(np.round(x))
y = int(np.round(y))
x0, y0, width, height = x - radius, y - radius, 2 * radius, 2 * radius
cutout = slice(y0 - 1, y0 + height), slice(x0 - 1, x0 + width)
curr_star = image.subclean[cutout] / np.max(image.subclean[cutout])
npix = len(curr_star)
ypos, xpos = np.mgrid[:npix, :npix]
gfit = gfitter(g2d, x=xpos, y=ypos, z=curr_star)
fwhms[i] = gfit.x_fwhm
mask = ~np.isfinite(fwhms) | (fwhms <= fwhm_min) | (fwhms >= fwhm_max)
masked_fwhms = np.ma.masked_array(fwhms, mask)
fwhm = np.mean(sigma_clip(masked_fwhms, maxiters=20, sigma=2.0))
logger.info("FWHM: %f", fwhm)
# Use DAOStarFinder to search the image for stars, and only use reference-stars where a
# star was actually detected close to the references-star coordinate:
cleanout_references = (len(references) > 50)
if cleanout_references:
daofind_tbl = DAOStarFinder(100, fwhm=fwhm, roundlo=-0.5,
roundhi=0.5).find_stars(image.subclean,
mask=image.mask)
indx_good = np.zeros(len(references), dtype='bool')
for k, ref in enumerate(references):
dist = np.sqrt((daofind_tbl['xcentroid'] -
ref['pixel_column'])**2 +
(daofind_tbl['ycentroid'] - ref['pixel_row'])**2)
if np.any(dist <= fwhm / 4): # Cutoff set somewhat arbitrary
indx_good[k] = True
references = references[indx_good]
fig, ax = plt.subplots(1, 1, figsize=(20, 18))
plot_image(image.subclean,
ax=ax,
scale='log',
cbar='right',
title=target_name)
ax.scatter(references['pixel_column'],
references['pixel_row'],
c='r',
alpha=0.3)
if cleanout_references:
ax.scatter(daofind_tbl['xcentroid'],
daofind_tbl['ycentroid'],
c='g',
alpha=0.3)
ax.scatter(target_pixel_pos[0], target_pixel_pos[1], marker='+', c='r')
fig.savefig(os.path.join(output_folder, 'positions.png'),
bbox_inches='tight')
plt.close(fig)
#==============================================================================================
# CREATE EFFECTIVE PSF MODEL
#==============================================================================================
# Make cutouts of stars using extract_stars:
# Scales with FWHM
size = int(np.round(29 * fwhm / 6))
if size % 2 == 0:
size += 1 # Make sure it's a uneven number
size = max(size, 15) # Never go below 15 pixels
hsize = (size - 1) / 2
x = references['pixel_column']
y = references['pixel_row']
mask_near_edge = ((x > hsize) & (x < (image.shape[1] - 1 - hsize))
& (y > hsize) & (y < (image.shape[0] - 1 - hsize)))
stars_for_epsf = Table()
stars_for_epsf['x'] = x[mask_near_edge]
stars_for_epsf['y'] = y[mask_near_edge]
# Store which stars were used in ePSF in the table:
logger.info("Number of stars used for ePSF: %d", len(stars_for_epsf))
references['used_for_epsf'] = mask_near_edge
# Extract stars sub-images:
stars = extract_stars(NDData(data=image.subclean, mask=image.mask),
stars_for_epsf,
size=size)
# Plot the stars being used for ePSF:
nrows = 5
ncols = 5
imgnr = 0
for k in range(int(np.ceil(len(stars_for_epsf) / (nrows * ncols)))):
fig, ax = plt.subplots(nrows=nrows,
ncols=ncols,
figsize=(20, 20),
squeeze=True)
ax = ax.ravel()
for i in range(nrows * ncols):
if imgnr > len(stars_for_epsf) - 1:
ax[i].axis('off')
else:
plot_image(stars[imgnr], ax=ax[i], scale='log', cmap='viridis')
imgnr += 1
fig.savefig(os.path.join(output_folder,
'epsf_stars%02d.png' % (k + 1)),
bbox_inches='tight')
plt.close(fig)
# Build the ePSF:
epsf = EPSFBuilder(oversampling=1.0,
maxiters=500,
fitter=EPSFFitter(fit_boxsize=2 * fwhm),
progress_bar=True)(stars)[0]
logger.info('Successfully built PSF model')
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(15, 15))
plot_image(epsf.data, ax=ax1, cmap='viridis')
fwhms = []
for a, ax in ((0, ax3), (1, ax2)):
# Collapse the PDF along this axis:
profile = epsf.data.sum(axis=a)
itop = profile.argmax()
poffset = profile[itop] / 2
# Run a spline through the points, but subtract half of the peak value, and find the roots:
# We have to use a cubic spline, since roots() is not supported for other splines
# for some reason
profile_intp = UnivariateSpline(np.arange(0, len(profile)),
profile - poffset,
k=3,
s=0,
ext=3)
lr = profile_intp.roots()
axis_fwhm = lr[1] - lr[0]
fwhms.append(axis_fwhm)
x_fine = np.linspace(-0.5, len(profile) - 0.5, 500)
ax.plot(profile, 'k.-')
ax.plot(x_fine, profile_intp(x_fine) + poffset, 'g-')
ax.axvline(itop)
ax.axvspan(lr[0], lr[1], facecolor='g', alpha=0.2)
ax.set_xlim(-0.5, len(profile) - 0.5)
# Let's make the final FWHM the largest one we found:
fwhm = np.max(fwhms)
logger.info("Final FWHM based on ePSF: %f", fwhm)
#ax2.axvspan(itop - fwhm/2, itop + fwhm/2, facecolor='b', alpha=0.2)
#ax3.axvspan(itop - fwhm/2, itop + fwhm/2, facecolor='b', alpha=0.2)
ax4.axis('off')
fig.savefig(os.path.join(output_folder, 'epsf.png'), bbox_inches='tight')
plt.close(fig)
#==============================================================================================
# COORDINATES TO DO PHOTOMETRY AT
#==============================================================================================
coordinates = np.array([[ref['pixel_column'], ref['pixel_row']]
for ref in references])
# Add the main target position as the first entry:
if datafile.get('template') is None:
coordinates = np.concatenate(([target_pixel_pos], coordinates), axis=0)
#==============================================================================================
# APERTURE PHOTOMETRY
#==============================================================================================
# Define apertures for aperture photometry:
apertures = CircularAperture(coordinates, r=fwhm)
annuli = CircularAnnulus(coordinates, r_in=1.5 * fwhm, r_out=2.5 * fwhm)
apphot_tbl = aperture_photometry(image.subclean, [apertures, annuli],
mask=image.mask,
error=image.error)
logger.debug("Aperture Photometry Table:\n%s", apphot_tbl)
logger.info('Apperature Photometry Success')
#==============================================================================================
# PSF PHOTOMETRY
#==============================================================================================
# Are we fixing the postions?
epsf.fixed.update({'x_0': False, 'y_0': False})
# Create photometry object:
photometry = BasicPSFPhotometry(group_maker=DAOGroup(fwhm),
bkg_estimator=SExtractorBackground(),
psf_model=epsf,
fitter=fitting.LevMarLSQFitter(),
fitshape=size,
aperture_radius=fwhm)
psfphot_tbl = photometry(image=image.subclean,
init_guesses=Table(coordinates,
names=['x_0', 'y_0']))
logger.debug("PSF Photometry Table:\n%s", psfphot_tbl)
logger.info('PSF Photometry Success')
#==============================================================================================
# TEMPLATE SUBTRACTION AND TARGET PHOTOMETRY
#==============================================================================================
if datafile.get('template') is not None:
# Find the pixel-scale of the science image:
pixel_area = proj_plane_pixel_area(image.wcs.celestial)
pixel_scale = np.sqrt(pixel_area) * 3600 # arcsec/pixel
#print(image.wcs.celestial.cunit) % Doesn't work?
logger.info("Science image pixel scale: %f", pixel_scale)
# Run the template subtraction, and get back
# the science image where the template has been subtracted:
diffimage = run_imagematch(datafile,
target,
star_coord=coordinates,
fwhm=fwhm,
pixel_scale=pixel_scale)
# Include mask from original image:
diffimage = np.ma.masked_array(diffimage, image.mask)
# Create apertures around the target:
apertures = CircularAperture(target_pixel_pos, r=fwhm)
annuli = CircularAnnulus(target_pixel_pos,
r_in=1.5 * fwhm,
r_out=2.5 * fwhm)
# Create two plots of the difference image:
fig, ax = plt.subplots(1, 1, squeeze=True, figsize=(20, 20))
plot_image(diffimage, ax=ax, cbar='right', title=target_name)
ax.plot(target_pixel_pos[0],
target_pixel_pos[1],
marker='+',
color='r')
fig.savefig(os.path.join(output_folder, 'diffimg.png'),
bbox_inches='tight')
apertures.plot(color='r')
annuli.plot(color='k')
ax.set_xlim(target_pixel_pos[0] - 50, target_pixel_pos[0] + 50)
ax.set_ylim(target_pixel_pos[1] - 50, target_pixel_pos[1] + 50)
fig.savefig(os.path.join(output_folder, 'diffimg_zoom.png'),
bbox_inches='tight')
plt.close(fig)
# Run aperture photometry on subtracted image:
target_apphot_tbl = aperture_photometry(diffimage, [apertures, annuli],
mask=image.mask,
error=image.error)
# Run PSF photometry on template subtracted image:
target_psfphot_tbl = photometry(diffimage,
init_guesses=Table(
target_pixel_pos,
names=['x_0', 'y_0']))
# Combine the output tables from the target and the reference stars into one:
apphot_tbl = vstack([target_apphot_tbl, apphot_tbl], join_type='exact')
psfphot_tbl = vstack([target_psfphot_tbl, psfphot_tbl],
join_type='exact')
# Build results table:
tab = references.copy()
tab.insert_row(
0, {
'starid': 0,
'ra': target['ra'],
'decl': target['decl'],
'pixel_column': target_pixel_pos[0],
'pixel_row': target_pixel_pos[1]
})
for key in ('pm_ra', 'pm_dec', 'gaia_mag', 'gaia_bp_mag', 'gaia_rp_mag',
'H_mag', 'J_mag', 'K_mag', 'g_mag', 'r_mag', 'i_mag', 'z_mag'):
tab[0][key] = np.NaN
# Subtract background estimated from annuli:
flux_aperture = apphot_tbl['aperture_sum_0'] - (
apphot_tbl['aperture_sum_1'] / annuli.area()) * apertures.area()
flux_aperture_error = np.sqrt(apphot_tbl['aperture_sum_err_0']**2 +
(apphot_tbl['aperture_sum_err_1'] /
annuli.area() * apertures.area())**2)
# Add table columns with results:
tab['flux_aperture'] = flux_aperture / image.exptime
tab['flux_aperture_error'] = flux_aperture_error / image.exptime
tab['flux_psf'] = psfphot_tbl['flux_fit'] / image.exptime
tab['flux_psf_error'] = psfphot_tbl['flux_unc'] / image.exptime
tab['pixel_column_psf_fit'] = psfphot_tbl['x_fit']
tab['pixel_row_psf_fit'] = psfphot_tbl['y_fit']
tab['pixel_column_psf_fit_error'] = psfphot_tbl['x_0_unc']
tab['pixel_row_psf_fit_error'] = psfphot_tbl['y_0_unc']
# Theck that we got valid photometry:
if not np.isfinite(tab[0]['flux_psf']) or not np.isfinite(
tab[0]['flux_psf_error']):
raise Exception("Target magnitude is undefined.")
#==============================================================================================
# CALIBRATE
#==============================================================================================
# Convert PSF fluxes to magnitudes:
mag_inst = -2.5 * np.log10(tab['flux_psf'])
mag_inst_err = (2.5 / np.log(10)) * (tab['flux_psf_error'] /
tab['flux_psf'])
# Corresponding magnitudes in catalog:
mag_catalog = tab[ref_filter]
# Mask out things that should not be used in calibration:
use_for_calibration = np.ones_like(mag_catalog, dtype='bool')
use_for_calibration[0] = False # Do not use target for calibration
use_for_calibration[~np.isfinite(mag_inst)
| ~np.isfinite(mag_catalog)] = False
# Just creating some short-hands:
x = mag_catalog[use_for_calibration]
y = mag_inst[use_for_calibration]
yerr = mag_inst_err[use_for_calibration]
# Fit linear function with fixed slope, using sigma-clipping:
model = models.Linear1D(slope=1, fixed={'slope': True})
fitter = fitting.FittingWithOutlierRemoval(fitting.LinearLSQFitter(),
sigma_clip,
sigma=3.0)
best_fit, sigma_clipped = fitter(model, x, y, weights=1.0 / yerr**2)
# Extract zero-point and estimate its error:
# I don't know why there is not an error-estimate attached directly to the Parameter?
zp = -1 * best_fit.intercept.value # Negative, because that is the way zeropoints are usually defined
zp_error = nanstd(y[~sigma_clipped] - best_fit(x[~sigma_clipped]))
# Add calibrated magnitudes to the photometry table:
tab['mag'] = mag_inst + zp
tab['mag_error'] = np.sqrt(mag_inst_err**2 + zp_error**2)
fig, ax = plt.subplots(1, 1)
ax.errorbar(x, y, yerr=yerr, fmt='k.')
ax.scatter(x[sigma_clipped], y[sigma_clipped], marker='x', c='r')
ax.plot(x, best_fit(x), color='g', linewidth=3)
ax.set_xlabel('Catalog magnitude')
ax.set_ylabel('Instrumental magnitude')
fig.savefig(os.path.join(output_folder, 'calibration.png'),
bbox_inches='tight')
plt.close(fig)
#==============================================================================================
# SAVE PHOTOMETRY
#==============================================================================================
# Descriptions of columns:
tab['flux_aperture'].unit = u.count / u.second
tab['flux_aperture_error'].unit = u.count / u.second
tab['flux_psf'].unit = u.count / u.second
tab['flux_psf_error'].unit = u.count / u.second
tab['pixel_column'].unit = u.pixel
tab['pixel_row'].unit = u.pixel
tab['pixel_column_psf_fit'].unit = u.pixel
tab['pixel_row_psf_fit'].unit = u.pixel
tab['pixel_column_psf_fit_error'].unit = u.pixel
tab['pixel_row_psf_fit_error'].unit = u.pixel
# Meta-data:
tab.meta['version'] = __version__
tab.meta['fileid'] = fileid
tab.meta['template'] = None if datafile.get(
'template') is None else datafile['template']['fileid']
tab.meta['photfilter'] = photfilter
tab.meta['fwhm'] = fwhm
tab.meta['obstime-bmjd'] = float(image.obstime.mjd)
tab.meta['zp'] = zp
tab.meta['zp_error'] = zp_error
# Filepath where to save photometry:
photometry_output = os.path.join(output_folder, 'photometry.ecsv')
# Write the final table to file:
tab.write(photometry_output,
format='ascii.ecsv',
delimiter=',',
overwrite=True)
toc = default_timer()
logger.info("Photometry took: %f seconds", toc - tic)
return photometry_output
def get_psf(data, starlist_x, starlist_y, do_plot='no', n_resample=4):
# create table with x and y coordinates for the epsf stars
stars_tbl = Table()
stars_tbl['x'], stars_tbl['y'] = starlist_x, starlist_y
# create bkg subtracted cutouts around stars
mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=2.)
data -= median_val
# extraction function requires data as NDData object
nddata = NDData(data=data)
# extract 30 x 30 px cutouts around the stars
stars = extract_stars(nddata, stars_tbl, size=30)
# build the EPSF from the selected stars
epsf_builder = EPSFBuilder(oversampling=n_resample, maxiters=10)
epsf, fitted_stars = epsf_builder(stars)
# fit gaussian through PSF to estimate FWHM
params = lmfit.Parameters()
params.add('h', 0.006, min=0.004, max=0.01, vary=True)
params.add('std', 5, min=1, max=10, vary=True)
params.add('cen', 50, min=45, max=55, vary=True)
len_x = len(epsf.data[0, :])
x = np.linspace(0, 100, len_x)
cutthrough = epsf.data[int(len(x) / 2), :]
minimizer = lmfit.Minimizer(misc.single_egauss,
params,
fcn_args=(x, cutthrough))
result = minimizer.minimize()
gauss_std = result.params['std']
h, cen = result.params['h'], result.params['cen']
if do_plot == 'yes':
fig_psf, ax_psf = plt.subplots(figsize=(8, 8))
ax_psf.imshow(epsf.data, origin='lower', cmap='inferno')
ax_psf.set_xlabel('x coordinate [px]')
ax_psf.set_ylabel('y coordinate [px]')
fig_psf.suptitle('3D PSF')
fig_psf.savefig('PSF_2D.pdf', bbox_inches='tight')
# plot cuts through the ePSF at different x positions
len_x = len(epsf.data[0, :])
x = np.linspace(0, 100, len_x)
fig1, ax1 = plt.subplots(figsize=(8, 8))
xvals = [int(len(x) / 4), int(len(x) / 2.1), int(len(x) / 2)]
for xval in xvals:
cutthrough = epsf.data[xval, :]
lab = 'x = ' + str(xval)
ax1.plot(x, cutthrough, label=lab)
g = misc.emission_gaussian(x, h, cen, gauss_std)
ax1.plot(x, g, label='Gaussian fit')
ax1.set_xlabel('x coordinate [px]')
ax1.set_ylabel('EPSF normalized flux')
ax1.legend()
fig1.suptitle('2D cut-through of the PSF')
fig1.savefig('PSF_cutthrough.pdf', bbox_inches='tight')
# return epsf, std devition of gaussian fit and resampling parameter
return epsf, gauss_std, n_resample
def jwst_camera_fpa_data(data_dir,
pattern,
standardized_data_dir,
parameters,
overwrite_source_extraction=False):
"""Generate standardized focal plane alignment (fpa) data
based on JWST camera image.
"""
save_plot = parameters['save_plot']
file_list = glob.glob(os.path.join(data_dir, '*{}'.format(pattern)))
if len(file_list) == 0:
raise RuntimeError('No data found')
file_list.sort()
for f in file_list:
plt.close('all')
print()
print('Data directory: {}'.format(data_dir))
print('Image being processed: {}'.format(f))
im = datamodels.open(f)
if hasattr(im, 'data') is False:
im.data = fits.getdata(f)
#im.dq = np.zeros(im.data.shape)
header_info = OrderedDict()
for attribute in 'telescope'.split():
header_info[attribute] = getattr(im.meta, attribute)
# observations
for attribute in 'date time visit_number visit_id visit_group activity_id program_number'.split(
):
header_info['observation_{}'.format(attribute)] = getattr(
im.meta.observation, attribute)
header_info['epoch_isot'] = '{}T{}'.format(
header_info['observation_date'], header_info['observation_time'])
# instrument
for attribute in 'name filter pupil detector'.split():
header_info['instrument_{}'.format(attribute)] = getattr(
im.meta.instrument, attribute)
# subarray
for attribute in 'name'.split():
header_info['subarray_{}'.format(attribute)] = getattr(
im.meta.subarray, attribute)
# aperture
for attribute in 'name position_angle pps_name'.split():
try:
value = getattr(im.meta.aperture, attribute)
except AttributeError:
value = None
header_info['aperture_{}'.format(attribute)] = value
header_info['INSTRUME'] = header_info['instrument_name']
header_info['SIAFAPER'] = header_info['aperture_name']
instrument_name = getattr(im.meta.instrument, 'name')
instrument_detector = getattr(im.meta.instrument, 'detector')
instrument_filter = getattr(im.meta.instrument, 'filter')
# temporary solution, this should come from populated aperture attributes
#if header_info['subarray_name'] == 'FULL':
# master_apertures = pysiaf.read.read_siaf_detector_layout()
# if header_info['instrument_name'].lower() in ['niriss', 'miri']:
# header_info['SIAFAPER'] = master_apertures['AperName'][np.where(master_apertures['InstrName']==header_info['instrument_name'])[0][0]]
# elif header_info['instrument_name'].lower() in ['fgs']:
# header_info['SIAFAPER'] = 'FGS{}_FULL'.format(header_info['instrument_detector'][-1])
# elif header_info['instrument_name'].lower() in ['nircam']:
# header_info['SIAFAPER'] = header_info['aperture_name']
#else:
# sys.exit('Only FULL arrays are currently supported.')
# target
for attribute in 'ra dec catalog_name proposer_name'.split():
header_info['target_{}'.format(attribute)] = getattr(
im.meta.target, attribute)
# pointing
for attribute in 'ra_v1 dec_v1 pa_v3'.split():
try:
value = getattr(im.meta.pointing, attribute)
except AttributeError:
value = None
header_info['pointing_{}'.format(attribute)] = value
# add HST style keywords
header_info['PROGRAM_VISIT'] = '{}_{}'.format(
header_info['observation_program_number'],
header_info['observation_visit_id'])
header_info['PROPOSID'] = header_info['observation_program_number']
header_info['DATE-OBS'] = header_info['observation_date']
header_info['TELESCOP'] = header_info['telescope']
header_info['INSTRUME'] = header_info['instrument_name']
try:
header_info['APERTURE'] = header_info['SIAFAPER']
except KeyError:
header_info['APERTURE'] = None
header_info['CHIP'] = 0
# TBD: Need to remove making yet another directory
#extracted_sources_dir = os.path.join(standardized_data_dir, 'extraction')
#if os.path.isdir(extracted_sources_dir) is False:
# os.makedirs(extracted_sources_dir)
extracted_sources_file = os.path.join(
standardized_data_dir, #extracted_sources_dir,
'{}_extracted_sources.fits'.format(
os.path.basename(f).split('.')[0]))
mask_extreme_slope_values = False
parameters['maximum_slope_value'] = 1000.
# Check if extracted_sources_file exists, or overwrite_source_extraction is set to True
if (not os.path.isfile(extracted_sources_file)) or (
overwrite_source_extraction):
data = copy.deepcopy(im.data)
#dq = copy.deepcopy(im.dq)
# Convert image data to counts per second
photmjsr = getattr(im.meta.photometry, 'conversion_megajanskys')
data_cps = data / photmjsr
if mask_extreme_slope_values:
# clean up extreme slope values
bad_index = np.where(
np.abs(data) > parameters['maximum_slope_value'])
data[bad_index] = 0.
dq[bad_index] = -1
bkgrms = MADStdBackgroundRMS()
mmm_bkg = MMMBackground()
bgrms = bkgrms(data_cps)
bgavg = mmm_bkg(data_cps)
# Default parameters that generally works for NIRCam/NIRISS images
sigma_factor = 10
round_lo, round_hi = 0.0, 0.6
sharp_lo, sharp_hi = 0.3, 1.4
fwhm_lo, fwhm_hi = 1.0, 20.0
fwhm = 2.0
minsep_fwhm = 7 # NOTE: minsep_fwhm>5 to reject artifacts around saturated stars
flux_percent_lo, flux_percent_hi = 10, 99
# if 'sharp_lo' in parameters:
# sharp_lo = parameters['sharp_lo']
###
### TBD1: Relocate params below to config parts/files
###
# Use different criteria for selecting good stars
if parameters['nominalpsf']:
# If using Nominal PSF models
if instrument_name == 'NIRISS':
#fwhm_lo, fwhm_hi = 1.0, 2.0
sharp_lo, sharp_hi = 0.6, 1.4
elif instrument_name == 'FGS':
#fwhm_lo, fwhm_hi = 1.0, 1.4
sharp_lo, sharp_hi = 0.6, 1.4
elif instrument_name == 'NIRCAM':
sharp_lo, sharp_hi = 0.6, 1.4
elif instrument_name == 'MIRI':
sharp_lo, sharp_hi = 0.8, 1.0
fwhm_lo, fwhm_hi = 1.5, 2.2
sigma_factor = 3
elif instrument_name == 'NIRSPEC':
sharp_lo, sharp_hi = 0.6, 0.8
round_lo, round_hi = 0.0, 0.3
fwhm_lo, fwhm_hi = 1.0, 1.75
else:
###
### For OTE commissioning, tweak the params below after finding
### the correct ranges by runnin the photometry notebook.
###
# If using Commissioning (non-phased) PSF models
if instrument_name == 'NIRISS':
sharp_lo, sharp_hi = 0.6, 1.4
fwhm_lo, fwhm_hi = 1.4, 2.4
################################################################################
################################################################################
################################################################################
elif instrument_name == 'FGS':
sigma_factor = 10
minsep_fwhm = 2.5
sharp_lo, sharp_hi = 0.45, 0.7
round_lo, round_hi = 0.0, 0.3
flux_percent_lo, flux_percent_hi = 2, 99
fwhm = 4
################################################################################
################################################################################
################################################################################
# Below works well for F200W and F356W images
elif instrument_name == 'NIRCAM':
sigma_factor = 3
minsep_fwhm = 2.5
sharp_lo, sharp_hi = 0.5, 0.7
round_lo, round_hi = 0.0, 0.2
flux_percent_lo, flux_percent_hi = 2, 99
if 'F200W' in instrument_filter:
fwhm = 10
elif 'F356W' in instrument_filter:
fwhm = 8
elif 'F090W' in instrument_filter:
fwhm = 5.5
elif 'F277W' in instrument_filter:
fwhm = 6.5
else:
fwhm = 3
################################################################################
################################################################################
################################################################################
elif instrument_name == 'MIRI':
sharl_lo, sharp_hi = 0.5, 1.0
fwhm_lo, fwhm_hi = 1.5, 2.2
sigma_factor = 3
elif instrument_name == 'NIRSPEC':
sharp_lo, sharp_hi = 0.5, 0.8
round_lo, round_hi = 0.0, 0.3
fwhm_lo, fwhm_hi = 1.0, 1.75
# Use IRAFStarFinder for source detection
iraffind = IRAFStarFinder(threshold=sigma_factor * bgrms + bgavg,
fwhm=fwhm,
minsep_fwhm=minsep_fwhm,
roundlo=round_lo,
roundhi=round_hi,
sharplo=sharp_lo,
sharphi=sharp_hi)
# Create default mask with all False values
datamask = np.zeros(
data_cps.shape,
dtype=bool) # This creates an array with all False
# Mask the left (for NRS1) and right regions (for NRS2) for NIRSpec
if instrument_detector == 'NRS1':
datamask[:, :1023] = True # Mask everything on the left side
elif instrument_detector == 'NRS2':
datamask[:, 1024:] = True # Mask everything on the right side
iraf_extracted_sources = iraffind(data_cps, mask=datamask)
# Perform some basic filtering
# Remove sources based on flux percentile
# 10-99% works well for filtering out too faint or saturated sources
flux_min = np.percentile(iraf_extracted_sources['flux'],
flux_percent_lo)
flux_max = np.percentile(iraf_extracted_sources['flux'],
flux_percent_hi)
iraf_extracted_sources.remove_rows(
np.where(iraf_extracted_sources['flux'] < flux_min))
iraf_extracted_sources.remove_rows(
np.where(iraf_extracted_sources['flux'] > flux_max))
# Also remove sources based on fwhm
###
### Don't use below for now - 2/23/2022 (Don't use it unless we get lots of bad sources)
###
#iraf_extracted_sources.remove_rows(np.where(iraf_extracted_sources['fwhm']<fwhm_lo))
#iraf_extracted_sources.remove_rows(np.where(iraf_extracted_sources['fwhm']>fwhm_hi))
# Now improve the positions by re-running centroiding algorithm if necessary.
# NOTE: For now, re-centroiding will be turned off
###
### TBD2: Add re-centroiding algorithm adopted from Paul here
###
#xarr = sources_masked['xcentroid']
#yarr = sources_masked['ycentroid']
#newx, newy = centroid_sources(data_cps, xarr, yarr, box_size=5, centroid_func=centroid_2dg)
#coords = np.column_stack((newx, newy))
#srcaper = CircularAnnulus(coords, r_in=1, r_out=3)
#srcaper_masks = srcaper.to_mask(method='center')
#satflag = np.zeros((len(newx),),dtype=int)
#i = 0
#for mask in srcaper_masks:
# srcaper_dq = mask.multiply(dqarr)
# srcaper_dq_1d = srcaper_dq[mask.data>0]
# badpix = np.logical_and(srcaper_dq_1d>2, srcaper_dq_1d<7)
# reallybad = np.where(srcaper_dq_1d==1)
# if ((len(srcaper_dq_1d[badpix]) > 1) or (len(srcaper_dq_1d[reallybad]) > 0)):
# satflag[i] = 1
# i =+1
#goodx = newx[np.where(satflag==0)]
#goody = newy[np.where(satflag==0)]
#print('Number of sources before removing saturated or bad pixels: ', len(xarr))
#print('Number of sources without saturated or bad pixels: ', len(goodx))
#print(' ')
#coords = np.column_stack((goodx,goody))
print('Number of extracted sources after filtering: {} sources'.
format(len(iraf_extracted_sources)))
if parameters['use_epsf'] is True:
size = 25
hsize = (size - 1) / 2
x = iraf_extracted_sources['xcentroid']
y = iraf_extracted_sources['ycentroid']
mask = ((x > hsize) & (x < (data_cps.shape[1] - 1 - hsize)) &
(y > hsize) & (y < (data_cps.shape[0] - 1 - hsize)))
stars_tbl = Table()
stars_tbl['x'] = x[mask]
stars_tbl['y'] = y[mask]
print('Using {} stars to build epsf'.format(len(stars_tbl)))
data_cps_bkgsub = data_cps.copy()
data_cps_bkgsub -= bgavg
nddata = NDData(data=data_cps_bkgsub)
stars = extract_stars(nddata, stars_tbl, size=size)
#
# Figure - PSF stars
#
nrows = 10
ncols = 10
fig, ax = plt.subplots(nrows=nrows,
ncols=ncols,
figsize=(20, 20),
squeeze=True)
ax = ax.ravel()
for i in range(nrows * ncols):
if i <= len(stars) - 1:
norm = simple_norm(stars[i], 'log', percent=99.)
ax[i].imshow(stars[i],
norm=norm,
origin='lower',
cmap='viridis')
plt.title('{} sample stars for epsf'.format(
header_info['APERTURE']))
if save_plot:
figname = os.path.join(
extracted_sources_dir, '{}_sample_psfs.pdf'.format(
os.path.basename(f).split('.')[0]))
plt.savefig(figname)
if parameters['show_extracted_sources']:
plt.show()
#
# Timer for ePSF construction
#
tic = time.perf_counter()
epsf_builder = EPSFBuilder(oversampling=4,
maxiters=3,
progress_bar=False)
print("Building ePSF ...")
epsf, fitted_stars = epsf_builder(stars)
toc = time.perf_counter()
print("Time elapsed for building ePSF:", toc - tic)
#
# Figure - ePSF plot
#
norm_epsf = simple_norm(epsf.data, 'log', percent=99.)
plt.figure()
plt.imshow(epsf.data,
norm=norm_epsf,
origin='lower',
cmap='viridis')
plt.colorbar()
plt.title('{} epsf using {} stars'.format(
header_info['APERTURE'], len(stars_tbl)))
if save_plot:
figname = os.path.join(
extracted_sources_dir, '{}_epsf.pdf'.format(
os.path.basename(f).split('.')[0]))
plt.savefig(figname)
if parameters['show_extracted_sources']:
plt.show()
daogroup = DAOGroup(5.0 * 2.0)
psf_model = epsf.copy()
tic = time.perf_counter()
photometry = IterativelySubtractedPSFPhotometry(
finder=iraffind,
group_maker=daogroup,
bkg_estimator=mmm_bkg,
psf_model=psf_model,
fitter=LevMarLSQFitter(),
niters=1,
fitshape=(11, 11),
aperture_radius=5)
print('Performing source extraction and photometry ...')
epsf_extracted_sources = photometry(data_cps)
toc = time.perf_counter()
print("Time elapsed for PSF photometry:", toc - tic)
print('Final source extraction with epsf: {} sources'.format(
len(epsf_extracted_sources)))
epsf_extracted_sources['xcentroid'] = epsf_extracted_sources[
'x_fit']
epsf_extracted_sources['ycentroid'] = epsf_extracted_sources[
'y_fit']
extracted_sources = epsf_extracted_sources
extracted_sources.write(extracted_sources_file, overwrite=True)
norm = simple_norm(data_cps, 'sqrt', percent=99.)
diff = photometry.get_residual_image()
plt.figure()
ax1 = plt.subplot(1, 2, 1)
plt.xlabel("X [pix]")
plt.ylabel("Y [pix]")
ax1.imshow(data_cps, norm=norm, cmap='Greys')
ax2 = plt.subplot(1, 2, 2)
plt.xlabel("X [pix]")
plt.ylabel("Y [pix]")
ax2.imshow(diff, norm=norm, cmap='Greys')
plt.title('PSF subtracted image for {}'.format(
os.path.basename(f)))
if save_plot:
figname = os.path.join(
extracted_sources_dir,
'{}_psfsubtracted_image.pdf'.format(
os.path.basename(f).split('.')[0]))
plt.savefig(figname)
if parameters['show_psfsubtracted_image']:
plt.show()
else:
extracted_sources = iraf_extracted_sources
extracted_sources.write(extracted_sources_file, overwrite=True)
positions = np.transpose((extracted_sources['xcentroid'],
extracted_sources['ycentroid']))
apertures = CircularAperture(positions, r=10)
norm = simple_norm(data_cps, 'sqrt', percent=99.)
plt.figure(figsize=(12, 12))
plt.xlabel("X [pix]")
plt.ylabel("Y [pix]")
plt.imshow(data_cps, norm=norm, cmap='Greys', origin='lower')
apertures.plot(color='blue', lw=1.5, alpha=0.5)
title_string = '{}: {} selected sources'.format(
os.path.basename(f), len(extracted_sources))
plt.title(title_string)
plt.tight_layout()
if save_plot:
figname = os.path.join(
standardized_data_dir, '{}_extracted_sources.pdf'.format(
os.path.basename(f).split('.')[0]))
plt.savefig(figname)
if parameters['show_extracted_sources']:
plt.show()
plt.close()
else:
extracted_sources = Table.read(extracted_sources_file)
print('Extracted {} sources from {}'.format(len(extracted_sources), f))
impose_positive_flux = True
if impose_positive_flux and parameters['use_epsf']:
extracted_sources.remove_rows(
np.where(extracted_sources['flux_fit'] < 0)[0])
print('Only {} sources have positve flux'.format(
len(extracted_sources)))
astrometry_uncertainty_mas = 5
if len(extracted_sources) > 0:
# Cal images are in DMS coordinates which correspond to the SIAF Science (SCI) frame
extracted_sources['x_SCI'], extracted_sources[
'y_SCI'] = extracted_sources['xcentroid'], extracted_sources[
'ycentroid']
# For now, astrometric uncertainty defaults to 5 mas for each source.
extracted_sources['sigma_x_mas'] = np.ones(
len(extracted_sources)) * astrometry_uncertainty_mas
extracted_sources['sigma_y_mas'] = np.ones(
len(extracted_sources)) * astrometry_uncertainty_mas
# transfer info to astropy table header
for key, value in header_info.items():
extracted_sources.meta[key] = value
extracted_sources.meta['DATAFILE'] = os.path.basename(f)
extracted_sources.meta['DATAPATH'] = os.path.dirname(f)
extracted_sources.meta['EPOCH'] = header_info['epoch_isot']
out_file = os.path.join(
standardized_data_dir, '{}_FPA_data.fits'.format(
extracted_sources.meta['DATAFILE'].split('.')[0]))
print('Writing {}'.format(out_file))
with warnings.catch_warnings():
warnings.simplefilter('ignore', AstropyWarning, append=True)
extracted_sources.write(out_file, overwrite=True)
return im
def basic_psf_phot(target, centroided_sources, plots=False):
def hmsm_to_days(hour=0,min=0,sec=0,micro=0):
"""
Convert hours, minutes, seconds, and microseconds to fractional days.
"""
days = sec + (micro / 1.e6)
days = min + (days / 60.)
days = hour + (days / 60.)
return days / 24.
def date_to_jd(year,month,day):
"""
Convert a date to Julian Day.
Algorithm from 'Practical Astronomy with your Calculator or Spreadsheet',
4th ed., Duffet-Smith and Zwart, 2011.
"""
if month == 1 or month == 2:
yearp = year - 1
monthp = month + 12
else:
yearp = year
monthp = month
# this checks where we are in relation to October 15, 1582, the beginning
# of the Gregorian calendar.
if ((year < 1582) or
(year == 1582 and month < 10) or
(year == 1582 and month == 10 and day < 15)):
# before start of Gregorian calendar
B = 0
else:
# after start of Gregorian calendar
A = math.trunc(yearp / 100.)
B = 2 - A + math.trunc(A / 4.)
if yearp < 0:
C = math.trunc((365.25 * yearp) - 0.75)
else:
C = math.trunc(365.25 * yearp)
D = math.trunc(30.6001 * (monthp + 1))
jd = B + C + D + day + 1720994.5
return jd
def gaussian(p, x, y):
height, center_x, center_y, width_x, width_y, rotation = p
rotation = np.deg2rad(rotation)
x_0 = center_x * np.cos(rotation) - center_y * np.sin(rotation)
y_0 = center_x * np.sin(rotation) + center_y * np.cos(rotation)
def rotgauss(x,y):
xp = x * np.cos(rotation) - y * np.sin(rotation)
yp = x * np.sin(rotation) + y * np.cos(rotation)
g = height*np.exp(
-(((x_0-xp)/width_x)**2+
((y_0-yp)/width_y)**2)/2.)
return g
g = rotgauss(x,y)
return g
def moments(data):
total = np.nansum(data)
X, Y = np.indices(data.shape)
center_x = int(np.shape(data)[1]/2)
center_y = int(np.shape(data)[0]/2)
row = data[int(center_x), :]
col = data[:, int(center_y)]
width_x = np.nansum(np.sqrt(abs((np.arange(col.size)-center_y)**2*col))
/np.nansum(col))
width_y = np.nansum(np.sqrt(abs((np.arange(row.size)-center_x)**2*row))
/np.nansum(row))
height = np.nanmax(data)
rotation = 0.0
return height, center_x, center_y, width_x, width_y, rotation
def errorfunction(p, x, y, data):
return gaussian(p, x, y) - data
def fitgaussian(data):
params = moments(data)
X, Y = np.indices(data.shape)
mask = ~np.isnan(data)
x = X[mask]
y = Y[mask]
data = data[mask]
p, success = optimize.leastsq(errorfunction, params, args=(x, y, data))
return p
pines_path = pines_dir_check()
short_name = short_name_creator(target)
reduced_path = pines_path/('Objects/'+short_name+'/reduced/')
reduced_files = np.array(natsort.natsorted([x for x in reduced_path.glob('*.fits')]))
centroided_sources.columns = centroided_sources.columns.str.strip()
source_names = natsort.natsorted(list(set([i[0:-2].replace('X','').replace('Y','').rstrip().lstrip() for i in centroided_sources.keys()])))
#Declare a new dataframe to hold the information for all targets for this .
columns = ['Filename', 'Time UT', 'Time JD', 'Airmass', 'Seeing']
for i in range(0, len(source_names)):
columns.append(source_names[i]+' Flux')
columns.append(source_names[i]+' Flux Error')
psf_df = pd.DataFrame(index=range(len(reduced_files)), columns=columns)
output_filename = pines_path/('Objects/'+short_name+'/psf_phot/'+short_name+'_psf_phot.csv')
for i in range(len(reduced_files)):
#Read in image data/header.
file = reduced_files[i]
data = fits.open(file)[0].data
header = fits.open(file)[0].header
print('{}, image {} of {}.'.format(file.name, i+1, len(reduced_files)))
#Read in some supporting information.
log_path = pines_path/('Logs/'+file.name.split('.')[0]+'_log.txt')
log = pines_log_reader(log_path)
date_obs = header['DATE-OBS']
#Catch a case that can cause datetime strptime to crash; Mimir headers sometimes have DATE-OBS with seconds specified as 010.xx seconds, when it should be 10.xx seconds.
if len(date_obs.split(':')[-1].split('.')[0]) == 3:
date_obs = date_obs.split(':')[0] + ':' + date_obs.split(':')[1] + ':' + date_obs.split(':')[-1][1:]
#Keep a try/except clause here in case other unknown DATE-OBS formats pop up.
try:
date = datetime.datetime.strptime(date_obs, '%Y-%m-%dT%H:%M:%S.%f')
except:
print('Header DATE-OBS format does not match the format code in strptime! Inspect/correct the DATE-OBS value.')
pdb.set_trace()
days = date.day + hmsm_to_days(date.hour,date.minute,date.second,date.microsecond)
jd = date_to_jd(date.year,date.month,days)
psf_df['Filename'][i] = file.name
psf_df['Time UT'][i] = header['DATE-OBS']
psf_df['Time JD'][i] = jd
psf_df['Airmass'][i] = header['AIRMASS']
psf_df['Seeing'][i] = log['X seeing'][np.where(log['Filename'] == file.name.split('_')[0]+'.fits')[0][0]]
#Read in source centroids for this image
x = np.zeros(len(source_names))
y = np.zeros(len(source_names))
seeing = psf_df['Seeing'][i]
for j in range(len(source_names)):
source = source_names[j]
x[j] = centroided_sources[source+' X'][i]
y[j] = centroided_sources[source+' Y'][i]
#The extract_stars() function requires the input data as an NDData object.
nddata = NDData(data=data)
#Create table of good star positions
stars_tbl = Table()
stars_tbl['x'] = x
stars_tbl['y'] = y
size = 25
x, y = np.meshgrid(np.arange(0,size), np.arange(0,size))
#Extract star cutouts.
stars = extract_stars(nddata, stars_tbl, size=size)
fitter = fitting.LevMarLSQFitter()
fig, ax = plt.subplots(nrows=len(stars), ncols=3, sharex=True, sharey=True, figsize=(12,40))
#Fit a 2D Gaussian to each star.
for j in range(len(stars)):
star = stars[j]
source = source_names[j]
mmm_bkg = MMMBackground()
cutout = star.data - mmm_bkg(star.data)
#Get the star's centroid position in the cutout.
dtype = [('x_0', 'f8'), ('y_0', 'f8')]
pos = Table(data=np.zeros(1, dtype=dtype))
source_x = stars_tbl['x'][j]
source_y = stars_tbl['y'][j]
pos['x_0'] = source_x - int(source_x - size/2 + 1)
pos['y_0'] = source_y - int(source_y - size/2 + 1)
parameters = fitgaussian(cutout)
g2d_fit = gaussian(parameters, x, y)
avg, med, std = sigma_clipped_stats(cutout)
im = ax[j,0].imshow(cutout, origin='lower', vmin=med-std, vmax=med+8*std)
divider = make_axes_locatable(ax[j,0])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax[j,0].plot(pos['x_0'], pos['y_0'], 'rx')
ax[j,0].set_ylabel(source)
ax[j,0].text(pos['x_0'], pos['y_0']+1, '('+str(np.round(source_x,1))+', '+str(np.round(source_y,1))+')', color='r', ha='center')
ax[j,0].axis('off')
axins = ax[j,0].inset_axes([0.75, 0.75, 0.25, 0.25])
axins.set_yticklabels([])
axins.set_yticks([])
axins.set_xticklabels([])
axins.set_xticks([])
axins.imshow(data, origin='lower', vmin=med-std, vmax=med+8*std)
axins.plot(source_x, source_y, 'rx')
im = ax[j,1].imshow(g2d_fit, origin='lower', vmin=med-std, vmax=med+8*std)
divider = make_axes_locatable(ax[j,1])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax[j,1].axis('off')
avg, med, std = sigma_clipped_stats(cutout - g2d_fit)
im = ax[j,2].imshow(cutout - g2d_fit, origin='lower', vmin=med-std, vmax=med+8*std)
divider = make_axes_locatable(ax[j,2])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax[j,2].axis('off')
if j == 0:
ax[j,0].set_title('Data')
ax[j,1].set_title('2D Gaussian Model')
ax[j,2].set_title('Data - Model')
plt.tight_layout()
output_filename = pines_path/('Objects/'+short_name+'/basic_psf_phot/'+reduced_files[i].name.split('_')[0]+'_'+'source_modeling.pdf')
plt.savefig(output_filename)
plt.close()
return
def find_pinholes_regular(fname,
sname,
fdarkff,
fdark,
fff,
files,
ref_shape,
size,
threshold,
fwhm,
fitshape,
range_psf,
sigma=2.,
oversampling=4,
maxiters=3):
"""Finds and fits regullary spread pinhole positions with a ePSF in a FITS image.
Parameters
----------
fname : str
Folder name of the input fits files.
sname : str
Folder name of the returned found and matched pinhole positions (txt files)
fdarkff : string
Location of the dark images for the flat field images.
fdark : string
Location of the dark images for the raw images.
fff : string
Location of the flat field images.
files : (1, 2)-shaped int array
File range to create a median image
ref_shape : (1,2)-shaped array
Number of reference stars in x and y direction [x, y].
size : int
Rectangular size of the ePSF. Size must be an odd number.
threshold : float
The absolute image value above which to select sources.
fwhm : float
The full-width half-maximum (FWHM) of the major axis of the Gaussian kernel in units of pixels.
fitshape : int or length-2 array-like
Rectangular shape around the center of a star which will be used to collect the data to do the fitting.
Can be an integer to be the same along both axes. E.g., 5 is the same as (5, 5), which means to fit only at the following
relative pixel positions: [-2, -1, 0, 1, 2]. Each element of fitshape must be an odd number.
range_psf : (1, 4)-shaped int array
Position range to compute epsf [xmin,xmax,ymin,ymax]
sigma : float
Number of standard deviations used to perform sigma clip with a astropy.stats.SigmaClip object.
oversampling : int or tuple of two int
The oversampling factor(s) of the ePSF relative to the input stars along the x and y axes.
The oversampling can either be a single float or a tuple of two floats of the form (x_oversamp, y_oversamp).
If oversampling is a scalar then the oversampling will be the same for both the x and y axes.
maxiters : int
The maximum number of iterations to perform.
Returns
-------
positions_sort : (N,2)-shaped array
Found and matched positions of the pinholes.
ref_positions : (N,2)-shaped array
Matched reference grid positions.
"""
#Load the sample of fits images
entries = os.listdir(fname)
data_col = np.array([fits.getdata(fname + '/' + entries[files[0]], ext=0)])
for k in range(files[0], files[1]):
data_col = np.append(data_col,
[fits.getdata(fname + '/' + entries[k], ext=0)],
axis=0)
#Data reduction: Darc current + Flatfield + bias
data_col = data_correction(data_col, fdarkff, fdark, fff)
#Claculate median image
data_full = np.median(data_col, axis=0)
data = data_full[range_psf[2]:range_psf[3], range_psf[0]:range_psf[1]]
#Find peaks in data
peaks_tbl = find_peaks(data, threshold=threshold)
peaks_tbl['peak_value'].info.format = '%.8g'
#Load data around found peaks
hsize = (size - 1) / 2
x = peaks_tbl['x_peak']
y = peaks_tbl['y_peak']
mask = ((x > hsize) & (x < (data.shape[1] - 1 - hsize)) & (y > hsize) &
(y < (data.shape[0] - 1 - hsize)))
stars_tbl = Table()
stars_tbl['x'] = x[mask]
stars_tbl['y'] = y[mask]
#Calculate mean, median, std
mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=sigma)
data = data - median_val
#Find pinholes and create ePSF
nddata = NDData(data=data)
stars = extract_stars(nddata, stars_tbl, size=size)
epsf_builder = EPSFBuilder(oversampling=oversampling,
maxiters=maxiters,
progress_bar=False)
epsf, fitted_stars = epsf_builder(stars)
#Use ePSF to find precise locations of pinholes
daofind = DAOPhotPSFPhotometry(crit_separation=30,
threshold=threshold,
fwhm=fwhm,
psf_model=epsf,
fitshape=fitshape,
aperture_radius=12,
niters=1)
#Get positions
sources = daofind(data_full)
for col in sources.colnames:
sources[col].info.format = '%.8g'
pos_full = np.transpose((sources['x_fit'], sources['y_fit']))
#Plot found pinholes
apertures = CircularAperture(pos_full, r=10)
norm = ImageNormalize(stretch=SqrtStretch())
fig, ax = plt.subplots()
ax.set_title('Pinhole Positions')
ax.set(xlabel='x [pixel]', ylabel='y [pixel]')
ax.imshow(data_full, cmap='Greys', origin='lower', norm=norm)
apertures.plot(color='blue', lw=1.5, alpha=0.5)
ax.legend(['#pinholes = ' + str(len(pos_full[:, 0]))],
loc='lower left',
prop={'size': 12})
plt.show()
#Find central position
xcent = (np.max(pos_full[:, 0]) + np.min(pos_full[:, 0])) / 2
ycent = (np.max(pos_full[:, 1]) + np.min(pos_full[:, 1])) / 2
#Find positions at the edges to set base positions for linear transformatio to match pinholes with reference grid
distance = (pos_full[:, 0] - xcent)**2 + (pos_full[:, 1] - ycent)**2
pins = len(distance)
sort_distance = np.partition(distance,
(pins - 4, pins - 3, pins - 2, pins - 1))
maxpos = pos_full[distance == sort_distance[pins - 1]]
maxpos = np.append(maxpos,
pos_full[distance == sort_distance[pins - 2]],
axis=0)
maxpos = np.append(maxpos,
pos_full[distance == sort_distance[pins - 3]],
axis=0)
maxpos = np.append(maxpos,
pos_full[distance == sort_distance[pins - 4]],
axis=0)
b01 = maxpos[maxpos[:, 1] < ycent]
b23 = maxpos[maxpos[:, 1] > ycent]
posbase = np.array(b01[b01[:, 0] < xcent])
posbase = np.append(posbase, b01[b01[:, 0] > xcent], axis=0)
posbase = np.append(posbase, b23[b23[:, 0] < xcent], axis=0)
print(posbase)
#Sort positions by matching with reference grid
positions_sort, ref_positions = sort_positions_regular(
pos_full, posbase, ref_shape)
text = np.array([
positions_sort[:, 0], positions_sort[:, 1], ref_positions[:, 0],
ref_positions[:, 1]
])
text_trans = np.zeros((len(positions_sort[:, 0]), 4))
#Transpose text matrix
for k in range(0, 4):
for l in range(0, len(positions_sort[:, 0])):
text_trans[l][k] = text[k][l]
#Save data as txt file
np.savetxt(
sname + '.txt',
text_trans,
fmt='%1.9E',
delimiter='\t',
header=
' x-measured y-measured x-reference y-reference',
comments='')
return positions_sort, ref_positions
# ['ibg403010_drz.fits', 597, 385],
# ['ibg405010_drz.fits', 570, 701], # GAIA bad
# ['ibg455010_drz.fits', 263, 444],
# ['ibg456010_drz.fits', 530, 696],
# ['ibg456010_drz.fits', 549, 462], # GAIA bad
# ['ibg456010_drz.fits', 860, 408],
# ['ibg456010_drz.fits', 911, 115],
# ['ibg465010_drz.fits', 588, 723],
# ['ibg471010_drz.fits', 600, 685],
# ['ibg471010_drz.fits', 892, 511],
#]
# -1 because the above positions are measured in ds9, which counts from (1,1)
# while the python code counts from (0,0)
stars621 = extract_stars([NDData(fits.open(datadir + row[0])[1].data) for row in prflist],
[Table({'x': [row[1] - 1], 'y': [row[2] - 1]}) for row in prflist],
size=25)
stars845 = extract_stars([NDData(fits.open(datadir + row[0].replace('10_', '20_'))[1].data) for row in prflist],
[Table({'x': [row[1] - 1], 'y': [row[2] - 1]}) for row in prflist],
size=25)
def check_matching_source_exists(l1, l2, d,
xname='xcentroid', yname='ycentroid'):
'''Check for each source in l1, if one or more sources in l2 are close
This is not the most efficient way to do things, but very quick to code and
runtime is not a concern for this.
Parameters
----------
def build_ePSF_astrometry(image_file,
mask_file=None,
nstars=40,
image_source_file=None,
astrom_sigma=5.0,
psf_sigma=5.0,
alim=10000,
lowper=0.6,
highper=0.9,
keep=False,
cutout=35,
write=True,
output=None,
plot=False,
output_plot=None,
verbose=False):
"""Build the effective Point-Spread Function using a sample of stars from
some image acquired via the `image2xy` tool of `astrometry.net`.
Arguments
---------
image_file : str
Filename for a **background-subtracted** image
mask_file : str, optional
Filename for a mask file (default None)
nstars : int, optional
*Maximum* number of stars to use in building the ePSF (default 40;
set to None to impose no limit)
image_source_file : str, optional
Filename for a `.xy.fits` file containing detected sources with their
pixel coordinates and **background-subtracted** flux (default None, in
which case a new such file is produced)
astrom_sigma : float, optional
Detection significance when using `image2xy` in `astrometry.net` to
find sources (default 5.0)
psf_sigma : float, optional
Sigma of the approximate Gaussian PSF of the images (default 5.0)
alim : int, optional
*Maximum* allowed source area in square pixels for `astrometry.net`,
above which sources will be deblended (default 10000)
lowper, highper : float, optional
Lower and upper flux percentiles (as a fraction between 0 and 1) such
that sources outside the corresponding flux range will be excluded from
ePSF building (default 0.6 and 0.9, respectively)
keep : bool, optional
Whether to keep the source list file (`.xy.fits` files; default False)
cutout : int, optional
Cutout size around each star in pixels (default 35; must be **odd**;
rounded **down** if even)
write : bool, optional
Whether to write the ePSF to a new fits file (default True)
output : str, optional
Name for the output ePSF data fits file (default
`image_file.replace(".fits", "_ePSF.fits")`)
plot : bool, optional
Whether to plot the newly-built ePSF (default False)
output_plot : str, optional
Name for the output figure (default
`image_file.replace(".fits", "_ePSF.png")`)
verbose : bool, optional
Whether to be verbose (default False)
Returns
-------
np.ndarray
The ePSF data in a 2D array
Notes
-----
Uses `astrometry.net` to obtain a list of sources in the image with their
x, y coordinates, flux, and background at their location. (If a list of
sources has already been obtained `solve-field` or `image2xy`, this can
be input). Finally, selects stars between the `lowper` th and `highper`
percentile fluxes.
Finally, uses `EPSFBuilder` to empirically obtain the ePSF of these stars.
Optionally writes and/or plots the obtained ePSF.
**The ePSF obtained here should not be used in convolutions.** Instead, it
can serve as a tool for estimating the seeing of an image.
"""
# ignore annoying warnings from photutils
from astropy.utils.exceptions import AstropyWarning
warnings.simplefilter('ignore', category=AstropyWarning)
from astropy.nddata import NDData
from photutils.psf import extract_stars
from photutils import EPSFBuilder
# load in data
image_data = fits.getdata(image_file)
image_header = fits.getheader(image_file)
try:
instrument = image_header["INSTRUME"]
except KeyError:
instrument = "Unknown"
### source detection
## use pre-existing file obtained by astrometry.net, if supplied
if image_source_file:
image_sources = np.logical_not(fits.getdata(image_source_file))
## use astrometry.net to find the sources
# -b --> no background-subtraction
# -O --> overwrite
# -p <astrom_sigma> --> signficance
# -w <psf_sigma> --> estimated PSF sigma
# -m <alim> --> max object size for deblending is <alim>
else:
options = f" -b -O -p {astrom_sigma} -w {psf_sigma} -m {alim}"
run(f"image2xy {options} {image_file}", shell=True)
image_sources_file = image_file.replace(".fits", ".xy.fits")
image_sources = fits.getdata(image_sources_file)
if not (keep):
run(f"rm {image_sources_file}", shell=True) # file is not needed
print(f'\n{len(image_sources)} stars at >{astrom_sigma} sigma found ' +
f'in image {re.sub(".*/", "", image_file)} with astrometry.net')
sources = Table() # build a table
sources['x'] = image_sources['X'] # for EPSFBuilder
sources['y'] = image_sources['Y']
sources['flux'] = image_sources['FLUX']
if nstars:
sources = sources[:min(nstars, len(sources))]
## get WCS coords for all sources
w = wcs.WCS(image_header)
sources["ra"], sources["dec"] = w.all_pix2world(sources["x"], sources["y"],
1)
## mask out edge sources:
# a bounding circle for WIRCam, rectangle for MegaPrime
xsize = image_data.shape[1]
ysize = image_data.shape[0]
if "WIRCam" in instrument:
rad_limit = xsize / 2.0
dist_to_center = np.sqrt((sources['x'] - xsize / 2.0)**2 +
(sources['y'] - ysize / 2.0)**2)
mask = dist_to_center <= rad_limit
sources = sources[mask]
else:
x_lims = [int(0.05 * xsize), int(0.95 * xsize)]
y_lims = [int(0.05 * ysize), int(0.95 * ysize)]
mask = (sources['x'] > x_lims[0]) & (sources['x'] < x_lims[1]) & (
sources['y'] > y_lims[0]) & (sources['y'] < y_lims[1])
sources = sources[mask]
## empirically obtain the effective Point Spread Function (ePSF)
nddata = NDData(image_data) # NDData object
if mask_file: # supply a mask if needed
nddata.mask = fits.getdata(mask_file)
if cutout % 2 == 0: # if cutout even, subtract 1
cutout -= 1
stars = extract_stars(nddata, sources, size=cutout) # extract stars
## use only the stars with fluxes between two percentiles if using
## astrometry.net
if image_source_file:
stars_tab = Table() # temporary table
stars_col = Column(data=range(len(stars.all_stars)), name="stars")
stars_tab["stars"] = stars_col # column of indices of each star
fluxes = [s.flux for s in stars]
fluxes_col = Column(data=fluxes, name="flux")
stars_tab["flux"] = fluxes_col # column of fluxes
# get percentiles
per_low = np.percentile(fluxes, lowper * 100) # lower percentile flux
per_high = np.percentile(fluxes,
highper * 100) # upper percentile flux
mask = (stars_tab["flux"] >= per_low) & (stars_tab["flux"] <= per_high)
stars_tab = stars_tab[mask] # include only stars between these fluxes
idx_stars = (stars_tab["stars"]).data # indices of these stars
# update stars object
# have to manually update all_stars AND _data attributes
stars.all_stars = [stars[i] for i in idx_stars]
stars._data = stars.all_stars
## build the ePSF
nstars_epsf = len(stars.all_stars) # no. of stars used in ePSF building
if nstars_epsf == 0:
print(
"\nNo valid sources were found to build the ePSF with the given" +
" conditions. Exiting.")
return
if verbose:
print(f"{nstars_epsf} stars used in building the ePSF")
epsf_builder = EPSFBuilder(
oversampling=1,
maxiters=7, # build it
progress_bar=False)
epsf, fitted_stars = epsf_builder(stars)
epsf_data = epsf.data
if write: # write, if desired
epsf_hdu = fits.PrimaryHDU(data=epsf_data)
if not (output):
output = image_file.replace(".fits", "_ePSF.fits")
epsf_hdu.writeto(output, overwrite=True, output_verify="ignore")
if plot: # plot, if desired
if not (output_plot): # set output name if not given
output_plot = image_file.replace(".fits", "_ePSF.png")
__plot_ePSF(epsf_data=epsf_data, output=output_plot)
return epsf_data
def epsf_phot(target, centroided_sources, plots=False):
def hmsm_to_days(hour=0,min=0,sec=0,micro=0):
"""
Convert hours, minutes, seconds, and microseconds to fractional days.
"""
days = sec + (micro / 1.e6)
days = min + (days / 60.)
days = hour + (days / 60.)
return days / 24.
def date_to_jd(year,month,day):
"""
Convert a date to Julian Day.
Algorithm from 'Practical Astronomy with your Calculator or Spreadsheet',
4th ed., Duffet-Smith and Zwart, 2011.
"""
if month == 1 or month == 2:
yearp = year - 1
monthp = month + 12
else:
yearp = year
monthp = month
# this checks where we are in relation to October 15, 1582, the beginning
# of the Gregorian calendar.
if ((year < 1582) or
(year == 1582 and month < 10) or
(year == 1582 and month == 10 and day < 15)):
# before start of Gregorian calendar
B = 0
else:
# after start of Gregorian calendar
A = math.trunc(yearp / 100.)
B = 2 - A + math.trunc(A / 4.)
if yearp < 0:
C = math.trunc((365.25 * yearp) - 0.75)
else:
C = math.trunc(365.25 * yearp)
D = math.trunc(30.6001 * (monthp + 1))
jd = B + C + D + day + 1720994.5
return jd
pines_path = pines_dir_check()
short_name = short_name_creator(target)
reduced_path = pines_path/('Objects/'+short_name+'/reduced/')
reduced_filenames = natsort.natsorted([x.name for x in reduced_path.glob('*.fits')])
reduced_files = np.array([reduced_path/i for i in reduced_filenames])
centroided_sources.columns = centroided_sources.columns.str.strip()
source_names = natsort.natsorted(list(set([i.split(' ')[0]+' '+i.split(' ')[1] for i in centroided_sources.keys() if (i[0] == '2') or (i[0] == 'R')])))
#Create output plot directories for each source.
if plots:
for name in source_names:
#If the folders are already there, delete them.
source_path = (pines_path/('Objects/'+short_name+'/psf_phot/'+name+'/'))
if source_path.exists():
shutil.rmtree(source_path)
#Create folders.
os.mkdir(source_path)
#Declare a new dataframe to hold the information for all targets for this .
columns = ['Filename', 'Time UT', 'Time JD', 'Airmass', 'Seeing']
for i in range(0, len(source_names)):
columns.append(source_names[i]+' Flux')
columns.append(source_names[i]+' Flux Error')
psf_df = pd.DataFrame(index=range(len(reduced_files)), columns=columns)
output_filename = pines_path/('Objects/'+short_name+'/psf_phot/'+short_name+'_psf_phot.csv')
for i in range(0, len(reduced_files)):
#Read in image data/header.
file = reduced_files[i]
data = fits.open(file)[0].data
header = fits.open(file)[0].header
print('{}, image {} of {}.'.format(file.name, i+1, len(reduced_files)))
#Read in some supporting information.
log_path = pines_path/('Logs/'+file.name.split('.')[0]+'_log.txt')
log = pines_log_reader(log_path)
date_obs = header['DATE-OBS']
#Catch a case that can cause datetime strptime to crash; Mimir headers sometimes have DATE-OBS with seconds specified as 010.xx seconds, when it should be 10.xx seconds.
if len(date_obs.split(':')[-1].split('.')[0]) == 3:
date_obs = date_obs.split(':')[0] + ':' + date_obs.split(':')[1] + ':' + date_obs.split(':')[-1][1:]
#Keep a try/except clause here in case other unknown DATE-OBS formats pop up.
try:
date = datetime.datetime.strptime(date_obs, '%Y-%m-%dT%H:%M:%S.%f')
except:
print('Header DATE-OBS format does not match the format code in strptime! Inspect/correct the DATE-OBS value.')
pdb.set_trace()
days = date.day + hmsm_to_days(date.hour,date.minute,date.second,date.microsecond)
jd = date_to_jd(date.year,date.month,days)
psf_df['Filename'][i] = file.name
psf_df['Time UT'][i] = header['DATE-OBS']
psf_df['Time JD'][i] = jd
psf_df['Airmass'][i] = header['AIRMASS']
psf_df['Seeing'][i] = log['X seeing'][np.where(log['Filename'] == file.name.split('_')[0]+'.fits')[0][0]]
#Read in source centroids for this image
x = np.zeros(len(source_names))
y = np.zeros(len(source_names))
for j in range(len(source_names)):
source = source_names[j]
x[j] = centroided_sources[source+' X'][i]
y[j] = centroided_sources[source+' Y'][i]
#Extract pixel cutouts of our stars, so let’s explicitly exclude stars that are too close to the image boundaries (because they cannot be extracted).
size = 13
hsize = (size - 1) / 2
#mask = ((x > hsize) & (x < (data.shape[1] -1 - hsize)) & (y > hsize) & (y < (data.shape[0] -1 - hsize)) & (y > 100) & (y < 923))
#Create table of good star positions
stars_tbl = Table()
stars_tbl['x'] = x
stars_tbl['y'] = y
#Subtract background (star cutouts from which we build the ePSF must have background subtracted).
mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=2.)
data -= median_val
#Replace nans in data using Gaussian.
# kernel = Gaussian2DKernel(x_stddev=0.5)
# data = interpolate_replace_nans(data, kernel)
#The extract_stars() function requires the input data as an NDData object.
nddata = NDData(data=data)
#Extract star cutouts.
stars = extract_stars(nddata, stars_tbl, size=size)
#Plot.
nrows = 5
ncols = 5
fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(10, 10), squeeze=True)
ax = ax.ravel()
for j in range(len(stars)):
norm = simple_norm(stars[j], 'log', percent=99.)
ax[j].imshow(stars[j].data, norm=norm, origin='lower', cmap='viridis')
pdb.set_trace()
#Construct the ePSF using the star cutouts.
epsf_fitter = EPSFFitter()
epsf_builder = EPSFBuilder(maxiters=4, progress_bar=False, fitter=epsf_fitter)
try:
epsf, fitted_stars = epsf_builder(stars)
output_filename = pines_path/('Objects/'+short_name+'/psf_phot/'+short_name+'_psf_phot.csv')
for j in range(len(stars)):
star = stars[j]
source_name = source_names[j]
sigma_psf = 1.85
dtype = [('x_0', 'f8'), ('y_0', 'f8')]
pos = Table(data=np.zeros(1, dtype=dtype))
source_x = stars_tbl['x'][j]
source_y = stars_tbl['y'][j]
pos['x_0'] = source_x - int(source_x - size/2 + 1)
pos['y_0'] = source_y - int(source_y - size/2 + 1)
daogroup = DAOGroup(4.0*sigma_psf*gaussian_sigma_to_fwhm)
mmm_bkg = MMMBackground()
photometry = BasicPSFPhotometry(group_maker=daogroup,
bkg_estimator=mmm_bkg,
psf_model=epsf,
fitter=LevMarLSQFitter(),
fitshape=(13,13),
aperture_radius=4.)
result_tab = photometry(image=star, init_guesses=pos)
residual_image = photometry.get_residual_image()
psf_df[source_name+' Flux'][i] = result_tab['flux_fit'][0]
psf_df[source_name+' Flux Error'][i] = result_tab['flux_unc'][0]
if plots:
fig, ax = plt.subplots(nrows=1, ncols=3, figsize=(12,4))
im = ax[0].imshow(star, origin='lower')
divider = make_axes_locatable(ax[0])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im, cax=cax, orientation='vertical')
ax[0].plot(result_tab['x_fit'][0], result_tab['y_fit'][0], 'rx')
ax[0].set_title('Data')
im2 = ax[1].imshow(epsf.data, origin='lower')
ax[1].set_title('EPSF Model')
divider = make_axes_locatable(ax[1])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im2, cax=cax, orientation='vertical')
im3 = ax[2].imshow(residual_image, origin='lower')
ax[2].set_title('Residual Image')
divider = make_axes_locatable(ax[2])
cax = divider.append_axes('right', size='5%', pad=0.05)
fig.colorbar(im3, cax=cax, orientation='vertical')
plt.suptitle(source_name+'\n'+reduced_files[i].name+', image '+str(i+1)+' of '+str(len(reduced_files)))
plt.subplots_adjust(wspace=0.5, top=0.95, bottom = 0.05)
plot_output_name = pines_path/('Objects/'+short_name+'/psf_phot/'+source_name+'/'+str(i).zfill(4)+'.jpg')
plt.savefig(plot_output_name)
plt.close()
except:
print('')
print('EPSF BUILDER FAILED, SKIPPING IMAGE.')
print('')
#Plot the ePSF.
# plt.figure()
# norm = simple_norm(epsf.data, 'log', percent=99.)
# plt.imshow(epsf.data, norm=norm, origin='lower', cmap='viridis')
# cb = plt.colorbar()
# plt.tight_layout()
print('Saving psf photometry output to {}.'.format(output_filename))
with open(output_filename, 'w') as f:
for j in range(len(psf_df)):
if j == 0:
f.write('{:>21s}, {:>22s}, {:>17s}, {:>7s}, {:>7s}, '.format('Filename', 'Time UT', 'Time JD', 'Airmass', 'Seeing'))
for i in range(len(source_names)):
if i != len(source_names) - 1:
f.write('{:>20s}, {:>26s}, '.format(source_names[i]+' Flux', source_names[i]+' Flux Error'))
else:
f.write('{:>20s}, {:>26s}\n'.format(source_names[i]+' Flux', source_names[i]+' Flux Error'))
format_string = '{:21s}, {:22s}, {:17.9f}, {:7.2f}, {:7.1f}, '
#If the seeing value for this image is 'nan' (a string), convert it to a float.
#TODO: Not sure why it's being read in as a string, fix that.
if type(psf_df['Seeing'][j]) == str:
psf_df['Seeing'][j] = float(psf_df['Seeing'][j])
#Do a try/except clause for writeout, in case it breaks in the future.
try:
f.write(format_string.format(psf_df['Filename'][j], psf_df['Time UT'][j], psf_df['Time JD'][j], psf_df['Airmass'][j], psf_df['Seeing'][j]))
except:
print('Writeout failed! Inspect quantities you are trying to write out.')
pdb.set_trace()
for i in range(len(source_names)):
if i != len(source_names) - 1:
format_string = '{:20.11f}, {:26.11f}, '
else:
format_string = '{:20.11f}, {:26.11f}\n'
f.write(format_string.format(psf_df[source_names[i]+' Flux'][j], psf_df[source_names[i]+' Flux Error'][j]))
print('')
return
def build_ePSF_imsegm(image_file,
mask_file=None,
nstars=40,
thresh_sigma=5.0,
pixelmin=20,
etamax=1.4,
areamax=500,
cutout=35,
write=True,
output=None,
plot=False,
output_plot=None,
verbose=False):
"""Build the effective Point-Spread Function using a sample of stars from
some image acquired via image segmentation.
Arguments
---------
image_file : str
Filename for a **background-subtracted** image
mask_file : str, optional
Filename for a mask file (default None)
nstars : int, optional
*Maximum* number of stars to use in building the ePSF (default 40;
set to None to impose no limit)
thresh_sigma : float, optional
Sigma threshold for source detection with image segmentation (default
5.0)
pixelmin : float, optional
*Minimum* pixel area of an isophote to be considered a good source for
building the ePSF (default 20)
etamax : float, optional
*Maximum* allowed elongation for an isophote to be considered a good
source for building the ePSF (default 1.4)
areamax : float, optional
*Maximum* allowed area (in square pixels) for an isophote to be
considered a good source for building the ePSF (default 500)
cutout : int, optional
Cutout size around each star in pixels (default 35; must be **odd**;
rounded **down** if even)
write : bool, optional
Whether to write the ePSF to a new fits file (default True)
output : str, optional
Name for the output ePSF data fits file (default
`image_file.replace(".fits", "_ePSF.fits")`)
plot : bool, optional
Whether to plot the newly-built ePSF (default False)
output_plot : str, optional
Name for the output figure (default
`image_file.replace(".fits", "_ePSF.png")`)
verbose : bool, optional
Whether to be verbose (default False)
Returns
-------
np.ndarray
The ePSF data in a 2D array
Notes
-----
Uses image segmentation via `photutils` to obtain a list of sources in the
image with their x, y coordinates, flux, and background at their
location. Then uses `EPSFBuilder` to empirically obtain the ePSF of these
stars. Optionally writes and/or plots the obtained ePSF.
**The ePSF obtained here should not be used in convolutions.** Instead, it
can serve as a tool for estimating the seeing of an image.
"""
# ignore annoying warnings from photutils
from astropy.utils.exceptions import AstropyWarning
warnings.simplefilter('ignore', category=AstropyWarning)
# imports
from astropy.nddata import NDData
from photutils.psf import extract_stars
from photutils import EPSFBuilder
# load in data
image_data = fits.getdata(image_file)
image_header = fits.getheader(image_file)
try:
instrument = image_header["INSTRUME"]
except KeyError:
instrument = "Unknown"
## source detection
# add mask to image_data
image_data = np.ma.masked_where(image_data == 0.0, image_data)
# build an actual mask
mask = (image_data == 0)
if mask_file:
mask = np.logical_or(mask, fits.getdata(mask_file))
# set detection standard deviation
try:
std = image_header["BKGSTD"] # header written by bkgsub function
except KeyError:
# make crude source mask, get standard deviation of background
source_mask = make_source_mask(image_data,
snr=3,
npixels=5,
dilate_size=15,
mask=mask)
final_mask = np.logical_or(mask, source_mask)
std = np.std(np.ma.masked_where(final_mask, image_data))
# use the segmentation image to get the source properties
segm = detect_sources(image_data,
thresh_sigma * std,
npixels=pixelmin,
mask=mask)
cat = source_properties(image_data, segm, mask=mask)
## get the catalog and coordinate/fluxes for sources, do some filtering
try:
tbl = cat.to_table()
except ValueError:
print("SourceCatalog contains no sources. Exiting.")
return
# restrict elongation and area to obtain only unsaturated stars
tbl = tbl[(tbl["elongation"] <= etamax)]
tbl = tbl[(tbl["area"].value <= areamax)]
# build a table
sources = Table() # build a table
sources['x'] = tbl['xcentroid'] # for EPSFBuilder
sources['y'] = tbl['ycentroid']
sources['flux'] = tbl['source_sum'].data / tbl["area"].data
sources.sort("flux")
sources.reverse()
# restrict number of stars (if requested)
if nstars: sources = sources[:min(nstars, len(sources))]
## get WCS coords for all sources
w = wcs.WCS(image_header)
sources["ra"], sources["dec"] = w.all_pix2world(sources["x"], sources["y"],
1)
## mask out edge sources:
# a bounding circle for WIRCam, rectangle for MegaPrime
xsize = image_data.shape[1]
ysize = image_data.shape[0]
if "WIRCam" in instrument: # bounding circle
rad_limit = xsize / 2.0
dist_to_center = np.sqrt((sources['x'] - xsize / 2.0)**2 +
(sources['y'] - ysize / 2.0)**2)
mask = dist_to_center <= rad_limit
sources = sources[mask]
else: # rectangle
x_lims = [int(0.05 * xsize), int(0.95 * xsize)]
y_lims = [int(0.05 * ysize), int(0.95 * ysize)]
mask = (sources['x'] > x_lims[0]) & (sources['x'] < x_lims[1]) & (
sources['y'] > y_lims[0]) & (sources['y'] < y_lims[1])
sources = sources[mask]
## empirically obtain the effective Point Spread Function (ePSF)
nddata = NDData(image_data) # NDData object
if mask_file: # supply a mask if needed
nddata.mask = fits.getdata(mask_file)
if cutout % 2 == 0: # if cutout even, subtract 1
cutout -= 1
stars = extract_stars(nddata, sources, size=cutout) # extract stars
## build the ePSF
nstars_epsf = len(stars.all_stars) # no. of stars used in ePSF building
if nstars_epsf == 0:
print(
"\nNo valid sources were found to build the ePSF with the given" +
" conditions. Exiting.")
return
if verbose:
print(f"{nstars_epsf} stars used in building the ePSF")
epsf_builder = EPSFBuilder(
oversampling=1,
maxiters=7, # build it
progress_bar=False)
epsf, fitted_stars = epsf_builder(stars)
epsf_data = epsf.data
if write: # write, if desired
epsf_hdu = fits.PrimaryHDU(data=epsf_data)
if not (output):
output = image_file.replace(".fits", "_ePSF.fits")
epsf_hdu.writeto(output, overwrite=True, output_verify="ignore")
if plot: # plot, if desired
if not (output_plot): # set output name if not given
output_plot = image_file.replace(".fits", "_ePSF.png")
__plot_ePSF(epsf_data=epsf_data, output=output_plot)
return epsf_data
def __fit_PSF(image_file, mask_file=None, nstars=40,
thresh_sigma=5.0, pixelmin=20, elongation_lim=1.4, area_max=500,
cutout=35,
astrom_sigma=5.0, psf_sigma=5.0, alim=10000, clean=True,
source_lim=None,
write_ePSF=False, ePSF_output=None,
plot_ePSF=True, ePSF_plotname=None,
plot_residuals=False, resid_plotname=None,
verbose=False):
"""
Input:
general:
- filename for a **BACKGROUND-SUBTRACTED** image
- filename for a mask image (optional; default None)
- maximum number of stars to use (optional; default 40; set to None
to impose no limit)
source detection:
- sigma threshold for source detection with image segmentation
(optional; default 5.0)
- *minimum* number of isophotal pixels (optional; default 20)
- *maximum* allowed elongation for sources found by image segmentation
(optional; default 1.4)
- *maximum* allowed area for sources found by image segmentation
(optional; default 500 pix**2)
- cutout size around each star in pix (optional; default 35 pix; must
be ODD, rounded down if even)
astrometry.net:
- sigma threshold for astrometry.net source detection image (optional;
default 5.0)
- sigma of the Gaussian PSF of the image (optional; default 5.0)
- maximum allowed source area in pix**2 for astrometry.net for
deblending (optional; default 10000; only relevant if no source list
file is provided)
- whether to remove files output by image2xy once finished with them
(optional; default True)
misc:
- limit on number of sources to fit with ePSF (optional; default None
which imposes no limit)
writing, plotting, verbosity:
- whether to write the derived ePSF to a fits file (optional; default
False)
- name for output ePSF fits file (optional; default set below)
- whether to plot the derived ePSF (optional; default True)
- name for output ePSF plot (optional; default set below)
- whether to plot the residuals of the iterative PSF fitting (optional;
default False)
- name for output residuals plot (optional; default set below)
- be verbose (optional; default False)
Uses image segmentation to obtain a list of sources in the image with their
x, y coordinates. Uses EPSFBuilder to empirically obtain the ePSF of these
stars. Optionally writes and/or plots the obtaind ePSF. Finally, uses
astrometry.net to find all sources in the image, and fits them with the
empirically obtained ePSF.
The ePSF obtained here should NOT be used in convolutions. Instead, it can
serve as a tool for estimating the seeing of an image.
Output: table containing the coordinates and instrumental magnitudes of the
detected, ePSF-fit sources
"""
# load in data
image_data = fits.getdata(image_file)
image_header = fits.getheader(image_file)
try:
instrument = image_header["INSTRUME"]
except KeyError:
instrument = "Unknown"
pixscale = image_header["PIXSCAL1"]
### SOURCE DETECTION
### use image segmentation to find sources with an area > pixelmin pix**2
### which are above the threshold sigma*std
image_data = fits.getdata(image_file) # subfile data
image_data = np.ma.masked_where(image_data==0.0,
image_data) # mask bad pixels
## build an actual mask
mask = (image_data==0)
if mask_file:
mask = np.logical_or(mask, fits.getdata(mask_file))
## set detection standard deviation
try:
std = image_header["BKGSTD"] # header written by amakihi.bkgsub fn
except KeyError:
# make crude source mask, get standard deviation of background
source_mask = make_source_mask(image_data, snr=3, npixels=5,
dilate_size=15, mask=mask)
final_mask = np.logical_or(mask, source_mask)
std = np.std(np.ma.masked_where(final_mask, image_data))
## use the segmentation image to get the source properties
# use <mask>, which does not mask sources
segm = detect_sources(image_data, thresh_sigma*std, npixels=pixelmin,
mask=mask)
cat = source_properties(image_data, segm, mask=mask)
## get the catalog and coordinates for sources
try:
tbl = cat.to_table()
except ValueError:
print("SourceCatalog contains no sources. Exiting.")
return
# restrict elongation and area to obtain only unsaturated stars
tbl = tbl[(tbl["elongation"] <= elongation_lim)]
tbl = tbl[(tbl["area"].value <= area_max)]
sources = Table() # build a table
sources['x'] = tbl['xcentroid'] # for EPSFBuilder
sources['y'] = tbl['ycentroid']
sources['flux'] = tbl['source_sum'].data/tbl["area"].data
sources.sort("flux")
sources.reverse()
if nstars:
sources = sources[:min(nstars, len(sources))]
## setup: get WCS coords for all sources
w = wcs.WCS(image_header)
sources["ra"], sources["dec"] = w.all_pix2world(sources["x"],
sources["y"], 1)
## mask out edge sources:
# a bounding circle for WIRCam, rectangle for MegaPrime
xsize = image_data.shape[1]
ysize = image_data.shape[0]
if "WIRCam" in instrument:
rad_limit = xsize/2.0
dist_to_center = np.sqrt((sources['x']-xsize/2.0)**2 +
(sources['y']-ysize/2.0)**2)
dmask = dist_to_center <= rad_limit
sources = sources[dmask]
else:
x_lims = [int(0.05*xsize), int(0.95*xsize)]
y_lims = [int(0.05*ysize), int(0.95*ysize)]
dmask = (sources['x']>x_lims[0]) & (sources['x']<x_lims[1]) & (
sources['y']>y_lims[0]) & (sources['y']<y_lims[1])
sources = sources[dmask]
## empirically obtain the effective Point Spread Function (ePSF)
nddata = NDData(image_data) # NDData object
if mask_file: # supply a mask if needed
nddata.mask = fits.getdata(mask_file)
if cutout%2 == 0: # if cutout even, subtract 1
cutout -= 1
stars = extract_stars(nddata, sources, size=cutout) # extract stars
## build the ePSF
nstars_epsf = len(stars.all_stars) # no. of stars used in ePSF building
if nstars_epsf == 0:
print("\nNo valid sources were found to build the ePSF with the given"+
" conditions. Exiting.")
return
if verbose:
print(f"\n{nstars_epsf} stars used in building the ePSF")
start = timer()
epsf_builder = EPSFBuilder(oversampling=1, maxiters=7, # build it
progress_bar=False)
epsf, fitted_stars = epsf_builder(stars)
epsf_data = epsf.data
end = timer() # timing
time_elaps = end-start
# print ePSF FWHM, if desired
print(f"Time required for ePSF building {time_elaps:.2f} s\n")
if verbose:
ePSF_FWHM(epsf_data, True)
epsf_hdu = fits.PrimaryHDU(data=epsf_data)
if write_ePSF: # write, if desired
if not(ePSF_output):
ePSF_output = image_file.replace(".fits", "_ePSF.fits")
epsf_hdu.writeto(ePSF_output, overwrite=True, output_verify="ignore")
psf_model = epsf # set the model
psf_model.x_0.fixed = True # fix centroids (known beforehand)
psf_model.y_0.fixed = True
### USE ASTROMETRY.NET TO FIND SOURCES TO FIT
# -b --> no background-subtraction
# -O --> overwrite
# -p <astrom_sigma> --> signficance
# -w <psf_sigma> --> estimated PSF sigma
# -m <alim> --> max object size for deblending is <alim>
options = f"-O -b -p {astrom_sigma} -w {psf_sigma}"
options += f" -m {alim}"
run(f"image2xy {options} {image_file}", shell=True)
image_sources_file = image_file.replace(".fits", ".xy.fits")
image_sources = fits.getdata(image_sources_file)
if clean:
run(f"rm {image_sources_file}", shell=True) # this file is not needed
print(f'\n{len(image_sources)} stars at >{astrom_sigma}'+
f' sigma found in image {re.sub(".*/", "", image_file)}'+
' with astrometry.net')
astrom_sources = Table() # build a table
astrom_sources['x_mean'] = image_sources['X'] # for BasicPSFPhotometry
astrom_sources['y_mean'] = image_sources['Y']
astrom_sources['flux'] = image_sources['FLUX']
# initial guesses for centroids, fluxes
pos = Table(names=['x_0', 'y_0','flux_0'],
data=[astrom_sources['x_mean'], astrom_sources['y_mean'],
astrom_sources['flux']])
### FIT THE ePSF TO ALL DETECTED SOURCES
start = timer() # timing the fit
# sources separated by less than this critical separation are grouped
# together when fitting the PSF via the DAOGROUP algorithm
sigma_psf = 2.0 # 2 pix
crit_sep = 2.0*sigma_psf*gaussian_sigma_to_fwhm # twice the PSF FWHM
daogroup = DAOGroup(crit_sep)
# an astropy fitter, does Levenberg-Marquardt least-squares fitting
fitter_tool = LevMarLSQFitter()
# if we have a limit on the number of sources to fit
if source_lim:
try:
import random # pick a given no. of random sources
source_rows = random.choices(astrom_sources, k=source_lim)
astrom_sources = Table(names=['x_mean', 'y_mean', 'flux'],
rows=source_rows)
pos = Table(names=['x_0', 'y_0','flux_0'],
data=[astrom_sources['x_mean'],
astrom_sources['y_mean'],
astrom_sources['flux']])
except IndexError:
print("The input source limit exceeds the number of sources"+
" detected by astrometry, so no limit is imposed.\n")
photometry = BasicPSFPhotometry(group_maker=daogroup,
bkg_estimator=None, # bg subtract already done
psf_model=psf_model,
fitter=fitter_tool,
fitshape=(11,11))
result_tab = photometry(image=image_data, init_guesses=pos) # results
residual_image = photometry.get_residual_image() # residuals of PSF fit
residual_image = np.ma.masked_where(mask, residual_image)
residual_image.fill_value = 0 # set to zero
residual_image = residual_image.filled()
end = timer() # timing
time_elaps = end - start
print(f"Time required fit ePSF to all sources {time_elaps:.2f} s\n")
# include WCS coordinates
pos["ra"], pos["dec"] = w.all_pix2world(pos["x_0"], pos["y_0"], 1)
result_tab.add_column(pos['ra'])
result_tab.add_column(pos['dec'])
# mask out negative flux_fit values in the results
mask_flux = (result_tab['flux_fit'] >= 0.0)
psf_sources = result_tab[mask_flux] # PSF-fit sources
# compute magnitudes and their errors and add to the table
# error = (2.5/(ln(10)*flux_fit))*flux_unc
mag_fit = -2.5*np.log10(psf_sources['flux_fit']) # instrumental mags
mag_fit.name = 'mag_fit'
mag_unc = 2.5/(psf_sources['flux_fit']*np.log(10))
mag_unc *= psf_sources['flux_unc']
mag_unc.name = 'mag_unc'
psf_sources['mag_fit'] = mag_fit
psf_sources['mag_unc'] = mag_unc
# mask entries with large magnitude uncertainties
mask_unc = psf_sources['mag_unc'] < 0.4
psf_sources = psf_sources[mask_unc]
if plot_ePSF: # if we wish to see the ePSF
plt.figure(figsize=(10,9))
plt.imshow(epsf_data, origin='lower', aspect=1, cmap='magma',
interpolation="nearest")
plt.xlabel("Pixels", fontsize=16)
plt.ylabel("Pixels", fontsize=16)
plt.title("Effective Point-Spread Function (1 pixel = "
+str(pixscale)+
'")', fontsize=16)
plt.colorbar(orientation="vertical", fraction=0.046, pad=0.08)
plt.rc("xtick",labelsize=16) # not working?
plt.rc("ytick",labelsize=16)
if not(ePSF_plotname):
ePSF_plotname = image_file.replace(".fits", "_ePSF.png")
plt.savefig(ePSF_plotname, bbox_inches="tight")
plt.close()
if plot_residuals: # if we wish to see a plot of the residuals
if "WIRCam" in instrument:
plt.figure(figsize=(10,9))
else:
plt.figure(figsize=(12,14))
ax = plt.subplot(projection=w)
plt.imshow(residual_image, cmap='magma', aspect=1,
interpolation='nearest', origin='lower')
plt.xlabel("RA (J2000)", fontsize=16)
plt.ylabel("Dec (J2000)", fontsize=16)
plt.title("PSF residuals", fontsize=16)
cb = plt.colorbar(orientation='vertical', fraction=0.046, pad=0.08)
cb.set_label(label="ADU", fontsize=16)
ax.coords["ra"].set_ticklabel(size=15)
ax.coords["dec"].set_ticklabel(size=15)
if not(resid_plotname):
resid_plotname = image_file.replace(".fits", "_ePSFresiduals.png")
plt.savefig(resid_plotname, bbox_inches="tight")
plt.close()
return psf_sources
def generate_PSFs(self,
equivalent_radius=2.,
size=20.,
oversampling=1,
plot=None,
filepaths=None):
'''
Generate effective point spread fuctions (ePSFs) for each image
Parameters
----------
equivalent_radius : float, unit arcsec
radius criteria to indentify star
size : float, unit pixel
use what size box to extract stars
oversampling : int
oversample the ePSF
plot : None for not plot stars & ePSF
list like [1,2,3] to plot rgb image
filepaths : filepath to store the ePSFs
'''
stars = self.common_catalog.copy()
remolist = []
for loop in range(len(stars)):
for loop2 in range(self.__length):
a = (
self.image_list[loop2].sources_catalog['equivalent_radius']
[stars['sloop_{0}'.format(loop2 + 1)][loop]]
) * self.image_list[loop2].pixel_scales[0].value
if (a > equivalent_radius):
remolist.append(loop)
break
stars.remove_rows(remolist)
star_images = []
PSFs = []
for loop2 in range(self.__length):
newsc = self.image_list[loop2].sources_catalog.copy()
indexes = np.delete(
np.arange(len(self.image_list[loop2].sources_catalog)),
stars['sloop_{0}'.format(loop2 + 1)])
newsc.remove_rows(indexes)
stars_tbl = Table()
stars_tbl['x'] = np.array(newsc['maxval_xpos'])
stars_tbl['y'] = np.array(newsc['maxval_ypos'])
nddata = NDData(data=np.array(self.image_list[loop2].ss_data))
Tstar = extract_stars(nddata, stars_tbl, size=size)
epsf_builder = EPSFBuilder(oversampling=oversampling,
maxiters=15,
progress_bar=False)
epsf, fitted_stars = epsf_builder(Tstar)
self.image_list[loop2].PSF = epsf.data
if filepaths is not None:
hdu = fits.PrimaryHDU(epsf.data.astype('float32'))
After = fits.HDUList([hdu])
After.writeto(filepaths[loop2], overwrite=True)
if plot is not None:
star_images.append(Tstar)
PSFs.append(epsf.data)
if plot is not None:
tlens = len(stars)
if (((tlens // 5) + 1) * 5 - tlens) < ((
(tlens // 4) + 1) * 4 - tlens):
ncols = 5
nrows = (tlens // 5) + 1
else:
ncols = 4
nrows = (tlens // 4) + 1
fig, ax = plt.subplots(nrows=nrows,
ncols=ncols,
figsize=(3 * ncols, 3 * nrows),
squeeze=True)
ax = ax.ravel()
for i in range(tlens):
if len(plot) > 2:
star_b = star_images[plot[0]][i].data * 100. / np.sum(
star_images[plot[0]][i].data)
star_g = star_images[plot[1]][i].data * 100. / np.sum(
star_images[plot[1]][i].data)
star_r = star_images[plot[2]][i].data * 100. / np.sum(
star_images[plot[2]][i].data)
norm = simple_norm(star_b, 'log', percent=99.)
image = make_lupton_rgb(star_r, star_g, star_b, Q=10)
else:
image = star_images[plot[0]][i].data
norm = simple_norm(image, 'log', percent=99.)
ax[i].imshow(image, norm=norm, origin='lower')
plt.show()
fig = plt.figure(figsize=(10, 10))
if len(plot) > 2:
star_b = PSFs[plot[0]] * 100. / np.sum(PSFs[plot[0]])
star_g = PSFs[plot[1]] * 100. / np.sum(PSFs[plot[1]])
star_r = PSFs[plot[2]] * 100. / np.sum(PSFs[plot[2]])
norm = simple_norm(star_b, 'log', percent=99.)
image = make_lupton_rgb(star_r, star_g, star_b, Q=10)
else:
image = PSFs[plot[0]]
norm = simple_norm(image, 'log', percent=99.)
plt.imshow(image, norm=norm, origin='lower')
plt.show()
data += make_noise_image(data.shape, type='gaussian', mean=10.,
stddev=5., random_state=12345)
from photutils import find_peaks
peaks_tbl = find_peaks(data, threshold=500.)
from astropy.table import Table
stars_tbl = Table()
stars_tbl['x'] = peaks_tbl['x_peak']
stars_tbl['y'] = peaks_tbl['y_peak']
from astropy.stats import sigma_clipped_stats
mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=2.,
iters=None)
data -= median_val
from astropy.nddata import NDData
nddata = NDData(data=data)
from photutils.psf import extract_stars
stars = extract_stars(nddata, stars_tbl, size=25)
import matplotlib.pyplot as plt
nrows = 5
ncols = 5
fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(20, 20),
squeeze=True)
ax = ax.ravel()
for i in range(nrows*ncols):
norm = simple_norm(stars[i], 'log', percent=99.)
ax[i].imshow(stars[i], norm=norm, origin='lower', cmap='viridis')
def find_pinholes_irregular(fname,
freference,
sname,
fdarkff,
fdark,
fff,
files,
size,
threshold,
fwhm,
fitshape,
MAX_CONTROL_POINTS,
PIXEL_TOL,
range_psf,
sigma=2.,
oversampling=4,
maxiters=3,
MIN_MATCHES_FRACTION=0.8,
NUM_NEAREST_NEIGHBORS=5):
"""Finds and fits irregularly spread pinhole positions with a ePSF in a FITS image. Then matches them to the reference positions.
Parameters
----------
fname : str
Folder name of the input fits files.
freference : str
File name of the reference positions (txt file).
sname : str
Folder name of the returned found and matched pinhole positions (txt files).
fdarkff : string
Location of the dark images for the flat field images.
fdark : string
Location of the dark images for the raw images.
fff : string
Location of the flat field images.
files : (1, 2)-shaped int array
File range to create a median image
size : int
Rectangular size of the ePSF. Size must be an odd number.
threshold : float
The absolute image value above which to select sources.
fwhm : float
The full-width half-maximum (FWHM) of the major axis of the Gaussian kernel in units of pixels.
fitshape : int or length-2 array-like
Rectangular shape around the center of a star which will be used to collect the data to do the fitting.
Can be an integer to be the same along both axes.
E.g., 5 is the same as (5, 5), which means to fit only at the following
relative pixel positions: [-2, -1, 0, 1, 2]. Each element of fitshape must be an odd number.
MAX_CONTROL_POINTS : int
The maximum control points (stars) to use to build the invariants.
PIXEL_TOL : int
The pixel distance tolerance to assume two invariant points are the same.
range_psf : (1, 4)-shaped int array
Position range to compute epsf [xmin,xmax,ymin,ymax]
sigma : float
Number of standard deviations used to perform sigma clip with a astropy.stats.SigmaClip object.
oversampling : int or tuple of two int
The oversampling factor(s) of the ePSF relative to the input stars along the x and y axes.
The oversampling can either be a single float or a tuple of two floats of the form (x_oversamp, y_oversamp).
If oversampling is a scalar then the oversampling will be the same for both the x and y axes.
maxiters : int
The maximum number of iterations to perform.
MIN_MATCHES_FRACTION : float (0,1]
The minimum fraction of triangle matches to accept a transformation.
If the minimum fraction yields more than 10 triangles, 10 is used instead.
NUM_NEAREST_NEIGHBORS : int
The number of nearest neighbors of a given star (including itself) to construct the triangle invariants.
Returns
-------
s_list : (N,2)-shaped array
Found and matched positions of the pinholes.
t_list : (N,2)-shaped array
Matched reference grid positions.
"""
#Load the sample of fits images
entries = os.listdir(fname)
data_col = np.array([fits.getdata(fname + '/' + entries[files[0]], ext=0)])
for k in range(files[0] + 1, files[1] + 1):
data_col = np.append(data_col,
[fits.getdata(fname + '/' + entries[k], ext=0)],
axis=0)
#Data reduction: Darc current + Flatfield
data_col = data_correction(data_col, fdarkff, fdark, fff)
#Claculate median image
data_full = np.median(data_col, axis=0)
pos_full = np.array([[0, 0]])
data = data_full[range_psf[2]:range_psf[3], range_psf[0]:range_psf[1]]
#Find peaks in data
peaks_tbl = find_peaks(data, threshold=threshold)
peaks_tbl['peak_value'].info.format = '%.8g'
#Load data around found peaks
hsize = (size - 1) / 2
x = peaks_tbl['x_peak']
y = peaks_tbl['y_peak']
mask = ((x > hsize) & (x < (data.shape[1] - 1 - hsize)) & (y > hsize) &
(y < (data.shape[0] - 1 - hsize)))
stars_tbl = Table()
stars_tbl['x'] = x[mask]
stars_tbl['y'] = y[mask]
#Calculate mean, median, std
mean_val, median_val, std_val = sigma_clipped_stats(data, sigma=sigma)
data = data - median_val
#Find pinholes and create ePSF
nddata = NDData(data=data)
stars = extract_stars(nddata, stars_tbl, size=size)
epsf_builder = EPSFBuilder(oversampling=oversampling,
maxiters=maxiters,
progress_bar=False)
epsf, fitted_stars = epsf_builder(stars)
#Use ePSF to find precise locations of pinholes
daofind = DAOPhotPSFPhotometry(crit_separation=30,
threshold=threshold,
fwhm=fwhm,
psf_model=epsf,
fitshape=fitshape,
aperture_radius=12,
niters=1)
#Get positions
sources = daofind(data_full)
for col in sources.colnames:
sources[col].info.format = '%.8g'
pos = np.transpose((sources['x_fit'], sources['y_fit']))
pos_full = np.append(pos_full, pos, axis=0)
pos_full = pos_full[1:]
#Plot found pinholes
apertures = CircularAperture(pos_full, r=10)
norm = ImageNormalize(stretch=SqrtStretch())
#Plot found pinholes
fig, ax = plt.subplots()
ax.set_title('Pinhole Positions')
ax.set(xlabel='x [pixel]', ylabel='y [pixel]')
ax.imshow(data_full, cmap='Greys', origin='lower', norm=norm)
apertures.plot(color='blue', lw=1.5, alpha=0.5)
ax.legend(['#pinholes = ' + str(len(pos_full[:, 0]))],
loc='lower left',
prop={'size': 12})
plt.show()
#Sort positions by matching with reference grid
positions_sort = pos_full
ref_positions = np.genfromtxt(freference, skip_header=0)
transf, (s_list, t_list) = find_transform(positions_sort, ref_positions,
MAX_CONTROL_POINTS, PIXEL_TOL,
MIN_MATCHES_FRACTION,
NUM_NEAREST_NEIGHBORS)
text = np.array([s_list[:, 0], s_list[:, 1], t_list[:, 0], t_list[:, 1]])
text_trans = np.zeros((len(s_list[:, 0]), 4))
#Transpose text matrix
for k in range(0, 4):
for l in range(0, len(s_list[:, 0])):
text_trans[l][k] = text[k][l]
#Save data as txt file
np.savetxt(sname + '.txt',
text_trans,
fmt='%1.9E',
delimiter='\t',
comments='',
header='x-measured y-measured x-reference y-reference')
return s_list, t_list
if not os.path.exists(epsf_file):
mean, median, std = sigma_clipped_stats(image)
threshold = median + 10 * std
finder = IRAFStarFinder(threshold=threshold, fwhm=4, minsep_fwhm=5, peakmax=image.max() / 0.8)
star_table = finder(image)
star_table.rename_columns(('xcentroid', 'ycentroid'),('x','y'))
sigma_clip = SigmaClip(sigma=5.0)
bkg_estimator = MMMBackground()
bkg = Background2D(image, 5, filter_size=(3, 3),
sigma_clip=sigma_clip, bkg_estimator=bkg_estimator)
nddata = NDData(image-bkg.background)
stars = extract_stars(nddata, star_table, size=51)
epsf, fitted_stars = EPSFBuilder(oversampling=4, maxiters=3, progress_bar=True, smoothing_kernel='quadratic')(stars)
epsf_model = prepare_psf_model(epsf, renormalize_psf=False)
with open(epsf_file,'wb') as f:
pickle.dump([epsf_model, finder], f)
else:
with open(epsf_file, 'rb') as f:
epsf_model, finder = pickle.load(f)
phot = IterativelySubtractedPSFPhotometry(group_maker=DAOGroup(5),
bkg_estimator=MMMBackground(),
psf_model=epsf_model,
fitshape=[31,31],
finder=finder,
