async def __call__(self, image: Image) -> Image:
"""Remove background from image.
Args:
image: Image to remove background from.
Returns:
Image without background.
"""
from photutils.background import Background2D, MedianBackground
# init objects
sigma_clip = SigmaClip(sigma=self.sigma)
bkg_estimator = MedianBackground()
# calculate background
bkg = Background2D(image.data,
self.box_size,
filter_size=self.filter_size,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
# copy image and remove background
img = image.copy()
img.data = img.data - bkg.background
return img
def run(self, image):
sigma_clip = SigmaClip(sigma=3.)
self.bkg = Background2D(image.data,
box_size=self.box_size,
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=self.bkg_estimator).background
if self.subtract:
image.bkg = self.bkg
image.data = image.data - self.bkg
def photutilsky(image, box_size=(50, 50), filter_size=(3, 3)):
""" Estimate sky background using photutils."""
sigma_clip = SigmaClip(sigma=3.)
bkg_estimator = MedianBackground()
bkg = Background2D(image.data,
box_size,
mask=image.mask,
filter_size=filter_size,
sigma_clip=sigma_clip)
return bkg.background
def get_mask(self,
flt_name,
kernel_fwhm=1.25,
background_box=20,
thr=0.05,
npixels=100):
"""
Function to create a mask (set to 0 for no detection and 1 for detection) appropriate to mask WFC3 slitless data.
Attributes
----------
flt_name string containing the name of the FLT name to create a mask for
kernel_fwhm Float The size of the detection kernel (default = 1.25 pixel)
background_box Int The saie fo the background box when estimating the background (default = 20 pixels)
thr Float Threshold above noise to detect signal (default = 0.25)
npixels Int number of pixels for a spectrum to be detected (default = 15)
Output
------
A numpy array containing the mask
"""
h = fits.open(flt_name)[0].header
filt = h["FILTER"]
fin = fits.open(flt_name)
image = fin["SCI"].data
err = fin["ERR"].data
dq = fin["DQ"].data
dq = np.bitwise_and(dq,
np.zeros(np.shape(dq), np.int16) + self.bit_mask)
g = Gaussian1D(mean=0., stddev=kernel_fwhm / 2.35)
x = np.arange(16.) - 8
a = g(x)
kernel = np.tile(a, (16 * int(kernel_fwhm + 1), 1)).T
kernel = kernel / np.sum(kernel)
b = Background2D(image, background_box)
image = image - b.background
threshold = thr * err
image[dq > 0] = 0. #np.nan
mask = detect_sources(image,
threshold,
npixels=npixels,
filter_kernel=kernel).data
ok = (mask == 0.) & (dq == 0)
mask[~ok] = 1.
return mask
def find_sources_via_segmentation(data,
sigma=3,
fwhm=2.0,
min_pix=5,
make_plot=False):
"""
"""
yd, xd = data.shape
# Let's define the background using boxes of ~50x50 pixels
nboxx = int(xd / 150)
nboxy = int(yd / 150)
bkg_estimator = MedianBackground()
bkg = Background2D(data, (nboxy, nboxx),
filter_size=(3, 3),
bkg_estimator=bkg_estimator)
data -= bkg.background # subtract the background
threshold = sigma * bkg.background_rms
#threshold = detect_threshold(data, nsigma=sigma)
gaussian_sigma = fwhm * gaussian_fwhm_to_sigma
kernel = Gaussian2DKernel(gaussian_sigma, x_size=3, y_size=3)
kernel.normalize(mode='integral')
segm = detect_sources(data,
threshold,
npixels=min_pix,
filter_kernel=kernel)
segm_deblend = deblend_sources(data,
segm,
npixels=min_pix,
filter_kernel=kernel,
nlevels=32,
contrast=0.001)
if make_plot:
norm = ImageNormalize(stretch=SqrtStretch())
fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(10, 12.5))
ax1.imshow(data, origin='lower', cmap='Greys_r', norm=norm)
ax1.set_title('Data')
cmap = segm.make_cmap(seed=123)
ax2.imshow(segm, origin='lower', cmap=cmap, interpolation='nearest')
ax2.set_title('Segmentation Image')
ax3.imshow(segm_deblend,
origin='lower',
cmap=cmap,
interpolation='nearest')
ax3.set_title('Deblended Segmentation Image')
plt.show()
#plt.save('testing.jpg')
return data, segm_deblend, bkg
def _background2d(self):
"""
Estimate the 2D background and background RMS noise in an image.
Returns
-------
background : `photutils.background.Background2D`
A Background2D object containing the 2D background and
background RMS noise estimates.
"""
sigma_clip = SigmaClip(sigma=3.)
bkg_estimator = MedianBackground()
filter_size = (3, 3)
try:
bkg = Background2D(self.data,
self.box_size,
filter_size=filter_size,
mask=self.mask,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
except ValueError:
# use the entire unmasked array
bkg = Background2D(self.data,
self.data.shape,
filter_size=filter_size,
mask=self.mask,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
exclude_percentile=100.)
log.info('Background could not be estimated in meshes. '
'Using the entire unmasked array for background '
f'estimation: bkg_boxsize={self.data.shape}.')
# apply the coverage mask
bkg.background *= np.logical_not(self.mask)
bkg.background_rms *= np.logical_not(self.mask)
return bkg
def return_backgrounds(image, mask, mskimg_fname, saveBackground=True):
rms_fname = mskimg_fname.replace('mskimg', 'rmsimg')
if os.path.exists(rms_fname):
with fits.open(rms_fname) as f:
rms = f[0].data
else:
bkg = Background2D(image, (10, 10),
mask=mask,
filter_size=1,
bkg_estimator=MedianBackground(sigma_clip=None),
bkgrms_estimator=StdBackgroundRMS(sigma_clip=None))
rms = bkg.background_rms.astype(np.float32)
if saveBackground:
create_fits(rms_fname, rms)
return rms
def get_background_level(image, npix=50, sextractor=False, ordinary=False):
" Extimate the background level using photutils 0.3"
from photutils.background import Background2D, SExtractorBackground
from photutils import SigmaClip
if npix > image.shape[0]:
print "Image shape", image.shape
npix = image.shape[0] - 1
if sextractor:
sigma_clip = SigmaClip(sigma=2.)
bkg = SExtractorBackground(sigma_clip)
bkg_value = bkg.calc_background(image)
elif ordinary:
from scipy.ndimage import filters
pix = image.ravel()
bkg_value = filters.median_filter(pix, 50).reshape(image.shape)
else:
bkg = Background2D(image, box_size=npix)
bkg_value = bkg.background
return bkg_value
def remove_background(image: torch.Tensor,
box_size: int = 4,
filter_size: int = 3,
mask=None,
pass_background=False) -> torch.Tensor:
""" remove background noise from given image """
sigma_clip = SigmaClip(sigma=3.)
# bkg_estimator = MedianBackground()
bkg_estimator = SExtractorBackground()
bkg = Background2D(
image,
box_size=box_size,
filter_size=filter_size,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask,
)
clean_image = image - bkg.background
return (clean_image, bkp.background) if pass_background else clean_image
def low_res_interpolation(frame, downsample_factor, mask=None):
"""
This function ...
:param frame:
:param downsample_factor:
:param mask:
:return:
"""
# Calculate the x and y size of the low-resolution map of the data
low_res_x_size = int(round(frame.xsize / downsample_factor))
low_res_y_size = int(round(frame.ysize / downsample_factor))
# Create the background by interpolation
back = Background2D(np.asarray(frame), (low_res_y_size, low_res_x_size), filter_shape=(3, 3), filter_threshold=None, mask=mask,
method='sextractor', backfunc=None, interp_order=3, sigclip_sigma=3.0, sigclip_iters=10)
# Return the data
return back
def run_photometry(img_file, epsf, fwhm, x, y, subtract_back=False,
forced=False):
img_hdu = fits.open(img_file)
if subtract_back:
bkg = Background2D(img_hdu[0].data, (21,21), filter_size=(3,3))
image = img_hdu[0].data - bkg.background
ndimage = NDData(data=backsub)
else:
image = img_hdu[0].data
ndimage = NDData(data=img_hdu[0].data)
psf = copy.copy(epsf)
stars_tbl = Table()
stars_tbl['x'] = x
stars_tbl['y'] = y
stars = extract_stars(ndimage, stars_tbl, size=51)
stars_tbl['flux'] = np.array([stars[0].estimate_flux()])
targets = Table()
targets['x_0'] = stars_tbl['x']
targets['y_0'] = stars_tbl['y']
targets['flux_0'] = stars_tbl['flux']
if forced:
psf.x_0.fixed = True
psf.y_0.fixed = True
daogroup = DAOGroup(fwhm)
photometry = BasicPSFPhotometry(group_maker=daogroup,
bkg_estimator=mmm_bkg,
psf_model=psf,
fitter=fitter,
fitshape=(51,51))
result_tab = photometry(image=image, init_guesses=targets)
return(result_tab)
#print(imdata[10,1995:])
#print(imdata[1995:,10])
#print(np.nanmean(np.array(imdata1)))
#print(np.nanmean(np.array(imdata2)))
#print(np.nanmean(np.array(imdata3)))
#print(np.mean(np.array(imdata4)))
#print(np.nanmean(np.array(imdata4)))
#print(np.median(np.array(imdata4)))
sigma_clip = SigmaClip(sigma=3.)
bkg_estimator = MedianBackground()
bkg1 = Background2D(imdata1,
imdata1.shape,
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
print('bkg1.background_median = ', bkg1.background_median)
print('bkg1.background_rms_median = ', bkg1.background_rms_median)
bkg2 = Background2D(imdata2,
imdata2.shape,
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
print('bkg2.background_median = ', bkg2.background_median)
print('bkg2.background_rms_median = ', bkg2.background_rms_median)
bkg3 = Background2D(imdata3,
imdata3.shape,
def do_aperture_photometry(filename):
fwhm,filename=iraf_fwhm()
xpix,ypix=source_list(filename)
#ast=AstrometryNet()
#ast.api_key= 'iqmqwvazpvolmjmn'
data,header=fits.getdata(filename,header=True)
exposure_time=header['EXPOSURE']
sigma_clip = SigmaClip(sigma=3., maxiters=10)
bkg_estimator = MedianBackground()
bkg = Background2D(data, (10,10), filter_size=(3, 3),sigma_clip=sigma_clip, bkg_estimator=bkg_estimator)
back=bkg.background # this is the background we need for the background subtraction.
back2=np.median(bkg.background)
mask = data == 0
unit = u.electron / u.s
xdf_image = CCDData(data, unit=unit, meta=header, mask=mask)
norm_image = ImageNormalize(vmin=1e-4, vmax=5e-2, stretch=LogStretch(), clip=False)
xdf_image_clipped = np.clip(xdf_image, 1e-4, None)
mean, median, std = sigma_clipped_stats(xdf_image.data, sigma=3.0, maxiters=20, mask=xdf_image.mask)
print('Finding the sources')
#daofind = DAOStarFinder(fwhm=fwhm, threshold=5*std) # 3 sigma above the background.
#sources = daofind(data - median)
#sources_findpeaks = find_peaks(xdf_image.data, mask=xdf_image.mask, threshold=30.*std, box_size=30, centroid_func=centroid_2dg)
#print('We have found:',len(sources),' sources')
#print(sources)
#print(sources['xcentroid'], sources['ycentroid'],sources['fwhm'])
#positions=sources['xcentroid'], sources['ycentroid']
positions=np.genfromtxt('co_ordinates_list_1.txt',unpack=True,usecols=(0,1))
#print(positions)
radii=[ fwhm,2*fwhm, 4*fwhm, 6*fwhm]
#positions=(sources['xcentroid'], sources['ycentroid'])
apertures = [CircularAperture(positions, r=r) for r in radii]
an_ap = CircularAnnulus(positions, r_in=8*fwhm, r_out=10*fwhm)
#apers = [apertures, annulus_apertures]
#bkg_sigma=mad_std(data)
effective_gain=exposure_time
error=calc_total_error(data,back,effective_gain)
#error=0.1*data
phot_table = aperture_photometry(data, apertures,error=error)
phot_table2=aperture_photometry(data,an_ap)
bkg_mean = phot_table2['aperture_sum'] / an_ap.area()
bkg_sum = bkg_mean * an_ap.area()
final_sum0=phot_table['aperture_sum_0']-bkg_sum
final_sum1=phot_table['aperture_sum_1']-bkg_sum
final_sum2=phot_table['aperture_sum_2']-bkg_sum
final_sum3=phot_table['aperture_sum_3']-bkg_sum
mag_0=-2.5*np.log10(final_sum0/exposure_time)+25
mag_1=-2.5*np.log10(final_sum1/exposure_time)+25
mag_2=-2.5*np.log10(final_sum2/exposure_time)+25
mag_3=-2.5*np.log10(final_sum3/exposure_time)+25
fig=plt.figure()
plt.imshow(data,cmap='gray',origin='lower',vmin=50,vmax=400)
colors=['red','green','yellow','blue']
for i in range(len(apertures)):
apertures[i].plot(color=colors[i], alpha=0.7)
an_ap.plot(color='green', alpha=0.7)
plt.show()
for i in range (len(phot_table)):
print(mag_0[i],mag_1[i],mag_2[i],mag_3[i])
'''
mag=-2.5*np.log10(final_sum/30)+25
flux=final_sum
flux_err=phot_table['aperture_sum_err_0']
mag_err=1.09*flux_err/flux
x=[phot.value for phot in phot_table['xcenter']]
y=[phot.value for phot in phot_table['ycenter']]
#with open('result.dat', 'w') as f:
#with open('out.txt', 'w') as f:
for i in range(len(x)):
print(x[i],y[i],'\t',mag[i],mag_err[i])
outfile=' '
for i in range (len(phot_table)):
outfile+=x[i]+ " "+ y[i]+" "+ mag[i]+" " +mag_err[i]
outfile+='\n'
out=open('result.txt','w')
out.write(outfile,overwrite=True)
out.close()
'''
'''
im1=plt.imshow(data*~mask, origin='lower', cmap='Greys_r', norm=norm)
plt.colorbar(im1)
plt.savefig('data_masked.png')
#plt.savefig('dilate'+str(stdev)+'.png')
# Save extended mask image:
#hdu.header.update(wcs.to_header())
#hdu.data = segmap_gauss
#hdu.writeto("seg_dilate"+str(stdev)+".fits", overwrite=True)
# Make another mask for blank areas in the original image
bkgmask = (data == 0.0)
# Create a Background2D object using a box size of mesh_size x mesh_size and a 5x5 median filter
# 50: 1 100:7 150:10 200:15 300:25 500:26
mesh_size=50
bkg = Background2D(data, (mesh_size, mesh_size), filter_size=(3, 3), mask=mask, coverage_mask=bkgmask,sigma_clip=sigma_clip, bkg_estimator=bkg_estimator,exclude_percentile=1,fill_value=0.0)
print(bkg.background_median, bkg.background_rms_median)
fig = plt.figure(figsize=(12, 4))
plt.subplots_adjust(hspace=0., wspace=0.,left=None, right=None, bottom=None, top=None)
plt.subplot(1, 3, 1)
# Background image
plt.xlim(250,3000)
plt.ylim(1500,3500)
plt.axis('off')
im1=plt.imshow(bkg.background, origin='lower', cmap='Greys_r')
#plt.colorbar(im1,cmap='Greys_r')
plt.subplot(1, 3, 2)
#plt.title(str(mesh_size)+" pix")
# Original image with meshes
plt.xlim(250,3000)
kernel.normalize()
# Make a source mask to enable optimal background estimation
mask = make_source_mask(off_scaled.data,
nsigma=2,
npixels=5,
filter_kernel=kernel,
mask=off_scaled.mask,
dilate_size=11)
#impl = plt.imshow(mask, origin='lower',
# cmap=plt.cm.gray,
# filternorm=0, interpolation='none')
#plt.show()
box_size = int(np.mean(on.shape) / 10)
back = Background2D(off_scaled,
box_size,
mask=mask,
coverage_mask=off_scaled.mask)
threshold = back.background + (2.0 * back.background_rms)
print(
f'off_scaled background_median = {back.background_median}, background_rms_median = {back.background_rms_median}'
)
impl = plt.imshow(back.background,
origin='lower',
cmap=plt.cm.gray,
filternorm=0,
interpolation='none')
back.plot_meshes()
plt.show()
smooth_off_scaled = back.background * u.electron / u.s
def movie(long_target_name,
dates_to_examine=[],
target_number=0,
box_size=300,
show_centroid=1):
import time
'''Authors:
Patrick Tamburo, Boston University, Jan 2020, July 2020
Purpose:
Inputs:
Outputs:
TODO:
'''
warnings.filterwarnings(
'ignore', category=UserWarning, append=True
) #This turns of NaN warnings in sigma_clipped_stats, otherwise we'd get a warning every line.
kernel = Gaussian2DKernel(x_stddev=0.5)
pines_path = pines_dir_check()
target_name = short_name_creator(long_target_name)
object_path = pines_path / ('Objects/' + target_name + '/')
reduced_path = object_path / 'reduced/'
reduced_files = np.array(
natsort.natsorted([x for x in reduced_path.glob('*.fits')]))
centroid_path = object_path / 'sources/'
positions = pd.read_csv(centroid_path /
'target_and_references_centroids.csv')
x_positions = np.array(positions[positions.keys()[2 * target_number]])
y_positions = np.array(positions[positions.keys()[2 * target_number + 1]])
initial_position = (int(x_positions[0]), int(y_positions[0]))
#Get list of files
all_frame_list = np.array([])
for i in range(np.size(reduced_files)):
file_name = reduced_files[i].name
all_frame_list = np.append(all_frame_list, file_name)
num_files = len(reduced_files)
dates = np.array([
reduced_files[i].name.split('.')[0] for i in range(len(reduced_files))
])
plt.ion()
fig, ax = plt.subplots(1, 1, figsize=(8, 7))
for i in range(len(reduced_files)):
if len(dates_to_examine) != 0:
if reduced_files[i].name.split('.')[0] in dates_to_examine:
image_path = reduced_files[i]
title = image_path.name
image = fits.open(image_path)[0].data
header = fits.open(image_path)[0].header
image = interpolate_replace_nans(image, kernel)
image_bg = Background2D(image, 64)
image = image - image_bg.background
frame = image[initial_position[1] -
int(box_size / 2):initial_position[1] +
int(box_size / 2), initial_position[0] -
int(box_size / 2):initial_position[0] +
int(box_size / 2)]
norm = ImageNormalize(frame,
interval=ZScaleInterval(),
stretch=SquaredStretch())
im = ax.imshow(image, origin='lower', norm=norm)
ax.set_xlim(initial_position[0] - int(box_size / 2),
initial_position[0] + int(box_size / 2))
ax.set_ylim(initial_position[1] - int(box_size / 2),
initial_position[1] + int(box_size / 2))
if show_centroid:
ax.plot(x_positions[i], y_positions[i], 'mo')
ax.set_title(title)
plt.pause(0.01)
ax.cla()
else:
image_path = reduced_files[i]
title = image_path.name
image = fits.open(image_path)[0].data
header = fits.open(image_path)[0].header
frame = image[initial_position[1] -
int(box_size / 2):initial_position[1] +
int(box_size / 2), initial_position[0] -
int(box_size / 2):initial_position[0] +
int(box_size / 2)]
avg, med, std = sigma_clipped_stats(image)
im = ax.imshow(image, origin='lower', vmin=med, vmax=med + 5 * std)
ax.set_xlim(initial_position[0] - int(box_size / 2),
initial_position[0] + int(box_size / 2))
ax.set_ylim(initial_position[1] - int(box_size / 2),
initial_position[1] + int(box_size / 2))
cb = fig.colorbar(im, orientation='vertical', label='Counts')
if show_centroid:
ax.plot(x_positions[i], y_positions[i], 'bx')
ax.set_title(title)
plt.pause(0.01)
ax.cla()
cb.remove()
veri_signal_aperture, veri_final, veri_patch = aper_photom(
data, centre=veri_centroid, radius=veri_aper_diam * 0.5)
fits_inf.close()
#%%
fits_inf = fits.open("../verification/veri_circle_60.0_radius.fit")
data = fits_inf[0].data
veri_weight = np.ones((512, 512))
veri_centroid = centroid_func(data, veri_weight)
veri_aper_diam = 120.
veri_signal_aperture, veri_final, veri_patch = aper_photom(
data, centre=veri_centroid, radius=veri_aper_diam * 0.5)
fits_inf.close()
#%%
fits_inf = fits.open(
"../verification/veri_circle_30.0_radius_signal_strength_10.0.fit")
data = fits_inf[0].data
veri_weight = np.ones((512, 512))
veri_centroid = centroid_func(data, veri_weight)
veri_aper_diam = 115.
veri_signal_aperture, veri_final, veri_patch = aper_photom(
data, centre=veri_centroid, radius=veri_aper_diam * 0.5)
bkg = Background2D(data, (128, 128))
result_imag = data - bkg.background
fits_inf.close()
print(veri_final)
def readimageanddatared(image_name,
fits_data_slice,
output_name,
plot_title,
fits_cut,
fileloc=0):
'''
Function: Read images and create simple plots
'''
print('>>> Running readimageanddatared')
if fileloc == 0:
image_file = fits.open(image_name + '.fits')
else:
image_file = fits.open('./All_Files/' + image_name + '.fits')
image_file.info()
image_org_wcs = wcs.WCS(image_file[fits_data_slice].header)
# Create cut out
if fits_cut[0] > 0:
image_sci_full = image_file[fits_data_slice].data
image_sci_cutout = Cutout2D(image_sci_full, (fits_cut[0], fits_cut[1]),
(fits_cut[2] * 2, fits_cut[3] * 2),
wcs=image_org_wcs)
image_sci = image_sci_cutout.data
image_wcs = image_sci_cutout.wcs
else:
image_sci_full = image_file[fits_data_slice].data
image_sci = image_sci_full
image_wcs = image_org_wcs
# Create rms normalized results (based on region chosen)
image_sigma = mad_std(image_sci)
image_sci_rmsnorm = image_sci / image_sigma
image_sci_full_rmsnorm = image_sci_full / image_sigma
# Create arrays for plotting (setting negatives to zero)
image_sci_rmsnorm_plot = image_sci_rmsnorm
image_sci_full_rmsnorm_plot = image_sci_full_rmsnorm
image_sci_rmsnorm_plot[image_sci_rmsnorm_plot < 0] = 0
image_sci_full_rmsnorm_plot[image_sci_full_rmsnorm_plot < 0] = 0
# Create background image
bkg = Background2D(image_sci, (7, 7))
# Calculate Error Array
image_exptime = float(image_file[0].header['EXPTIME'])
print('From Header: Exposure Time = {}'.format(image_exptime))
image_error = calc_total_error(image_sci, bkg.background_rms,
image_exptime)
# Plot Full File + Rectangle
fig, ax = plt.subplots(1, figsize=(20, 20))
plt.imshow(np.sqrt(image_sci_full_rmsnorm),
origin='lower',
cmap='Greys_r',
vmin=sn_vmin,
vmax=sn_vmax)
plt.plot(fits_cut[0], fits_cut[1], 'rs')
rect = patches.Rectangle(
(fits_cut[0] - fits_cut[2], fits_cut[1] - fits_cut[3]),
fits_cut[2] * 2,
fits_cut[3] * 2,
linewidth=1,
edgecolor='r',
facecolor='none')
ax.add_patch(rect)
plt.title(plot_title)
plt.axis([
fits_cut[0] - 1000, fits_cut[0] + 1000, fits_cut[1] - 1000,
fits_cut[1] + 1000
])
plt.savefig(output_name + '01_rms_full.jpg')
plt.close()
# Plot cut region
fig, ax = plt.subplots(1, figsize=(20, 20))
plt.imshow(np.sqrt(image_sci_rmsnorm_plot),
origin='lower',
cmap='Greys_r',
vmin=sn_vmin,
vmax=sn_vmax)
plt.title(plot_title)
plt.savefig(output_name + '02_rms_cut.jpg')
plt.close()
# Plot cut region (background image)
fig, ax = plt.subplots(1, figsize=(20, 20))
plt.imshow(bkg.background, origin='lower', cmap='Greys_r')
plt.title(plot_title)
plt.savefig(output_name + '03_background.jpg')
plt.close()
print('Size of Output Array: {}'.format(len(image_sci)))
print('{} Pixel Value Range: {:.2e} to {:.2e}'.format(
image_name, np.nanmin(image_sci), np.nanmax(image_sci)))
print('{} S/N Value Range: {:.2e} to {:.2e}'.format(
image_name, np.nanmin(image_sci_rmsnorm),
np.nanmax(image_sci_rmsnorm)))
return image_sci, image_sigma, image_wcs, image_error, image_exptime
def do_aperture_photometry(filename,count, fwhm,date):
#fwhm,files=iraf_fwhm()
#xpix,ypix=source_list(files)
#ast=AstrometryNet()
#ast.api_key= 'iqmqwvazpvolmjmn'
'''
choice=input("Enter the mode. Please use the keywords\n 'single' for single image, 'multiple' for multiple images: \n\n")
if (choice=='single'):
print('single mode')
elif (choice ==' multiple'):
print(' multiple image mode')
else:
print('Enter valid choice!!!')
'''
data,header=fits.getdata(filename,header=True)
#exposure=header['EXPOSURE']
exposure=300
#print('Exposure is',exposure)
sigma_clip = SigmaClip(sigma=3, maxiters=10)
bkg_estimator = SExtractorBackground()
bkg = Background2D(data, (10,10), filter_size=(3, 3),sigma_clip=sigma_clip, bkg_estimator=bkg_estimator)
back=bkg.background # this is the background we need for the background subtraction.
back2=np.median(bkg.background)
#print('median background is',back2)
mask = data == 0
unit = u.electron / u.s
xdf_image = CCDData(data, unit=unit, meta=header, mask=mask)
norm_image = ImageNormalize(vmin=1e-4, vmax=5e-2, stretch=LogStretch(), clip=False)
xdf_image_clipped = np.clip(xdf_image, 1e-4, None)
mean, median, std = sigma_clipped_stats(xdf_image.data, sigma=3.0, maxiters=20, mask=xdf_image.mask)
#print('Finding the sources')
#daofind = DAOStarFinder(fwhm=fwhm, threshold=5*std) # 3 sigma above the background.
#sources = daofind(data-back)
#sources_findpeaks = find_peaks(xdf_image.data, mask=xdf_image.mask, threshold=30.*std, box_size=30, centroid_func=centroid_2dg)
#print('We have found:',len(sources),' sources')
#print(sources)
#print(sources['xcentroid'], sources['ycentroid'],sources['fwhm'])
#positions=sources['xcentroid'], sources['ycentroid']
positions=np.genfromtxt('co_ordinates_list.txt',unpack=True,usecols=(0,1))
#print(positions)
radii=[ fwhm,2*fwhm, 3*fwhm,4*fwhm,5*fwhm]
#positions=(sources['xcentroid'], sources['ycentroid'])
apertures = [CircularAperture(positions, r=r) for r in radii]
an_ap = CircularAnnulus(positions, r_in=6*fwhm, r_out=6.2*fwhm)
#apers = [apertures, annulus_apertures]
#bkg_sigma=mad_std(data)
effective_gain=exposure
error=calc_total_error(data,back,effective_gain)
#error=0.1*data
phot_table = aperture_photometry(data-back, apertures,error=error)
phot_table2=aperture_photometry(data-back,an_ap)
bkg_mean = phot_table2['aperture_sum'] / an_ap.area
bkg_sum = bkg_mean * an_ap.area
final_sum0=phot_table['aperture_sum_0']-bkg_sum
final_sum1=phot_table['aperture_sum_1']-bkg_sum
final_sum2=phot_table['aperture_sum_2']-bkg_sum
final_sum3=phot_table['aperture_sum_3']-bkg_sum
final_sum4=phot_table['aperture_sum_4']-bkg_sum
mag_back=-2.5*np.log10(bkg_mean/exposure)+22
mag_0=-2.5*np.log10(final_sum0/exposure)+22
mag_1=-2.5*np.log10(final_sum1/exposure)+22
mag_2=-2.5*np.log10(final_sum2/exposure)+22
mag_3=-2.5*np.log10(final_sum3/exposure)+22
mag_4=-2.5*np.log10(final_sum4/exposure)+22
#print(mag_back,mag_0,mag_1,mag_2,mag_3,mag_4)
flux_err_0=phot_table['aperture_sum_err_0']
mag_err_0=1.09*flux_err_0/final_sum0
flux_err_1=phot_table['aperture_sum_err_1']
mag_err_1=1.09*flux_err_1/final_sum1
flux_err_2=phot_table['aperture_sum_err_2']
mag_err_2=1.09*flux_err_2/final_sum2
flux_err_3=phot_table['aperture_sum_err_3']
mag_err_3=1.09*flux_err_3/final_sum3
flux_err_4=phot_table['aperture_sum_err_4']
mag_err_4=1.09*flux_err_4/final_sum4
'''
fig=plt.figure()
plt.imshow(data,cmap='gray',origin='lower',vmin=mean-4*std,vmax=mean+4*std)
colors=['red','salmon','yellow','blue','cyan']
for i in range(len(apertures)):
apertures[i].plot(color=colors[i], alpha=0.7)
an_ap.plot(color='green', alpha=0.7)
plt.show()
'''
with open ('{}_r.dat'.format(date),'w') as r:
for i in range (len(phot_table)):
#print(final_sum0[i],final_sum1[i],final_sum2[i],final_sum3[i],final_sum4[i],final_sum5[i],file=r)
print(mag_back[i],mag_0[i],mag_err_0[i],mag_1[i],mag_err_1[i],mag_2[i],mag_err_2[i],mag_3[i],mag_err_3[i],mag_4[i],mag_err_4[i],file=r)
print('No. {} file has been processed'.format(count+1))
'''
mag=-2.5*np.log10(final_sum/30)+25
flux=final_sum
flux_err=phot_table['aperture_sum_err_0']
mag_err=1.09*flux_err/flux
x=[phot.value for phot in phot_table['xcenter']]
y=[phot.value for phot in phot_table['ycenter']]
#with open('result.dat', 'w') as f:
#with open('out.txt', 'w') as f:
for i in range(len(x)):
print(x[i],y[i],'\t',mag[i],mag_err[i])
outfile=' '
for i in range (len(phot_table)):
outfile+=x[i]+ " "+ y[i]+" "+ mag[i]+" " +mag_err[i]
outfile+='\n'
out=open('result.txt','w')
out.write(outfile,overwrite=True)
out.close()
'''
'''
zorder=2)
ax1.add_patch(outer_sky)
radial = radial_profile(new_image, [new_image_halfwidth, new_image_halfwidth])
#perform the aperture photometry
#currently 2% different from IDL atv
aperture = CircularAperture((x, y), r=aperture_radius)
annulus = CircularAnnulus((x, y),
r_in=inner_sky_radius,
r_out=outer_sky_radius)
phot_table = aperture_photometry(data, [aperture, annulus])
#new background estimation
bkg_mask = np.ma.masked_outside(radial[0].reshape(new_image.shape),
inner_sky_radius, outer_sky_radius)
bkg_map = Background2D(new_image,
tuple(np.array(new_image.shape) / 4),
mask=bkg_mask.mask,
exclude_mesh_method='all')
bkg_map_med = MMMBackground().calc_background(
bkg_map.data) #bkg_map_med=np.median(bkg_map.background)
#print 'Map sky mean '+str(bkg_map_med)
#bkg_mean=phot_table['aperture_sum_1']/annulus.area()
#print 'Aperture sky mean '+str(bkg_mean)
#phot_table['residual_aperture_sum']=phot_table['aperture_sum_0']-bkg_mean*aperture.area()
phot_table['residual_aperture_sum'] = phot_table[
'aperture_sum_0'] - bkg_map_med * aperture.area()
#print 'Map sky result: '+str(phot_table['aperture_sum_0']-bkg_map_med*aperture.area())
#print "Aperture Photometry Result: "+str(phot_table['residual_aperture_sum'])
fig2 = Figure(figsize=(int(6 * scaling), int(6 * scaling)))
fig2.set_facecolor('0.85')
ax2 = fig2.add_axes([0.1, 0.1, .9, .9])
ax2.grid(True, color='white', linestyle='-', linewidth=1)
def epoxi_vis_read(folder,year,observations,trim_primary,repair_middle,remove_background,\
min_aperature,max_aperature,width_trim, astronomical_unit = 149.597870691e06):
df = pd.DataFrame()
# MAIN
### Get filepath of all fit files and access all files one by one
# * causes it to access every file within the folder
for filepath in glob.iglob(
r'../../DATA/dif-e-hriv-3_4-epoxi-earth-v2.0/data/' + folder +
'/' + year + '/' + observations + '/*.fit'):
# for filepath in glob.iglob(r'./verification/*.fit'): ### verification
#for filepath in glob.iglob(r'../DATA/dif-e-hriv-3_4-epoxi-earth-v2.0/data/rad/2008/078/*.fit'):
print(filepath)
fits_inf = fits.open(filepath)
#fits_inf.info()
### NAN values not filtered out for flags and snr and destripe
# np.nan_to_num(fits_inf[1].data, nan = 1.0) can be used for image_flags
image_prim = np.nan_to_num(
rotate_image(fits_inf[0].data)
) # Rotate the image to get North up, 90 deg CW !!!CHECK IF THIS IS NORTH UP OR SOUTH UP!!!
image_flags = rotate_image(
fits_inf[1].data
) # Order should be the same for all visible light files
# image_snr = rotate_image(fits_inf[2].data)
# image_destripe = rotate_image(fits_inf[3].data)
########################### verification only prim and flags
# image_prim = np.nan_to_num(fits_inf[0].data) # Rotate the image to get North up, 90 deg CW !!!CHECK IF THIS IS NORTH UP OR SOUTH UP!!!
# image_flags = fits_inf[1].data # Order should be the same for all visible light files
# Subtract the quality-flag image from unity. If the quality flag is 0,
# then the resulting weight is 1. If the quality flag is greater than 1,
# the resulting weight is negative. Retain only values greater than 0.
weight = np.ones(
image_flags.shape
) - image_flags #################### if NAN values = 0 will result in 1 weight!!!!!!!!!!!!!
weight = np.where(weight < 0, 0,
weight) # setting negative values to zero
# Set the weight at the edge of the image to zero. The weight of this region
# is zero, regardless of whether the region is trimmed off before returning
# the image from the function.
weight = trim_edges(weight, width_trim)
# If the keyword is set to trim the edges of the image array, then zero the
# values of rows and columns in the outer edge, to a width of 5 pixels.
if trim_primary == True:
image_prim = trim_edges(image_prim, width_trim)
# The middle column of the image (actually, one to the right of that) is
# overly amplified by the flat-field correction. If the keyword REPAIR_MIDDLE is
# set, replace this column by the average of the neighboring two columns.
# (Technically, the middle is at (NAXIS1-1)/2 = for instance, 255.5. 256 is
# the column whose value needs to be corrected).
if repair_middle == True:
image_prim = repair_middle_func(image_prim)
# If the keyword REMOVE_BACKGROUND is set, then we need to estimate an
# image of the background that can be subtracted from the image. We will
# do this by interpolating across the image array from an average of a
# few rows or columns at the edge of the non-zero-weighted image region.
# The interpolation step runs from the middle of the 3 rows or columns
# that are averaged, so it extrapolates slightly to include the outermost
# of the averaged rows/columns.
if remove_background == True:
med_image_prim = ndim.median_filter(
image_prim,
3) # 3, default value, sig.medfilt alternative but slower
med_weight = ndim.median_filter(weight, 3)
earth_radius_pxl = 150 #60 150 for polar 1
centroid_last = np.array([
20, 10
]) + (512 - 1) / 2. #np.array([10,30]) + (512-1)/2. for polar 1
radius, phi = make_polar(512, 512, centre=centroid_last)
image_disk = np.zeros([512, 512])
mask_disk = np.where(
np.logical_and(
radius <= earth_radius_pxl,
np.logical_or(phi <= 0.5 * np.pi, phi >= 1.5 * np.pi)))
image_disk[mask_disk] = True
width_trim = 6
bkg = Background2D(med_image_prim, (256, 128),
mask=image_disk,
exclude_percentile=0,
sigma_clip=None) # (128,128) (ny,nx) (256,128)
#bkg = Background2D(med_image_prim,(128,128)) # doesn't work exclude_percentile fixes it
result_imag = med_image_prim - bkg.background
result_imag = trim_edges(result_imag, 6)
first_non_zero_row = width_trim
last_non_zero_row = med_image_prim.shape[
0] - width_trim - 1 # -1 for index
# Added image of the top 3 rows and added weight of the bottom 3 rows
rows_top_imag = background_average(
med_image_prim, med_weight, first_non_zero_row, True,
True) # weighted average of the 3 bottom rows filtered >0
rows_bot_imag = background_average(med_image_prim, med_weight,
last_non_zero_row, False, True)
for i in np.arange(first_non_zero_row, last_non_zero_row +
1): # +1 as last is not included in np.arange
# weigthed average of the background subtracted from all rows, starting at the middle of the averaged edge (3 rows thick)
# so first_non_zero_row+1 = 7 from the top and bottom.
# -1 for the index, -7 for the edge, -i to let rows closer to the top have a higher average background from the top background
middle_averaged = first_non_zero_row + 1
image_prim[i,:] -= (rows_top_imag*(image_prim.shape[0]-1-middle_averaged-i) + rows_bot_imag*(i-middle_averaged))\
/(image_prim.shape[0]-1-middle_averaged-middle_averaged)
# Need to calculate a new median image from the now-modified image.
med_image_prim = ndim.median_filter(image_prim, 3)
first_non_zero_col = width_trim
last_non_zero_col = med_image_prim.shape[
1] - width_trim - 1 #-1 for indexs
cols_left_imag = background_average(med_image_prim, med_weight,
first_non_zero_col, True,
False)
cols_right_imag = background_average(med_image_prim, med_weight,
last_non_zero_col, False,
False)
# for the columns, left half subtract left average column, the right half the right average column
for i in np.arange(first_non_zero_col,
int((med_image_prim.shape[1] - 1) * 0.5)):
image_prim[:, i] -= cols_left_imag
for i in np.arange(int(med_image_prim.shape[1] * 0.5),
last_non_zero_col + 1):
image_prim[:, i] -= cols_right_imag
##### NEW BACKGROUND METHOD
image_prim = result_imag
naxis1 = image_prim.shape[1]
naxis2 = image_prim.shape[0]
epoxi_hrivis = {'file': 'no file processed', \
'exposure_ID': 0, 'image_number': 0, \
'date_calendar': ' ', 'date_julian': 0.0, \
'calibration': 'unknown', 'units': 'arbitrary', \
'duration': 0.0, \
'filter_name': 'unknown', 'filter_cw': 0.0, \
'minimum': 0.0, 'maximum': 0.0, \
'median': 0.0, 'one_sigma': 0.0, \
'num_saturated': 0, \
'mode_number': 0, 'mode': 'unknown', \
'mission': 'supposed to be EPOXI', \
'platform': 'supposed to be fly-by', \
'instrument': 'supposed to be HRIIR', 'cpu': 'none', \
'target': 'unknown', \
'RA': 0.0, 'DEC': 0.0, \
'range_SC': 0.0, 'diameter': 0.0, \
'range_Sun': 0.0, \
'nppa': 0.0, 'oblateness': 0.0, \
'illum': 100.0, 'phase_angle': 0.0, \
'sub_SC': np.zeros(2), 'sub_Sun': np.zeros(2), \
'sub_SC_nom': np.zeros(2), 'sub_Sun_nom': np.zeros(2), \
'north_angle': 0.0, 'Sun_angle': 0.0, \
'image': np.zeros((naxis1,naxis2)), \
'weight': np.zeros((naxis1,naxis2)), \
'naxis1': naxis1, 'naxis2': naxis2, \
'target_center': np.zeros(2), \
'signal': 0.0, 'signal_rms': 0.0, \
'background': 0.0, 'background_rms':0.0, \
'aperture': 0 }
# get all info from the header
image_prim_header = fits_inf[0].header
epoxi_hrivis['image'] = image_prim
epoxi_hrivis['weight'] = weight
epoxi_hrivis['naxis1'] = image_prim_header['NAXIS1']
epoxi_hrivis['naxis2'] = image_prim_header['NAXIS2']
epoxi_hrivis['file'] = image_prim_header['FILESDC'] # image file name
epoxi_hrivis['exposure_ID'] = image_prim_header['EXPID'] # exposure ID
epoxi_hrivis['image_number'] = image_prim_header[
'IMGNUM'] # image number within exposure set
epoxi_hrivis['date_calendar'] = image_prim_header[
'OBSMIDDT'] # image midpoint date-time
epoxi_hrivis['date_julian'] = image_prim_header[
'OBSMIDJD'] # image midpoint Julian date
epoxi_hrivis['calibration'] = image_prim_header[
'CALTYPE'] # calibration type
epoxi_hrivis['units'] = image_prim_header['BUNIT'] # calibrated units
epoxi_hrivis['duration'] = image_prim_header[
'INTTIME'] # image integration time
epoxi_hrivis['filter_name'] = image_prim_header[
'FILTER'] # name of filter
epoxi_hrivis['filter_cw'] = image_prim_header[
'FILTERCW'] # filter center wavelength
epoxi_hrivis['minimum'] = image_prim_header[
'DATAMIN'] # minimum data value in image
epoxi_hrivis['maximum'] = image_prim_header[
'DATAMAX'] # maximum data value in image
epoxi_hrivis['median'] = image_prim_header[
'MEDPVAL'] # median data value in image
epoxi_hrivis['one_sigma'] = image_prim_header[
'STDPVAL'] # standard deviation of image pixels
epoxi_hrivis['num_saturated'] = image_prim_header[
'PSATNUM'] # number saturated pixels in image
epoxi_hrivis['mode_number'] = image_prim_header[
'IMGMODE'] # image-acquisition mode number
epoxi_hrivis['mode'] = image_prim_header[
'IMGMODEN'] # image-acquisition mode name
epoxi_hrivis['mission'] = image_prim_header[
'MISSION'] # name of spacecraft mission
epoxi_hrivis['platform'] = image_prim_header[
'OBSERVAT'] # 'observatory' name
epoxi_hrivis['instrument'] = image_prim_header[
'INSTRUME'] # instrument name
epoxi_hrivis['cpu'] = image_prim_header[
'SCPROCU'] # which spacecraft CPU
epoxi_hrivis['target'] = image_prim_header['OBJECT'] # target of image
epoxi_hrivis['RA'] = image_prim_header[
'BORERA'] # RA of instrument bore-sight
epoxi_hrivis['DEC'] = image_prim_header[
'BOREDEC'] # DEC of instrument bore-sight
epoxi_hrivis['range_SC'] = image_prim_header[
'RANGECEN'] / astronomical_unit # target range to spacecraft, AU
epoxi_hrivis['range_Sun'] = image_prim_header[
'TARSUNR'] / astronomical_unit # target range to Sun, AU
epoxi_hrivis['phase_angle'] = image_prim_header[
'PHANGLE'] # target phase angle, degrees
epoxi_hrivis['north_angle'] = image_prim_header[
'CELESTN'] # J2000 Equat. North angle, CW from up, deg
epoxi_hrivis['Sun_angle'] = image_prim_header[
'SOLARCLK'] # Sun clock angle wrt bore, CW from up, deg
epoxi_hrivis['Sun_angle'] = epoxi_hrivis['north_angle'] - epoxi_hrivis[
'Sun_angle'] # Sun clock angle, CCW from Celestial north
epoxi_hrivis['nppa'] = image_prim_header[
'RECPPAZ'] # Body North pole, CW from up, degs
epoxi_hrivis['nppa'] = epoxi_hrivis['north_angle'] - epoxi_hrivis[
'nppa'] # Body North pole, CCW from Celestial north
epoxi_hrivis['north_angle'] = 360. - epoxi_hrivis[
'north_angle'] # Equatorial North angle, CCW from 'up' on image
# Correct the direction of north for rotation of the image.
epoxi_hrivis['north_angle'] = (epoxi_hrivis['north_angle'] +
90.0) % 360
# [W Longitude, latitude] of sub-spacecraft and sub-Solar points.
# The header actually stores East longitude and thus must be converted
# by subtraction from 360 degrees.
# nom is also kept as REC sometimes has nan values (-1399,-999)
epoxi_hrivis['sub_SC'] = [
360 - image_prim_header['RECSCLON'], image_prim_header['RECSCLAT']
]
epoxi_hrivis['sub_SC_nom'] = [
360 - image_prim_header['NOMSCLON'], image_prim_header['NOMSCLAT']
]
epoxi_hrivis['sub_Sun'] = [
360 - image_prim_header['RECSOLON'], image_prim_header['RECSOLAT']
]
epoxi_hrivis['sub_Sun_nom'] = [
360 - image_prim_header['NOMSOLON'], image_prim_header['NOMSOLAT']
]
# Centroid the image as an approximation to the center of the target image.
# Collapse the rows and columns to create two vectors that can be multiplied
# by index vectors in order to compute the centroid position.
centroid = centroid_func(image_prim, weight)
initial_centroid = centroid
# Employ a succession of circular apertures of increasing dimension to
# estimate the total signal in the image. This is just a crude measure
# to get an initial estimate; however, for images with the Earth fully
# within the field of view, it ought to be pretty accurate. Stop
# tweaking the aperture size when the total signal within the aperture
# (ignore the background-subtracted signal, at this point) ceases to vary.
# The largest permissible aperture diameter is the smaller of the image
# dimensions. However, if APER_FINISH reaches this point with a value of
# zero, then it has not yet been set and needs to be set to the default
# value of the smallest of the two image dimensions.
# Do an initial estimate of the background, then subtract the image with
# the aperture replaced by the background in order to recalculate the
# centroid. This may help in rejecting other sources and background
# signal that may distort the identification of the centroid position.
#################
aper_start = np.around(min_aperature)
aper_finish = np.around(max_aperature)
if aper_finish < aper_start:
aper_finish = np.min([image_prim.shape[0], image_prim.shape[1]])
aper_finish = np.min(
[aper_finish, image_prim.shape[0], image_prim.shape[1]])
aper_diam = np.min(
[aper_start * 1.5, image_prim.shape[0],
image_prim.shape[1]]) # *1.5 here
if aper_start > 0:
signal_aperture, final, patch = aper_photom(image_prim,
centre=centroid,
radius=aper_diam * 0.5)
else:
patch = image_prim * 0 + np.sum(image_prim) / np.float(
image_prim.size)
test = image_prim - patch
########################################################
n_columns = image_prim.shape[1]
row_indexes = np.arange(n_columns)
row_vector = np.zeros(n_columns)
for i in np.arange(0, n_columns):
row_vector += test[i, :] * weight[i, :]
x_centroid = np.sum(row_indexes * row_vector) / np.sum(row_vector)
# Modify the centroid position to be a value relative to the center of the
# image and use that as the target center.
x_centroid -= (n_columns - 1) * 0.5 # -1 to get index position
n_rows = image_prim.shape[0]
col_indexes = np.arange(n_rows)
col_vector = np.zeros(n_rows)
for i in np.arange(0, n_rows):
col_vector += image_prim[:,
i] * weight[:,
i] ##### WHY IMAGE AND NOT TEST? the difference is small, image slightly higher,
# which is closer to the true middle of the Earth
y_centroid = np.sum(col_indexes * col_vector) / np.sum(col_vector)
y_centroid -= (n_rows - 1) * 0.5
centroid = np.array([x_centroid, y_centroid])
# Now we finally are going to calculate the actual aperture photometry.
# Work on a lightly median-filtered version of the image, to resist cosmic-ray
# hits. This is an unwise choice for stellar photometry, but it should
# work just fine for the Earth. It also eliminates the pesky influence
# of scattered zero-weighted pixels.
med_image = ndim.median_filter(image_prim * weight, 3)
aper_diam = np.min(
[aper_start, image_prim.shape[0],
image_prim.shape[1]]) # aper_start*1.5 works better
# The verification of the code is successful however, it does not work (dividing by zero error) when the background is zero
# as the annulus signal is then zero. Furthermore, if the aperture radius is too small, the average background from
# the annulus will be equal to the signal within the aperture which will also lead to dividing by zero.
# This problem occurs due to the fact that when you find the centroid position, you multiply the starting radius
# by 1.5, but you do not do that when you do the iterations when finding the correct aperture.
# With the real data it did not lead to problems as there are no real constant values as in the verification data
# (10.0 values in a circle, 0.5 outside of the circle).
if aper_start > 0:
done = False
prev_signal = 0
while done == False:
signal_aperture, final, patch = aper_photom(med_image,
centre=centroid,
radius=aper_diam *
0.5)
### figure
# fig2, ax2 = plt.subplots()
# plt.title('Median data')
# plt.imshow(med_image, cmap='gray')
# circle1 = plt.Circle(centroid+255.5, aper_diam*0.5, color='r', fill=False)
# ax2.add_artist(circle1)
# plt.scatter(centroid[0]+255.5, centroid[1]+255.5, s=10)
# plt.colorbar()
epoxi_hrivis['signal'] = final[0]
epoxi_hrivis['signal_rms'] = final[3]
epoxi_hrivis['background'] = final[2]
epoxi_hrivis['background_rms'] = final[5]
epoxi_hrivis['aperture'] = final[6] * 2
epoxi_hrivis['target_center'] = final[7:8 + 1]
aper_diam = aper_diam * 1.05
if aper_diam > aper_finish or np.abs(
(signal_aperture - prev_signal) /
signal_aperture) < 1e-3: #1e-3
done = True
prev_signal = signal_aperture
else:
signal_aperture = np.sum(med_image)
n_pixels = np.float(med_image.size)
# final = [signal_aperture, signal_aperture, 0, np.sqrt(np.sum((med_image-signal_aperture/n_pixels)**2)/n_pixels),0,]
epoxi_hrivis['signal'] = signal_aperture
epoxi_hrivis['signal_rms'] = np.sqrt(
np.sum((med_image - signal_aperture / n_pixels)**2) / n_pixels)
epoxi_hrivis['background'] = 0
epoxi_hrivis['background_rms'] = np.copy(
epoxi_hrivis['signal_rms'])
epoxi_hrivis['aperture'] = np.sqrt(n_pixels)
epoxi_hrivis['target_center'] = [0, 0]
#print(final)
### imaging
# fig, ax = plt.subplots()
# plt.title('Primary data')
# plt.imshow(image_prim, cmap='gray')
# circle1 = plt.Circle(centroid+255.5, aper_diam*0.5, color='r', fill=False)
# circle2 = plt.Circle(centroid+255.5, aper_diam*0.5/1.05, color='g', fill=False)
# circle3 = plt.Circle(centroid+255.5, annulus_radius, color='b', fill=False)
# ax.add_artist(circle1)
# ax.add_artist(circle2)
# ax.add_artist(circle3)
# plt.scatter(centroid[0]+255.5, centroid[1]+255.5, s=10)
# plt.colorbar()
fig2, ax2 = plt.subplots()
plt.title('Polar Observation 2: South')
plt.imshow(med_image, cmap='gray')
# circle1 = plt.Circle(centroid+255.5, aper_diam*0.5, color='r', fill=False, label = 'final aperture')
# circle2 = plt.Circle(centroid+255.5, aper_diam*0.5/1.05, color='g', fill=False, label = 'previous aperture')
# circle3 = plt.Circle(centroid+255.5, annulus_radius, color='b', fill=False, label = 'annulus')
# ax2.add_artist(circle1)
# ax2.add_artist(circle2)
# ax2.add_artist(circle3)
# leg_1 = ax2.legend((circle1,circle2,circle3), ('final aperture','previous aperture','annulus'), loc='lower left')
# plt.scatter(centroid[0]+255.5, centroid[1]+255.5, s=10,color = 'b', label = 'final centroid')
# plt.scatter(255.5, 255.5, s=10, color = 'r', label = 'centre image')
# plt.scatter(initial_centroid[0]+255.5, initial_centroid[1]+255.5, s=10,color = 'g', label = 'initial centroid')
# plt.legend()
plt.colorbar()
# ax2.add_artist(leg_1)
# ### saving
df_temp = pd.DataFrame.from_dict(epoxi_hrivis, orient='index')
df_temp = df_temp.transpose()
df = df.append(df_temp, ignore_index=True)
fits_inf.close()
########################################################################## SAVING
#df.to_hdf('../output/RADREV_'+ filepath.split('/')[-2]+'_'+observations+'_'+'min_aper'+'_'+str(min_aperature)+'_'+'dictionary_info.h5','epoxi_hrivis') ### manually change this
return df
def fill_sources(self, image, save_plot=True):
"""
detect sources, mask and fill the gaps using photutils
Input
=========================
image (2D np array)
image to process
save_plot (boolean)
if True, saves a diagnostic plot of pre and post source removal
Returns
=========================
filled_image (2D np array)
image with sources removed and filled
"""
from photutils.background import Background2D
from astropy.stats import gaussian_fwhm_to_sigma
from photutils import detect_sources
from astropy.convolution import Gaussian2DKernel, interpolate_replace_nans, convolve
# get the background level
bkg = Background2D(image, 30)
threshold = bkg.background + (5.0 * bkg.background_rms)
# create the size of the sources
sigma = 2.0 * gaussian_fwhm_to_sigma # FWHM = 2.
kernel = Gaussian2DKernel(sigma, x_size=2, y_size=2)
kernel.normalize()
# search for the sources and create a mask
seg_image = detect_sources(image,
threshold,
npixels=5,
filter_kernel=kernel)
source_mask = np.clip(seg_image, 0, 1)
# fill the masked source regions
masked_image = image.copy()
masked_image[source_mask > 0] = np.nan
kernel = Gaussian2DKernel(x_stddev=5)
filled_image = interpolate_replace_nans(masked_image, kernel)
# smooth out hard edges around filled sources
kernel = Gaussian2DKernel(x_stddev=3)
filled_image = convolve(filled_image, kernel)
# plot
if save_plot:
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 8))
ax1.imshow(image,
cmap='Greys',
origin='lower',
norm=LogNorm(vmin=0.1))
ax2.imshow(filled_image,
cmap='Greys',
origin='lower',
norm=LogNorm(vmin=0.1))
# display the plot
plt.tight_layout()
# save as a pdf
plot_name = os.path.join(
str(self.name),
str(self.name) + '_' + str(self.wavelength) +
'_source_removal.pdf')
try:
os.remove(plot_name)
except:
pass
fig.savefig(plot_name, dpi=200)
return filled_image
def onpress(event):
if event.key == 'p':
filename = d.get('file')
#hdu=fits.open(filename)
hdu = d.get_pyfits()
data = hdu[0].data
x = d.get('crosshair image')
x, y = x.split()
x, y = int(float(x)), int(float(y))
new_image_halfwidth = int(entry1.get())
aperture_radius = int(entry2.get())
inner_sky_radius = int(entry3.get())
outer_sky_radius = int(entry4.get())
new_image = data[y - new_image_halfwidth:y + new_image_halfwidth,
x - new_image_halfwidth:x + new_image_halfwidth]
x_grid = np.arange(x - new_image_halfwidth, x + new_image_halfwidth, 1)
y_grid = np.arange(y - new_image_halfwidth, y + new_image_halfwidth, 1)
x_grid, y_grid = np.meshgrid(x_grid, y_grid)
guess = (np.amax(new_image) - new_image[0, 0], x, y, 3, 3, 0,
new_image[0, 0])
popt, pcov = curve_fit(gauss2d, (x_grid, y_grid),
new_image.ravel(),
p0=guess)
#print popt
x = int(popt[1])
y = int(popt[2])
labeltext.set('X: ' + str(x))
label2text.set('Y: ' + str(y))
new_image = data[y - new_image_halfwidth:y + new_image_halfwidth,
x - new_image_halfwidth:x + new_image_halfwidth]
# for artist in ax1.get_children():
# if hasattr(artist,'get_label') and artist.get_label()=='centroid':
# artist.remove()
ax1.clear()
ax1.matshow(np.flip(new_image, axis=0),
cmap='gray',
origin='upper',
clim=zscale(new_image),
zorder=0)
ax1.scatter(
[new_image_halfwidth + 1], [new_image_halfwidth - 1],
marker='+',
s=120,
c='k',
zorder=1
) #ax1.scatter([popt[1]-x+new_image_halfwidth],[popt[2]-y+new_image_halfwidth],marker='+',s=120,c='k',zorder=1)
aperture_circle = plt.Circle((popt[1] - x + new_image_halfwidth,
popt[2] - y + new_image_halfwidth),
radius=aperture_radius,
linewidth=3,
color='hotpink',
fill=False,
lw=3,
zorder=2)
ax1.add_patch(aperture_circle)
inner_sky = plt.Circle((popt[1] - x + new_image_halfwidth,
popt[2] - y + new_image_halfwidth),
radius=inner_sky_radius,
linewidth=3,
color='lime',
fill=False,
zorder=2)
ax1.add_patch(inner_sky)
outer_sky = plt.Circle((popt[1] - x + new_image_halfwidth,
popt[2] - y + new_image_halfwidth),
radius=outer_sky_radius,
linewidth=3,
color='red',
fill=False,
zorder=2)
ax1.add_patch(outer_sky)
canvas.draw()
#update the radial plot
ax2.clear()
radial = radial_profile(new_image,
[new_image_halfwidth, new_image_halfwidth])
#perform the aperture photometry
#currently 2% different from IDL atv
aperture = CircularAperture((x, y), r=aperture_radius)
annulus = CircularAnnulus((x, y),
r_in=inner_sky_radius,
r_out=outer_sky_radius)
phot_table = aperture_photometry(data, [aperture, annulus])
#new background estimation
bkg_mask = np.ma.masked_outside(radial[0].reshape(new_image.shape),
inner_sky_radius, outer_sky_radius)
bkg_map = Background2D(new_image,
tuple(np.array(new_image.shape) / 4),
mask=bkg_mask.mask,
exclude_mesh_method='all')
bkg_map_med = MMMBackground().calc_background(
bkg_map.data) #bkg_map_med=np.median(bkg_map.background)
#print 'Map sky mean '+str(bkg_map_med)
#bkg_mean=phot_table['aperture_sum_1']/annulus.area()
#print 'Aperture sky mean '+str(bkg_mean)
#phot_table['residual_aperture_sum']=phot_table['aperture_sum_0']-bkg_mean*aperture.area()
phot_table['residual_aperture_sum'] = phot_table[
'aperture_sum_0'] - bkg_map_med * aperture.area()
#print 'Map sky result: '+str(phot_table['aperture_sum_0']-bkg_map_med*aperture.area())
#print "Aperture Photometry Result: "+str(phot_table['residual_aperture_sum'])
label8text.set('Sky Value: ' + str(int(bkg_map_med)))
label9text.set('Aperture Counts: ' +
str(int(phot_table['residual_aperture_sum'][0])))
label10text.set(
'Mag: ' +
str(-2.5 * np.log10(int(phot_table['residual_aperture_sum'][0])) +
25.)[:5])
ax2.scatter(radial[0], radial[1])
if var10.get() == 1:
ax2.plot(np.linspace(0, new_image_halfwidth, num=50),
gauss1d(np.linspace(0, new_image_halfwidth, num=50),
popt[0], 0, np.mean([popt[3], popt[4]]), popt[6]),
c='k',
lw=2)
ax2.text(0.5,
0.93,
'Gaussian FWHM: ' +
str(2.35482 * np.mean([popt[3], popt[4]]))[:5],
transform=ax2.transAxes,
fontsize=int(15 * scaling))
if var11.get() == 1:
moffat1d_guess = (np.amax(new_image) - bkg_map_med, 0, 3, 1,
bkg_map_med)
popt2, pcov2 = curve_fit(moffat1d,
radial[0],
radial[1],
p0=moffat1d_guess)
ax2.plot(np.linspace(0, new_image_halfwidth, num=50),
moffat1d(np.linspace(0, new_image_halfwidth,
num=50), popt2[0], popt2[1],
popt2[2], popt2[3], popt2[4]),
c='r',
lw=2)
ax2.text(
0.5,
0.85,
'Moffat FWHM: ' +
str(2.0 * popt2[2] * np.sqrt(2.0**(1. / popt2[3]) - 1.))[:5],
transform=ax2.transAxes,
fontsize=int(15 * scaling))
ax2.grid(True, color='white', linestyle='-', linewidth=1)
ax2.set_axisbelow(True)
ax2.autoscale(False)
ax2.set_xlim([0, new_image_halfwidth])
ax2.set_xlabel('Radius (Pixels)')
ax2.set_ylim([np.amin(radial[1]), np.amax(radial[1])])
ax2.set_axis_bgcolor('0.85')
ax2.axvline(aperture_radius, linewidth=2, color='hotpink')
ax2.axvline(inner_sky_radius, linewidth=2, color='lime')
ax2.axvline(outer_sky_radius, linewidth=2, color='red')
ax2.axhline(bkg_map_med, linewidth=2, color='yellow')
canvas2.draw()
def do_phot(img_file, write_out_back=False, write_out_residual=False,
write_out_epsf_img=True, write_out_epsf_file=True, write_out_psf_stars=True,
outdir='', subtract_back=False, fwhm_scale_psf=3.0, log=None,
star_param={'sharp_cut': 1.0, 'round_cut': 0.5, 'snthresh_psf': 25.0,
'fwhm_init': 5.0, 'snthresh_final': 5.0}):
stars = get_star_catalog(img_file, fwhm_init=star_param['fwhm_init'])
img_hdu = fits.open(img_file)
# Get image statistics
mask = img_hdu[0].data!=0.0
mean, median, std_sky = sigma_clipped_stats(img_hdu[0].data[mask],
sigma=5.0)
if log:
log.info('Found {0} stars'.format(len(stars)))
else:
print('Found {0} stars'.format(len(stars)))
stars = stars['xcentroid','ycentroid','fwhm','sharpness','roundness',
'npix','pa','flux','sky']
# Estimate the uncertainty from sky and flux values
stars['flux_err'] = np.sqrt(stars['flux']+stars['npix']*stars['sky'])
mask = ((stars['sharpness'] < np.median(stars['sharpness'])+np.std(stars['sharpness'])) &\
(stars['roundness'] < np.median(stars['roundness'])+3*np.std(stars['roundness'])) &\
(stars['roundness'] > np.median(stars['roundness'])-3*np.std(stars['roundness'])))
fwhm_stars = stars[mask]
m='Masked to {0} stars based on sharpness, roundness'
if log:
log.info(m.format(len(fwhm_stars)))
else:
print(m.format(len(fwhm_stars)))
fwhm_clipped, _, _ = sigmaclip(fwhm_stars['fwhm'])
fwhm = np.median(fwhm_clipped)
std_fwhm = np.std(fwhm_clipped)
mask = (fwhm_stars['fwhm'] > fwhm-3*std_fwhm) &\
(fwhm_stars['fwhm'] < fwhm+3*std_fwhm)
fwhm_stars = fwhm_stars[mask]
fwhm = np.median(fwhm_stars['fwhm'])
if log:
log.info('Masked to {0} stars based on FWHM'.format(len(fwhm_stars)))
else:
print('Masked to {0} stars based on FWHM'.format(len(fwhm_stars)))
step_size = 0.5
radii = np.arange(step_size, 2.5*fwhm, step_size)
x_data = np.arange(0, 2.5*fwhm, step_size)
apers_area = [np.pi*(step_size**2)]
for r in radii:
apers_area.append(np.pi*((r+step_size)**2 - r**2))
coords = [(fwhm_stars['xcentroid'][i],fwhm_stars['ycentroid'][i]) for i in range(len(fwhm_stars))]
apertures = [CircularAperture(coords, r=step_size)] # For circle aperture around center
for r in radii:
apertures.append(CircularAnnulus(coords, r_in=r, r_out=r+step_size)) # Annuli apertures
phot_table = aperture_photometry(img_hdu[0].data, apertures)
sigmas = []
for i, source in enumerate(phot_table):
sums = np.array([source[i] / apers_area[i - 3] for i in range(3, len(source))])
if median+10*std_sky <= sums[0] <= 60000 and median-3*std_sky < sums[-1] < median+3*std_sky:
w_fit = x_data[~np.isnan(sums)]
f_fit = sums[~np.isnan(sums)]
g, _ = curve_fit(fix_x0, w_fit, f_fit)
sigmas.append(np.abs(g[1]))
sigmas_post_clipping, _, _ = sigmaclip(sigmas, low=3, high=3)
fwhm = 2.35482*np.median(sigmas_post_clipping)
std_fwhm = np.std(sigmas_post_clipping)
if log:
log.info('FWHM={0}+/-{1}'.format('%2.4f'%fwhm,'%2.4f'%std_fwhm))
else:
print('FWHM={0}+/-{1}'.format('%2.4f'%fwhm,'%2.4f'%std_fwhm))
metadata={'FWHM':fwhm, 'EFWHM':std_fwhm, 'SKYADU': std_sky}
if subtract_back:
bkg = Background2D(img_hdu[0].data, (21,21), filter_size=(3,3))
backsub = img_hdu[0].data - bkg.background
ndimage = NDData(data=backsub)
backhdu = fits.PrimaryHDU(bkg.background)
backsubhdu = fits.PrimaryHDU(backsub)
else:
ndimage = NDData(data=img_hdu[0].data)
if write_out_back:
if log:
log.info('Writing out background and background-subtracted file...')
else:
print('Writing out background and background-subtracted file...')
back_file = img_file.replace('.fits','.back.fits')
backsub_file = img_file.replace('.fits','.backsub.fits')
if log:
log.info('Background file:',back_file)
log.info('Background-subtracted file:',backsub_file)
else:
print('Background file:',back_file)
print('Background-subtracted file:',backsub_file)
backhdu.writeto(back_file, overwrite=True)
backsubhdu.writeto(backsub_file, overwrite=True)
mask = (fwhm_stars['flux']/fwhm_stars['flux_err'] > star_param['snthresh_psf'])
bright = fwhm_stars[mask]
m='Masked to {0} PSF stars based on flux.'
if log:
log.info(m.format(len(bright)))
else:
print(m.format(len(bright)))
metadata['NPSFSTAR']=len(bright)
# Instantiate EPSF
size=int(fwhm*fwhm_scale_psf)
if size%2==0: size=size+1
epsf = generate_epsf(img_file, bright['xcentroid'],
bright['ycentroid'], size=size, oversampling=2, maxiters=5)
print('\n')
mask = (stars['flux']/stars['flux_err'] > star_param['snthresh_final'])
all_stars = stars[mask]
if log:
log.info('Final catalog is {0} stars'.format(len(all_stars)))
log.info('Getting final photometry...')
else:
print('Final catalog is {0} stars'.format(len(all_stars)))
print('Getting final photometry...')
photometry = run_photometry(img_file, epsf, fwhm, all_stars['xcentroid'],
all_stars['ycentroid'], subtract_back=subtract_back)
# Get RA/Dec from final positions
w = WCS(img_hdu[0].header)
coords = w.pixel_to_world(photometry['x_fit'], photometry['y_fit'])
# Join the photometry and all star catalogs and rename columns
photometry['FWHM'] = all_stars['fwhm']
photometry['PA'] = all_stars['pa']
photometry['NPIX'] = all_stars['npix']
photometry['SKY'] = all_stars['sky']
photometry['SHARP'] = all_stars['sharpness']
photometry['ROUND'] = all_stars['roundness']
photometry['SN'] = photometry['flux_fit']/photometry['flux_unc']
photometry['mag'] = -2.5*np.log10(photometry['flux_fit'])
photometry['mag_err'] = 2.5/np.log(10) * 1./photometry['SN']
photometry['RA'] = [c.ra.degree for c in coords]
photometry['Dec'] = [c.dec.degree for c in coords]
photometry.rename_column('x_fit', 'Xpos')
photometry.rename_column('y_fit', 'Ypos')
photometry.rename_column('x_0_unc', 'Xpos_err')
photometry.rename_column('y_0_unc', 'Ypos_err')
photometry.rename_column('flux_fit','flux')
photometry.rename_column('flux_unc','flux_err')
# Sort from brightest to faintest
photometry.sort('flux', reverse=True)
# Get final list of column names we want in catalog in order and the number
# of significant figures they should all have
colnames=['Xpos','Ypos','mag','mag_err','flux','flux_err','SN','SKY',
'FWHM','PA','SHARP','ROUND','NPIX','RA','Dec']
sigfig=[4,4,4,4,4,4,4,4,4,4,4,4,0,7,7]
if log:
log.info('Got {0} stars for final photometry'.format(len(photometry)))
else:
print('Got {0} stars for final photometry'.format(len(photometry)))
metadata['NOBJECT']=len(photometry)
# We should always write out catalog for stars
phot_file = os.path.join(outdir,img_file.replace('.fits','.pcmp'))
write_out_catalog(photometry, img_file, colnames, sigfig, phot_file,
metadata)
if write_out_psf_stars:
all_stars.sort('flux')
outname = os.path.join(outdir,img_file.replace('.fits','.psf.stars'))
all_stars.write(outname, format='ascii.no_header', overwrite=True)
if write_out_residual:
subdata = img_hdu[0].data
for row in result_tab:
subdata = subtract_psf(subdata, epsf, Table(row))
newhdu = fits.PrimaryHDU(subdata)
outname = os.path.join(outdir,
img_file.replace('.fits','.residual.fits'))
newhdu.writeto(outname, overwrite=True)
if write_out_epsf_img:
norm = simple_norm(epsf.data, 'log', percent=99.)
plt.imshow(epsf.data, norm=norm, origin='lower', cmap='viridis')
plt.colorbar()
outname = os.path.join(outdir, img_file.replace('.fits','.epsf.png'))
plt.savefig(outname)
plt.clf()
if write_out_epsf_file:
hdu = fits.PrimaryHDU(epsf.data)
hdu.header['FWHM']=fwhm
outname = os.path.join(outdir, img_file.replace('.fits','.psf.fits'))
hdu.writeto(outname, overwrite=True)
# Finally return EPSF
return(epsf, fwhm)