def estimate_sky_background(data,
method='scalar',
mask_sources=True,
path=None):
"""Estimate a scalar or image sky background with uncertainties.
Args:
data (np.array or str):
Image or data cube with sky observations used to extract the sky background mean and uncertainty. Str type
input is interpreted as a file name to read the data from.
method (str, optional):
Can be `scalar` or `image` for setting the shape of the output.
mask_sources (bool, optional):
Creates a source mask to exclude sources from the measurement. This should not be set if working in `image`
mode.
path (str, optional):
Path to the data files. This is used only if data is provided as a file name.
Returns:
mean (float or np.array):
Mean sky background as a scalar or image, depending on the method parameter.
std (float or np.array):
Uncertainty on the sky background estimate as a scalar or image, depending on the method parameter.
"""
# Handle str type data
if isinstance(data, str):
file = data
if path is not None:
file = os.path.join(path, file)
data = fits.getdata(filename=file)
# Create source mask
if mask_sources:
if data.ndim == 3:
mask = make_source_mask(np.sum(data, axis=0),
nsigma=2,
npixels=5,
dilate_size=11)
mask = np.repeat(np.expand_dims(mask, 0), data.shape[0], axis=0)
else:
mask = make_source_mask(data, nsigma=2, npixels=5, dilate_size=11)
else:
mask = None
# Derive statistics
if method == 'scalar':
mean, _, std = sigma_clipped_stats(data, sigma=3.0, mask=mask)
else:
raise NotImplementedError(
f"Method {method} is not implemented yet for sky background estimation!"
)
return mean, std
def create_flat(flat_list,fil,red_path,mbias=None,log=None):
log.info('Processing files for filter: '+fil)
log.info(str(len(flat_list))+' files found.')
flats = []
masks = []
flat_scale = []
for flat in flat_list:
log.info('Loading file: '+flat)
raw = CCDData.read(flat,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron)
log.info('Exposure time of image is '+str(red.header['TRUITIME']*red.header['COADDONE']))
mask = make_source_mask(red, nsigma=3, npixels=5)
bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
masked = np.array(red)
masked[mask] = bkg.background[mask]
log.info('Median flat level: '+str(np.median(masked)))
norm = 1/np.median(masked[224:1824,224:1824])
log.info('Flat normalization: '+str(norm))
flat_scale.append(norm)
flats.append(CCDData(masked,meta=red.header,unit=u.electron))
mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True)
log.info('Created master flat for filter: '+fil)
mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True)
log.info('Master flat written to mflat_'+fil+'.fits')
return
def findStars(image):
# In order to use the MAD as a consistent estimator for the estimation
# of the standard deviation σ, one takes σ = k * MAD where k is a constant
# scale factor, which depends on the distribution. For normally distributed
# data k is taken to be k = 1.48[26]
bkg_sigma = 1.48 * mad(image)
t = 5 * bkg_sigma
daofind = DAOStarFinder(fwhm=3.0, threshold=t)
stars = daofind(image)
#stars['signal'] = stars['flux'] * t
#
data = image
mask = make_source_mask(data, snr=2, npixels=5, dilate_size=11)
mean, median, std = sigma_clipped_stats(data, sigma=3.0, mask=mask)
madstd = mad_std(data)
snrs = []
for peak in stars['peak']:
snrs.append(peak / madstd / 7.4)
stars['snr'] = snrs
print((mean, median, std, bkg_sigma, t, madstd))
#
#print stars
return stars
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction, dark subtraction, overscan correction, flat correction, and trimming image.')
raw = CCDData.read(sci,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of science image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)#dark_exposure=1*u.second, data_exposure=red.header['EXPTIME']*u.second, scale=True)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
red = ccdproc.flat_correct(red, mflat)
processed_data = ccdproc.ccd_process(red, trim=raw.header['DATASEC'])
log.info('File proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci,processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(red.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (510, 510), filter_size=(9, 9),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits'),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['EXPTIME']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/'),overwrite=True)
processed.append(final)
return processed, masks
def create_flat(flat_list,fil,red_path,mbias=None,log=None):
log.info('Processing files for filter: '+fil)
log.info(str(len(flat_list))+' files found.')
flats = []
flat_scale = []
for flat in flat_list:
log.info('Loading file: '+flat)
raw = CCDData.read(flat,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
mask = make_source_mask(red, nsigma=3, npixels=5)
bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
masked = np.array(red)
masked[mask] = bkg.background[mask]
log.info('Median flat level: '+str(np.median(masked)))
norm = 1/np.median(masked[500:1500,500:1500])
log.info('Flat normalization: '+str(norm))
flat_scale.append(norm)
flats.append(CCDData(masked,meta=red.header,unit=u.electron))
mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True)
log.info('Created master flat for filter: '+fil)
mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True)
log.info('Master flat written to mflat_'+fil+'.fits')
return
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction and flat correction.')
with fits.open(sci) as hdr:
header = hdr[0].header
data = hdr[0].data
data[np.isnan(data)] = np.nanmedian(data)
raw = CCDData(data,meta=header,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron)
log.info('Exposure time of science image is '+str(red.header['TRUITIME']*red.header['COADDONE']))
flat = np.array(ccdproc.flat_correct(red, mflat))
flat[np.isnan(flat)] = np.nanmedian(flat)
processed_data = CCDData(flat,unit=u.electron,header=red.header,wcs=red.wcs)
log.info('File proccessed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci.replace('.gz',''),processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(raw.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (128, 128), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits').replace('.gz',''),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['TRUITIME']*red.header['COADDONE']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/').replace('.gz',''),overwrite=True)
processed.append(final)
return processed, masks
def make_side_msk(img, snr=2.5, npixels=5, dilate_size=5, ct_QSO_mask=False):
from photutils import make_source_mask
mask_o = make_source_mask(img,
snr=snr,
npixels=npixels,
dilate_size=dilate_size)
exp_center = np.zeros_like(mask_o)
cent = len(exp_center) / 2
exp_center[cent, cent] = 1
count = [-1, 0, 1]
i = 0
exp_center_comp = np.zeros_like(mask_o)
while exp_center_comp.sum() != exp_center.sum():
exp_center_comp = copy.deepcopy(exp_center)
for i in range(exp_center.sum()):
x, y = np.where(exp_center == 1)[0][i], np.where(
exp_center == 1)[1][i]
for j in count:
for k in count:
if exp_center[x + j, y + k] == 0 and mask_o[x + j,
y + k] == 1:
exp_center[x + j, y + k] = 1
if ct_QSO_mask == False:
mask_ = mask_o * ~exp_center
return ~mask_
elif ct_QSO_mask == True:
return exp_center
def __init__(self, gid, field, flter, nearby_fitting_region, nearby_gids = None):
if not (field == 'S' or field == 'N'):
raise Exception('Field must be S or N.')
if not type(flter) == str:
raise Exception('Filter must be string.')
self.gid = gid
self.field = field
self.flter = flter
self.nearby_fitting_region = nearby_fitting_region
if self.flter == '850':
if self.field == 'N':
self.fitsfile = 'GOODS-N_acsz_sci_sub.fits'
self.fullfield = 'North'
if self.field == 'S':
self.fitsfile = 'goodss_3dhst.v4.0.F850LP_orig_sci.fits'
self.fullfield = 'South'
else:
if self.field =='S':
self.fitsfile = 'goodss_3dhst.v4.0.F'+flter+'W_orig_sci.fits'
self.fullfield = 'South'
if self.field =='N':
self.fitsfile = 'goodsn_3dhst.v4.0.F'+flter+'W_orig_sci.fits'
self.fullfield = 'North'
### CATALOG ###
if (field == 'S') or (field == 'South'):
catalog = '/Users/rosaliaobrien/research/website/catalogs/goodss_3dhst.v4.4.cat'
if (field == 'N') or (field == 'North'):
catalog = '/Users/rosaliaobrien/research/website/catalogs/goodsn_3dhst.v4.4.cat'
self.cat = ascii.read(catalog)
### GET SKY ESTIMATE ###
if not nearby_gids == None:
xmin, xmax, ymin, ymax = self.get_boundaries(nearby_gids)
path = '/Users/rosaliaobrien/research/GALFIT_CLEAR/running_GALFIT/'+self.fullfield+'/'+self.flter+'/'
skydata = fits.open(path+self.fitsfile)[0].data
xmid = (xmin+xmax)/2
ymid = (ymin+ymax)/2
cutout = Cutout2D(skydata, (xmid, ymid), (400, 400), copy=True, mode='partial')
masked_cutout = make_source_mask(cutout.data, snr = 1.5,
npixels=10, dilate_size=11)
cutout.data[masked_cutout] = np.nan
self.scidata = cutout.data
sigma_clip = SigmaClip(sigma=3)
bkg = SExtractorBackground(sigma_clip)
rms = MADStdBackgroundRMS(sigma_clip)
self.sky_value = bkg.calc_background(cutout.data)
self.rms_value = rms.calc_background_rms(cutout.data)
def estimate_background(im):
# im must be BW
if len(im.shape) == 3: im=im.squeeze()
mask = make_source_mask(im, snr=2, npixels=5, dilate_size=11)
mean, median, std = sigma_clipped_stats(im, sigma=3.0, mask=mask)
flat_background = np.ones_like(im) * mean
fake_background = np.random.normal(mean, std, size=im.shape)
return flat_background, fake_background, std
def reduce_object(self, object_frame, flatfield, master_dark, bkg_method="mesh"):
print("Opening object frame", object_frame)
obj_frame = fits.open(object_frame)
obj_frame_data = obj_frame[0].data
obj_frame_header = obj_frame[0].header
# Subtract master dark
print("Opening master dark frame")
master_dark = fits.open(master_dark)
master_dark_data = master_dark[0].data
master_dark_header = master_dark[0].header
print("Subtracting master dark from object")
reduced_obj_frame_data = obj_frame_data - master_dark_data
# Flatfield correct
print("Opening flatfield frame")
flatfield = fits.open(flatfield)
flatfield_data = flatfield[0].data
flatfield_header = flatfield[0].header
print("Dividing object by flatfield")
reduced_obj_frame_data /= flatfield_data
# Remove cosmic rays
#sepmed = False
#cleantype = "medmask"
#print("Detecting cosmic rays")
#artifact_mask, reduced_obj_frame_data = astroscrappy.detect_cosmics(reduced_obj_frame_data, sepmed=sepmed, cleantype=cleantype)
# Subtract background
if bkg_method == "mesh":
nsigma = 2.0
npixels = 5
dilate_size = 31
print("Creating mask")
mask = make_source_mask(reduced_obj_frame_data, nsigma=nsigma, npixels=npixels, dilate_size=dilate_size)
print("Subtracting background mesh")
background = sep.Background(reduced_obj_frame_data, mask=mask)
reduced_obj_frame_data -= background
elif bkg_method == "sigma":
print("Subtracting sigma-clipped background")
mean, median, std = sigma_clipped_stats(reduced_obj_frame_data, sigma=bkg_sigma)
reduced_obj_frame_data -= median
print("Converting reduced array to FITS")
reduced_hdu = fits.PrimaryHDU(reduced_obj_frame_data, header=obj_frame_header)
return reduced_hdu
def set_mask(
self,
*args,
method: str = "source",
inner: float = None,
outer: float = None,
position: SkyCoord = None,
wcs: WCS = None,
**kwargs,
):
"""
Set a mask to the data image
:param method: the method to set mask, "source", or "annulus"defaults to "source"
:type method: str, optional
:param inner: the inner radius of the annulus, defaults to None
:type inner: float, optional
:param outer: the outer radius of the annulus, defaults to None
:type outer: float, optional
:param position: the center of the annulus, defaults to None
:type position: SkyCoord, optional
:param wcs: the wcs used to convert mask to pixel, defaults to None
:type wcs: WCS, optional
:raises ValueError: "annulus" must have position and inner/outer radius
"""
if method == "source":
self._mask = make_source_mask(self.data, *args, **kwargs)
elif method == "annulus":
if not position or not inner or not outer:
raise ValueError(
"Use annulus mask but missing position or inner/outer radius"
)
else:
if not wcs:
logging.warning(
"Not assign the wcs, try the wcs from the header")
wcs = self.header
mask = (SkyCircularAnnulus(
position, inner * u.arcsec,
outer * u.arcsec).to_pixel(wcs).to_mask("center"))
self._mask = self.ImageMask(shape="annulus",
position=position,
rin=inner,
rout=outer,
mask=mask)
# self.mask = self.aper.to_mask("center")
else:
raise ValueError(f"No supported method: {method}")
def Background_DilatedSources(IMG, results, options):
"""
Compute a global background value for an image. Performed by
identifying pixels which are beyond 3 sigma above the average
signal and masking them, also further masking a boarder
of 20 pixels around the initial masked pixels. Returns a
dictionary of parameters describing the background level.
"""
# Mask main body of image so only outer 1/5th is used
# for background calculation.
if 'mask' in results and not results['mask'] is None:
mask = results['mask']
else:
mask = np.zeros(IMG.shape)
mask[int(IMG.shape[0] / 5.):int(4. * IMG.shape[0] / 5.),
int(IMG.shape[1] / 5.):int(4. * IMG.shape[1] / 5.)] = 1
# Run photutils source mask to remove pixels with sources
# such as stars and galaxies, including a boarder
# around each source.
if not ('ap_set_background' in options
and 'ap_set_background_noise' in options):
source_mask = make_source_mask(
IMG,
nsigma=3,
npixels=int(1. / options['ap_pixscale']),
dilate_size=40,
filter_fwhm=1. / options['ap_pixscale'],
filter_size=int(3. / options['ap_pixscale']),
sigclip_iters=5)
mask = np.logical_or(mask, source_mask)
# Return statistics from background sky
bkgrnd = options[
'ap_set_background'] if 'ap_set_background' in options else np.median(
IMG[np.logical_not(mask)])
noise = options[
'ap_set_background_noise'] if 'ap_set_background_noise' in options else iqr(
IMG[np.logical_not(mask)], rng=[16, 84]) / 2
uncertainty = noise / np.sqrt(np.sum(np.logical_not(mask)))
return IMG, {
'background':
bkgrnd,
'background noise':
noise,
'background uncertainty':
uncertainty,
'auxfile background':
'background: %.5e +- %.2e flux/pix, noise: %.5e flux/pix' %
(bkgrnd, uncertainty, noise)
}
def sub_bkg(self, img):
sigma_clip = SigmaClip(sigma=3.)
bkg_estimator = SExtractorBackground()
mask_0 = make_source_mask(img, nsigma=3, npixels=5, dilate_size=11)
mask_1 = (np.isnan(img))
mask = mask_0 + mask_1
bkg = Background2D(img, (5, 5),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask)
back = bkg.background * ~mask_1
return img - back
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation.
"""
self.log.info(f"Running {self.__class__.__name__} action")
snr = 5
npixels = 5
self.action.args.source_mask = [None] * len(
self.action.args.kd.pixeldata)
for i, pd in enumerate(self.action.args.kd.pixeldata):
source_mask = photutils.make_source_mask(pd, snr, npixels)
self.action.args.source_mask[i] = source_mask
return self.action.args
def est_bkg(image, pltshow=1):
print("Estimating the background light......")
img = image # check the back grounp
sigma_clip = SigmaClip(sigma=3., iters=10)
bkg_estimator = SExtractorBackground()
mask_0 = make_source_mask(img, snr=2, npixels=5, dilate_size=11)
mask_1 = (np.isnan(img))
mask = mask_0 + mask_1
bkg = Background2D(img, (50, 50),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask)
from matplotlib.colors import LogNorm
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(img, norm=LogNorm(), origin='lower')
#bkg.plot_meshes(outlines=True, color='#1f77b4')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if pltshow == 0:
plt.close()
else:
plt.show()
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(mask, origin='lower')
#bkg.plot_meshes(outlines=True, color='#1f77b4')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if pltshow == 0:
plt.close()
else:
plt.show()
back = bkg.background * ~mask_1
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(back, origin='lower', cmap='Greys_r')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if pltshow == 0:
plt.close()
else:
plt.show()
# img -= back
# pyfits.PrimaryHDU(img).writeto('sub_coadd.fits',overwrite=True)
return back
def bkg_subs(image, box, snr=2, npixels=5, dilate_size=15):
"""Substrackt the background of an image. Sources are idenfified first and
masked, then the background is estimated as the (sigma-clipped) median.
"""
# get data from file
data = fits.getdata(image)
# create mask
nan_mask = np.isnan(data)
source_mask = make_source_mask(data, snr=snr, npixels=npixels,
dilate_size=dilate_size)
mask = np.logical_or(nan_mask, source_mask)
bkg_estimator = MedianBackground()
bkg = Background2D(data, box, filter_size=1, bkg_estimator=bkg_estimator,
mask=mask)
return data - bkg.background
def mask_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Masking sources ...")
# Create sources mask
mask = make_source_mask(self.sources_with_noise.data, snr=2, npixels=5, dilate_size=11, mask=self.rotation_mask)
self.sources_mask = Mask(mask)
# Plot
if self.config.plot: plotting.plot_mask(self.sources_mask, title="sources mask")
def _perform(self):
"""
Returns an Argument() with the parameters that depend on this
operation.
"""
self.log.info(f"Running {self.__class__.__name__} action")
snr = self.cfg['Extract'].getfloat('source_mask_snr', 5)
self.log.debug(f" Using snr = {snr}")
npixels = self.cfg['Extract'].getfloat('source_mask_npixels', 5)
self.log.debug(f" Using npixels = {npixels}")
source_mask = photutils.make_source_mask(self.action.args.ccddata, snr,
npixels)
self.action.args.source_mask = source_mask
return self.action.args
def sub_bkg(img, plot=True):
from astropy.stats import SigmaClip
from photutils import Background2D, SExtractorBackground
sigma_clip = SigmaClip(sigma=3., iters=10)
bkg_estimator = SExtractorBackground()
from photutils import make_source_mask
mask_0 = make_source_mask(img, snr=2, npixels=5, dilate_size=11)
mask_1 = (np.isnan(img))
mask = mask_0 + mask_1
bkg = Background2D(img, (50, 50),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask)
from matplotlib.colors import LogNorm
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(img, norm=LogNorm(), origin='lower')
#bkg.plot_meshes(outlines=True, color='#1f77b4')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if plot:
plt.show()
else:
plt.close()
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(mask, origin='lower')
#bkg.plot_meshes(outlines=True, color='#1f77b4')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if plot:
plt.show()
else:
plt.close()
back = bkg.background * ~mask_1
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(back, origin='lower', cmap='Greys_r')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if plot:
plt.show()
else:
plt.close()
return img - back, back
def starphot(imdata, position, radius, r_in, r_out):
"""
sources: http://photutils.readthedocs.io/en/stable/
photutils/aperture.html
ARGS:
imdata: Numpy array containing the star.
position: [x,y] coordinates where x corresponds to the second index of
imdata
radius: Radius of the circular aperture used to compute the flux of
the star.
r_in: Inner radius of the annulus used to compute the background
mean.
r_out: Outer radius of the annulus used to compute the background
mean.
Returns [flux, background variance, background mean]
"""
try:
statmask = photutils.make_source_mask(imdata,
snr=5,
npixels=5,
dilate_size=10)
except TypeError:
return None
bkg_annulus = photutils.CircularAnnulus(position, r_in, r_out)
bkg_phot_table = photutils.aperture_photometry(imdata,
bkg_annulus,
method='subpixel',
mask=statmask)
bkg_mean_per_pixel = bkg_phot_table['aperture_sum'] / bkg_annulus.area()
src_aperture = photutils.CircularAperture(position, radius)
src_phot_table = photutils.aperture_photometry(imdata,
src_aperture,
method='subpixel')
signal = src_phot_table['aperture_sum'] - bkg_mean_per_pixel*\
src_aperture.area()
#noise_squared = signal + bkg_mean_per_pixel*src_aperture.area()
mean, median, std = sigma_clipped_stats(imdata,
sigma=3.0,
iters=5,
mask=statmask)
noise_squared = std**2
return float(str(signal.data[0])), noise_squared,\
float(str(bkg_mean_per_pixel.data[0]))
def subtract_sky_ccd_mask(ccd, skysub='SKYSUB', skyval='SKYVAL', snr=3, npixels=5, dilate_size=11, sigma=3, check_skysub=True, max_val=10000):
if check_skysub and skysub is not None and skysub in ccd.header and ccd.header[skysub]:
return ccd
if not check_skysub and skysub is not None and skysub in ccd.header and ccd.header[skysub]:
print ('WARNING: Image already sky subtracted! Subtracting AGAIN the sky! [set check_skysub = True for checking]')
mask = make_source_mask(ccd.data, snr=sigma, npixels=npixels, dilate_size=dilate_size)
mean, median, std = sigma_clipped_stats(ccd.data, sigma=sigma, mask=mask)
sky = median
if max_val is not None and sky >= max_val:
if 'FILENAME' in ccd.header:
print ('WARNING: File "%s" has high sky level (%s >= %s). Correction NOT applied' % (ccd.header['FILENAME'], sky, max_val))
else:
print ('WARNING: File has high sky level (%s >= %s). Correction NOT applied' % (sky, max_val))
return ccd
ccd.data -= sky
ccd.header[skysub] = True
ccd.header[skyval] = sky
ccd.header['SKYTYPE'] = '1D'
return ccd
def bkgstd(im_data, mask):
"""Estimate the RMS standard deviation of the background in the `im_data`
image.
Arguments
---------
im_data : array_like
Image data
mask : array_like
Mask to apply to image
Returns
-------
np.ndarray
RMS standard deviation of the background image
"""
# use crude image segmentation to find sources above SNR=3, build a
# source mask, and estimate the background RMS
source_mask = make_source_mask(im_data, snr=3, npixels=5,
dilate_size=15, mask=mask)
# combine the bad pixel mask and source mask
rough_mask = np.logical_or(mask, source_mask)
# estimate the background standard deviation
try:
sigma_clip = SigmaClip(sigma=3, maxiters=5) # sigma clipping
except TypeError: # in old astropy, "maxiters" was "iters"
sigma_clip = SigmaClip(sigma=3, iters=5)
try:
bkg = Background2D(im_data, (50,50), filter_size=(5,5),
sigma_clip=sigma_clip,
bkg_estimator=MedianBackground(),
mask=rough_mask)
except ValueError:
e = sys.exc_info()
print("\nWhile attempting background estimation on the "+
"science image, the following error was raised: "+
f"\n{str(e[0])}\n{str(e[1])}\n--> exiting.")
return
return bkg.background_rms
def mask_sources(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Masking sources ...")
# Create sources mask
mask = make_source_mask(self.sources_with_noise.data,
snr=2,
npixels=5,
dilate_size=11,
mask=self.rotation_mask)
self.sources_mask = Mask(mask)
# Plot
if self.config.plot:
plotting.plot_mask(self.sources_mask, title="sources mask")
def detect_sources(image):
'''
By Anna Marini
Extract the light sources from the image
'''
# threshold = detect_threshold(image, nsigma=2.)
# sigma = 3.0 * gaussian_fwhm_to_sigma # FWHM = 3.
# kernel = Gaussian2DKernel(sigma, x_size=3, y_size=3)
# kernel.normalize()
if isinstance(image, str):
image = get_fits_data(image)
mask = make_source_mask(image, nsigma=2, npixels=5, dilate_size=11)
mean, median, std = sigma_clipped_stats(image, sigma=3, mask=mask)
daofind = DAOStarFinder(fwhm=3.0, threshold=5. * std)
sources = daofind(image - median)
# Pixel coordinates of the sources
x = np.array(sources['xcentroid'])
y = np.array(sources['ycentroid'])
return x, y
def background_median(data, sigma=3, npixels=5, maxiters=20, **kwargs):
"""Calculate background median for data.
Calculates the background median for an image by iteratively sigma
clipping. It first creates a source mask using segmentation and binary
dilation (see https://photutils.readthedocs.io/en/stable/api/photutils.make_source_mask.html)
for more details, before iteratively calculating the sigma-clipped
median of the data.
Args:
data (str or numpy.ndarray or astropy.io.fits.PrimaryHDU): Data
to calculate the background median for. If provided as a
string, this is interpreted as a filename for a .fits HDU.
sigma (float, optional): Level to perform sigma-clipping to.
Defaults to 3.
npixels (int, optional): Number of connected pixels greater than
sigma to consider for a pixel to be part of a source when
masking. Defaults to 5.
maxiters (int,optional): The maximum number of sigma-clipping
iterations to perform. Defaults to 20.
Returns:
float: Background median of the data.
"""
if isinstance(data, str):
hdu = fits.open(data)[0]
data = hdu.data
elif isinstance(data, fits.PrimaryHDU):
data = data.data
mask = make_source_mask(data, snr=sigma, npixels=npixels, **kwargs)
_, median, _ = sigma_clipped_stats(data,
mask=mask,
sigma=sigma,
maxiters=maxiters,
**kwargs)
return median
def getstats(img, ff):
gd = np.where(img != 0)
print('there are ', len(gd), ' elements')
arr = img[gd]
arr = sorted(arr)
n = len(arr)
print('array is ', n, ' elements')
i = round(ff * n)
vmax = arr[i]
print(ff, ' signal range value is ', vmax)
print('making mask')
mask = make_source_mask(img, snr=2, npixels=5, dilate_size=11)
print('calculating stats')
vmean, vmedian, vstd = sigma_clipped_stats(img,
sigma=3.0,
mask=mask,
mask_value=0.)
print('mean: ', vmean)
print('median: ', vmedian)
print('sigma: ', vstd)
return vmean, vmedian, vstd, vmax
def background_subtract(im_data):
"""calculates background using a mask routine from photutils.
Parameters
----------
im_data : numpy array
requires an image data array
returns
------
output_im: numpy array
The background subtracted data.
mask: numpy array bool
The mask used to shield the object
std: float
The standard deviation on the background
"""
# import numpy as np
# from astropy.io import fits
# store the data from the HDU argument
# im_data = HDU.data
# Generate mask
from photutils import make_source_mask
from astropy.stats import sigma_clipped_stats
mask = make_source_mask(im_data, snr=2, npixels=5, dilate_size=11)
# calculate bias using mean
# clipped stats are used, just in case
mean, median, std = sigma_clipped_stats(im_data, sigma=3.0, mask=mask)
print('Background mean: ' + str(mean))
print('Background median: ' + str(median))
print('Background standerd deviation: ' + str(std))
output_im = im_data - mean
return output_im, mask, std
def cc_spec(image,width,p_instr,theta_instr):
# global make_source_mask
#crop the images with separate o and e ray sizex is the cropped half xcoordinate sizey1,sizey2 the boundary in y direction
o_ray=[] ; e_ray=[]; noise=[]; ny=240
for img in image:
img=rebin(img[:,20:280],[240,130])
#print 'size of the image', img.shape
cen_o=int(round(101.0/(240/ny)))
cen_e=int(round(135/(240/ny)))
o_ray.append(img[(cen_o-width):(cen_o+width),:])
e_ray.append(img[(cen_e-width):(cen_e+width),:])
mask = make_source_mask(img, snr=1.5, npixels=10, dilate_size=10)
maskr= np.invert(mask)
mean, median, std = sigma_clipped_stats(img, sigma=2.3, iters=8, mask=mask)
noise.append(std)
xc=[];yc=[];crop_image_o=[];crop_image_e=[]
I_1=np.mean(o_ray[0]+e_ray[0],axis=0); Q_1=np.mean(o_ray[0]-e_ray[0],axis=0)
I_2=np.mean(o_ray[1]+e_ray[1],axis=0); Q_2=np.mean(o_ray[1]-e_ray[1],axis=0)
I_3=np.mean(o_ray[2]+e_ray[2],axis=0); U_1=np.mean(o_ray[2]-e_ray[2],axis=0)
I_4=np.mean(o_ray[3]+e_ray[3],axis=0); U_2=np.mean(o_ray[3]-e_ray[3],axis=0)
Q=(Q_1-Q_2)/2.0
U=(U_1-U_2)/2.0
I=(I_1+I_2+I_3+I_4)/4.0
n=I_1.shape
###Instrument correction
U_c=U-p_instr*np.cos(2.0*theta_instr*np.pi/180.0)*I
Q_c=Q-p_instr*np.sin(2.0*theta_instr*np.pi/180.0)*I
factor=((2.0*width+1.0)**0.5)
err_Q=(noise[0]+noise[1])/(2.0*factor)+np.zeros((n[0]))
#Q_1 noise =sqrt(2)*noise[0] Q noise=Q_1 noise/sqrt(2)
err_U=(noise[2]+noise[3])/(2.0*factor)+np.zeros((n[0]))
#same
err_I=np.mean(noise)/((2.0)**0.5*factor)+np.zeros((n[0]))
return Q, U, I, err_Q, err_U, err_I
def extract_source(images):
# Calculate image noise
stack = np.sum(images, axis = 0)
mask = make_source_mask(stack, snr=2, npixels=5, dilate_size=11)
mean, median, std = sigma_clipped_stats(stack, sigma=3.0, mask=mask)
# Pull out all sources
segm = detect_sources(stack, 2*std, npixels = 5)
# Screen for our source
cx = int(stack.shape[0]/2)
cy = int(stack.shape[1]/2)
center_pixel = segm.data[cx,cy]
segm.data[np.where(segm.data != center_pixel)] = 0
region = np.zeros(stack.shape, dtype = bool)
region[segm.data == center_pixel] = True
cutout = np.where(region)
pad = 3
xmin,xmax = min(cutout[0]) - pad, max(cutout[0]) + pad
ymin,ymax = min(cutout[1]) - pad, max(cutout[1]) + pad
rect_region = [xmin,xmax,ymin,ymax]
dof2 = (xmax - xmin)*(ymax - ymin)
dof = len(stack[region])
fig = plt.figure()
ax1 = plt.subplot(121)
ax1.imshow(stack[xmin:xmax,ymin:ymax], interpolation = 'none', cmap = 'gray')
ax2 = plt.subplot(122)
ax2.imshow(segm.data[xmin:xmax,ymin:ymax], interpolation = 'none')
plt.show()
return region, rect_region, std, dof
def bkgsub(im_file, mask_file=None,
bkg_box=(5,5), bkg_filt=(5,5),
plot_bkg=False, plot_bkgsubbed=False,
scale_bkg="linear", scale_bkgsubbed="linear",
write=True, output=None):
"""Subtract the background from an image.
Arguments
---------
im_file : str
Filename for image of interest
mask_file : str, optional
Filename for bad pixels mask (default None)
bkg_box : tuple, optional
Sizes of box along each axis in which background is estimated (default
(5,5))
bkg_filt : tuple, optional
Size of the median filter pre-applied to the background before
background estimation (default (5,5) where (1,1) --> no filtering)
plot_bkg : bool, optional
Whether to plot BACKGROUND image (default False)
plot_bkgsubbed : bool, optional
Whether to plot background-SUBTRACTED image (default False)
scale_bkg : {"linear", "log", "asinh"}, optional
Scale to apply to BACKGROUND image plot (default "linear")
scale_bkgsubbed : {"linear", "log", "asinh"}, optional
Scale to apply to background-SUBTRACTED image plot (default "linear")
write : bool, optional
Whether to write the background-SUBTRACTED image to a fits file
(default True)
output : str, optional
Name for output fits file (default
`im_file.replace(".fits", "_bkgsub.fits")`)
Returns
-------
astropy.io.fits.PrimaryHDU
New HDU (image + header) with the image background-subtracted
Notes
-----
**TO-DO:**
- Allow naming of output plots
"""
## load in the image data
image_data = fits.getdata(im_file)
## source detection and building masks
if mask_file: # load a bad pixel mask if one is provided
bp_mask = fits.getdata(mask_file).astype(bool)
zeromask = image_data == 0 # mask out pixels equal to 0
nansmask = np.isnan(image_data) # mask out nans
bp_mask = np.logical_or.reduce((bp_mask, zeromask, nansmask))
#bp_mask = np.logical_or(bp_mask, zeromask) # old way: chained
#bp_mask = np.logical_or(bp_mask, nansmask)
else:
zeromask = image_data == 0 # mask out pixels equal to 0
nansmask = np.isnan(image_data) # mask out nans
bp_mask = np.logical_or(nansmask, zeromask)
# make a crude source mask
source_mask = make_source_mask(image_data, snr=3, npixels=5,
dilate_size=15, mask=bp_mask)
# combine the bad pixel mask and source mask for background subtraction
# make the final mask
mask = np.logical_or(bp_mask, source_mask)
## estimate the background
try:
sigma_clip = SigmaClip(sigma=3, maxiters=5) # sigma clipping
except TypeError: # in old astropy, "maxiters" was "iters"
sigma_clip = SigmaClip(sigma=3, iters=5)
bkg = Background2D(image_data, box_size=bkg_box, filter_size=bkg_filt,
sigma_clip=sigma_clip, bkg_estimator=MedianBackground(),
mask=mask)
bkg_img = bkg.background
bkgstd = bkg.background_rms_median
## subtract the background
bkgsub_img = image_data - bkg_img
bkgstd = bkg.background_rms_median # save this quantity to write to header
## finally, mask bad pixels
# all bad pix are then set to 0 for consistency
bkg_img_masked = np.ma.masked_where(bp_mask, bkg_img)
bkg_img = np.ma.filled(bkg_img_masked, 0)
bkgsub_img_masked = np.ma.masked_where(bp_mask, bkgsub_img)
bkgsub_img = np.ma.filled(bkgsub_img_masked, 0)
## plotting (optional)
if plot_bkg: # plot the background, if desired
output_bkg = f"background_{scale_bkg}.png"
__plot_bkg(im_header=fits.getheader(im_file),
bkg_img_masked=bkg_img_masked,
scale=scale_bkg,
output=output_bkg)
if plot_bkgsubbed: # plot the background-subtracted image, if desired
output_bkgsubbed = f"background_subbed_{scale_bkgsubbed}.png"
__plot_bkgsubbed(im_header=fits.getheader(im_file),
bkgsub_img_masked=bkgsub_img_masked,
scale=scale_bkgsubbed,
output=output_bkgsubbed)
## building the final HDU
hdr = fits.getheader(im_file)
hdr["BKGSTD"] = bkgstd # useful header for later
bkgsub_hdu = fits.PrimaryHDU(data=bkgsub_img, header=hdr)
if write: # if we want to write the background-subtracted fits file
if not(output): # if no output name given, set default
output = im_file.replace(".fits", "_bkgsub.fits")
bkgsub_hdu.writeto(output, overwrite=True, output_verify="ignore")
return bkgsub_hdu
elif cut < 0:
PSF = PSF[:, -cut:cut]
PSF /= PSF.sum()
if PSF.shape[0] != PSF.shape[1]:
raise ValueError("PSF shape is not a square.")
PSF_list.append(PSF)
zp_list.append(zp)
#==============================================================================
# quickly evaluate the background rms
#==============================================================================
background_rms_list = []
for i in range(len(band_seq)):
if i in run_list:
mask = make_source_mask(QSO_im_list[i],
snr=3,
npixels=5,
dilate_size=11)
background_rms_list.append(np.std(QSO_im_list[i] * (1 - mask * 1)))
else:
background_rms_list.append([])
fit_frame_size = 81
#==============================================================================
# Start set up for fitting:
#==============================================================================
psf_l, QSO_img_l, QSO_std_l = [], [], []
for k in range(len(band_seq)):
if k in run_list:
ct = int((len(QSO_im_list[k]) - fit_frame_size) /
2) # If want to cut to 61, QSO_im[ct:-ct,ct:-ct]
bt = np.int(np.round(0.5*cy))
tp = bt + 1000
tmpImg = tmpImg[bt:tp, lf:rt]
# Grab the on-off target value for this image
thisAB = subGroup['AB'][iFile]
# Place the image in a list and store required background values
if thisAB == 'B':
# Place B images in the BimgList
BimgList.append(tmpImg)
# Place the median value of this off-target image in list
mask = make_source_mask(
tmpImg.data, snr=2, npixels=5, dilate_size=11
)
mean, median, std = sigma_clipped_stats(
tmpImg.data, sigma=3.0, mask=mask
)
BbkgList.append(median)
# Place the time of this image in a list of time values
BdatetimeList.append(tmpImg.julianDate)
if thisAB == 'A':
# Read in any associated masks and store them.
maskFile = os.path.join(maskDir, os.path.basename(filename))
# If there is a mask for this file, then apply it!
if os.path.isfile(maskFile):
