def get_bad_pixels(calib_file=None,
nsigma_gain=5,
nsigma_elecnoise=5,
dark_histo=None,
nsigma_dark=8,
plot="none",
output=None):
bad_pix = np.array([], dtype=int)
if calib_file is not None:
with open(calib_file) as file:
calibration_parameters = yaml.load(file)
gain = np.array(calibration_parameters['gain'])
elecnoise = np.array(calibration_parameters['sigma_e'])
nan_mask = np.logical_or(~np.isfinite(gain), ~np.isfinite(elecnoise))
sigclip_gain = SigmaClip(sigma=nsigma_gain,
iters=10,
cenfunc=np.nanmean,
stdfunc=np.nanstd)
sigclip_elecnoise = SigmaClip(sigma=nsigma_elecnoise,
iters=10,
cenfunc=np.nanmean,
stdfunc=np.nanstd)
gain_mask = sigclip_gain(gain).mask
elecnoise_mask = sigclip_elecnoise(elecnoise).mask
bad_mask = np.logical_or(nan_mask,
np.logical_or(gain_mask, elecnoise_mask))
bad_pix = np.where(bad_mask)[0]
if output is not None:
calibration_parameters['bad_pixels'] = bad_pix
with open(output, 'w') as file:
yaml.dump(calibration_parameters, file)
if plot is not None:
fig, (ax_gain, ax_elecnoise) = plt.subplots(2, 1)
gain_bins = np.linspace(np.nanmin(gain), np.nanmax(gain), 100)
ax_gain.hist(gain[~bad_mask], gain_bins, color='b')
ax_gain.hist(gain[bad_mask], gain_bins, color='r')
ax_gain.set_xlabel('integral gain [LSB p.e.$^{-1}$]')
elecnoise_bins = np.linspace(np.nanmin(elecnoise),
np.nanmax(elecnoise), 100)
ax_elecnoise.hist(elecnoise[~bad_mask], elecnoise_bins, color='b')
ax_elecnoise.hist(elecnoise[bad_mask], elecnoise_bins, color='r')
ax_elecnoise.set_xlabel('electronic noise [LSB]')
plt.tight_layout()
if plot != "show":
plt.savefig(plot)
else:
plt.show()
plt.close(fig)
if dark_histo is not None:
dark_histo = Histogram1D.load(dark_histo)
baseline = dark_histo.mean(method='mid')
sigclip_dark = SigmaClip(sigma=nsigma_dark,
iters=10,
cenfunc=np.nanmean,
stdfunc=np.nanstd)
dark_mask = sigclip_dark(baseline).mask
bad_dark = np.where(dark_mask)[0]
bad_pix = np.unique(np.concatenate((bad_pix, bad_dark)))
return bad_pix
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction, dark subtraction, overscan correction, flat correction, and trimming image.')
raw = CCDData.read(sci,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of science image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)#dark_exposure=1*u.second, data_exposure=red.header['EXPTIME']*u.second, scale=True)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
red = ccdproc.flat_correct(red, mflat)
processed_data = ccdproc.ccd_process(red, trim=raw.header['DATASEC'])
log.info('File proccessed and trimmed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci,processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(red.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (510, 510), filter_size=(9, 9),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits'),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['EXPTIME']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/'),overwrite=True)
processed.append(final)
return processed, masks
def _remove_background(self, mask=None):
"""
Background removal. It models the background using a median estimator, rejects
flux values with sigma clipping. It modiffies the attributes `flux` and
`flux_2d`. The background model are stored in the `background_model` attribute.
Parameters
----------
mask : numpy.ndarray of booleans
Mask to reject pixels containing sources. Default None.
"""
# model background for all cadences
self.background_model = np.array(
[
Background2D(
flux_2d,
mask=mask,
box_size=(64, 50),
filter_size=15,
exclude_percentile=20,
sigma_clip=SigmaClip(sigma=3.0, maxiters=5),
bkg_estimator=MedianBackground(),
interpolator=BkgZoomInterpolator(order=3),
).background
for flux_2d in self.flux_2d
]
)
# substract background
self.flux_2d -= self.background_model
# flatten flix image
self.flux = self.flux_2d.reshape(self.flux_2d.shape[0], -1)
return
def _model_bkg(self, data, mask=None):
"""
BkgZoomInterpolator:
This class generates full-sized background and background RMS images
from lower-resolution mesh images using the `~scipy.ndimage.zoom`
(spline) interpolator.
Parameters
----------
data : numpy.ndarray
Data arra with the pixel flux values.
mask : numpy.ndarray
Boolean array to mask pixels with sources.
Returns
-------
background : numpy.ndarray
Data array with background model
"""
model = Background2D(
data,
mask=mask,
box_size=(64, 50),
filter_size=15,
exclude_percentile=20,
sigma_clip=SigmaClip(sigma=3.0, maxiters=5),
bkg_estimator=MedianBackground(),
interpolator=BkgZoomInterpolator(order=3),
)
return model.background
def __init__(self,
oversampling=4.,
shape=None,
smoothing_kernel='quartic',
recentering_func=centroid_com,
recentering_maxiters=20,
fitter=EPSFFitter(),
maxiters=10,
progress_bar=True,
norm_radius=5.5,
shift_val=0.5,
recentering_boxsize=(5, 5),
center_accuracy=1.0e-3,
flux_residual_sigclip=SigmaClip(sigma=3,
cenfunc='median',
maxiters=10)):
if oversampling is None:
raise ValueError("'oversampling' must be specified.")
oversampling = np.atleast_1d(oversampling).astype(int)
if len(oversampling) == 1:
oversampling = np.repeat(oversampling, 2)
if np.any(oversampling <= 0.0):
raise ValueError('oversampling must be a positive number.')
self._norm_radius = norm_radius
self._shift_val = shift_val
self.oversampling = oversampling
self.shape = self._init_img_params(shape)
if self.shape is not None:
self.shape = self.shape.astype(int)
self.recentering_func = recentering_func
self.recentering_maxiters = recentering_maxiters
self.recentering_boxsize = self._init_img_params(recentering_boxsize)
self.recentering_boxsize = self.recentering_boxsize.astype(int)
self.smoothing_kernel = smoothing_kernel
if not isinstance(fitter, EPSFFitter):
raise TypeError('fitter must be an EPSFFitter instance.')
self.fitter = fitter
if center_accuracy <= 0.0:
raise ValueError('center_accuracy must be a positive number.')
self.center_accuracy_sq = center_accuracy**2
maxiters = int(maxiters)
if maxiters <= 0:
raise ValueError("'maxiters' must be a positive number.")
self.maxiters = maxiters
self.progress_bar = progress_bar
if not isinstance(flux_residual_sigclip, SigmaClip):
raise ValueError("'flux_residual_sigclip' must be an"
" astropy.stats.SigmaClip function.")
self.flux_residual_sigclip = flux_residual_sigclip
# store each ePSF build iteration
self._epsf = []
def create_flat(flat_list,fil,red_path,mbias=None,log=None):
log.info('Processing files for filter: '+fil)
log.info(str(len(flat_list))+' files found.')
flats = []
flat_scale = []
for flat in flat_list:
log.info('Loading file: '+flat)
raw = CCDData.read(flat,hdu=1,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['GAIN']*u.electron/u.adu, readnoise=raw.header['RDNOISE']*u.electron)
log.info('Exposure time of image is '+str(red.header['EXPTIME']))
log.info('Loading correct master dark')
mdark = CCDData.read(red_path+'DARK_'+str(red.header['EXPTIME'])+'.fits', unit=u.electron)
red = ccdproc.subtract_dark(red, mdark, exposure_time='EXPTIME', exposure_unit=u.second)
red = ccdproc.subtract_overscan(red, overscan=red[:,0:4], overscan_axis=1, model=models.Chebyshev1D(3))
mask = make_source_mask(red, nsigma=3, npixels=5)
bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
masked = np.array(red)
masked[mask] = bkg.background[mask]
log.info('Median flat level: '+str(np.median(masked)))
norm = 1/np.median(masked[500:1500,500:1500])
log.info('Flat normalization: '+str(norm))
flat_scale.append(norm)
flats.append(CCDData(masked,meta=red.header,unit=u.electron))
mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True)
log.info('Created master flat for filter: '+fil)
mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True)
log.info('Master flat written to mflat_'+fil+'.fits')
return
def bkg_subtraction(time, flux, scope="tpf", sigma=3):
"""Subtracts background flux from target pixel file.
Parameters
----------
scope : string, "tpf" or "postcard"
If `tpf`, will use data from the target pixel file only to estimate and remove the background.
If `postcard`, will use data from the entire postcard region to estimate and remove the background.
sigma : float
The standard deviation cut used to determine which pixels are representative of the background in each cadence.
"""
tpf_flux_bkg = []
sigma_clip = SigmaClip(sigma=sigma)
# bkg = MMMBackground(sigma_clip=sigma_clip)
bkg = SExtractorBackground(sigma_clip=sigma_clip)
for i in range(len(time)):
bkg_value = bkg.calc_background(flux[i])
tpf_flux_bkg.append(bkg_value)
tpf_flux_bkg = np.array(tpf_flux_bkg)
return tpf_flux_bkg
def test_epsfbuilder_inputs():
# invalid inputs
with pytest.raises(ValueError):
EPSFBuilder(oversampling=None)
with pytest.raises(ValueError):
EPSFBuilder(oversampling=-1)
with pytest.raises(ValueError):
EPSFBuilder(maxiters=-1)
with pytest.raises(ValueError):
EPSFBuilder(oversampling=3)
with pytest.raises(ValueError):
EPSFBuilder(oversampling=[3, 6])
with pytest.raises(ValueError):
EPSFBuilder(oversampling=[-1, 4])
# valid inputs
EPSFBuilder(oversampling=6)
EPSFBuilder(oversampling=[4, 6])
# invalid inputs
for sigclip in [None, [], 'a']:
with pytest.raises(ValueError):
EPSFBuilder(flux_residual_sigclip=sigclip)
# valid inputs
EPSFBuilder(
flux_residual_sigclip=SigmaClip(sigma=2.5, cenfunc='mean', maxiters=2))
def background_2D(image, sigma, iters, box_size, filter_size, plt_grid):
"""2D background estimation.
This function creates a 2D background estimate by dividing the image into
a grid, defined by box_size.
Args:
image(array, required): This is the image data
sigma(float, required): Sigma level
iters(int, required): Number of iterations
box_size(int, required): Defines the box dimesions, in pixels
filter_size(int, required): Defines the filter reach in pixels
plt_grid(boolean): Overplot grid on image
Returns:
bkg(array): 2D background level
bkgrms(array): RMS background
"""
sigma_clip = SigmaClip(sigma=sigma, iters=iters)
mask = (image == 0)
bkg_estimator = MedianBackground()
bkg = Background2D(image,
box_size=box_size,
filter_size=filter_size,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask,
edge_method=u'pad')
# print('Background Median: ' + str(bkg.background_median))
# print('Background RMS median: ' + str(bkg.background_rms_median))
if plt_grid is True:
plt.imshow(bkg.background, origin='lower', cmap='Greys')
bkg.plot_meshes(outlines=True, color='#1f77b4')
bkgrms = bkg.background_rms
return bkg, bkgrms
def load_all_lygos(datapath):
all_t = []
all_i = []
all_e = []
all_labels = []
for root, dirs, files in os.walk(datapath):
for name in files:
if name.startswith(("rflx")):
filepath = root + "/" + name
print(name)
label = name.split("_")
full_label = label[1] + label[2]
all_labels.append(full_label)
t, i, e = load_lygos_csv(name)
mean = np.mean(i)
sigclip = SigmaClip(sigma=4, maxiters=None, cenfunc='median')
clipped_inds = np.nonzero(np.ma.getmask(sigclip(i)))
i[clipped_inds] = mean #reset those values to the mean value (or remove??)
all_t.append(t)
all_i.append(i)
all_e.append(e)
return all_t, all_i, all_e, all_labels
def sigma_clip_flux(self, sigma_upper=4, sigma_lower=4, maxiters=None,
plot=False):
"""
Sigma-clip the light curve on fluxes (update mask).
Parameters
----------
sigma_upper : float
Factor of standard deviations above the median centroid position
to clip on.
sigma_lower : float
Factor of standard deviations below the median centroid position
to clip on.
maxiters : float or None
Number of sigma-clipping iterations. Default is None, which repeats
until there are no outliers left.
plot : float
Plot the accepted fluxes (in black) and the rejected fluxes (in red)
"""
sc = SigmaClip(sigma_upper=sigma_upper, sigma_lower=sigma_lower,
stdfunc=mad_std, maxiters=maxiters)
self.mask[~self.mask] |= sc(self.flux[~self.mask]).mask
if plot:
plt.plot(self.bjd_time[self.mask], self.flux[self.mask], 'r.')
plt.plot(self.bjd_time[~self.mask], self.flux[~self.mask], 'k.')
plt.xlabel('BJD')
plt.ylabel('Flux')
def process_science(sci_list,fil,red_path,mbias=None,mflat=None,proc=None,log=None):
masks = []
processed = []
for sci in sci_list:
log.info('Loading file: '+sci)
log.info('Applying gain correction and flat correction.')
with fits.open(sci) as hdr:
header = hdr[0].header
data = hdr[0].data
data[np.isnan(data)] = np.nanmedian(data)
raw = CCDData(data,meta=header,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron)
log.info('Exposure time of science image is '+str(red.header['TRUITIME']*red.header['COADDONE']))
flat = np.array(ccdproc.flat_correct(red, mflat))
flat[np.isnan(flat)] = np.nanmedian(flat)
processed_data = CCDData(flat,unit=u.electron,header=red.header,wcs=red.wcs)
log.info('File proccessed.')
log.info('Cleaning cosmic rays and creating mask.')
mask = make_source_mask(processed_data, nsigma=3, npixels=5)
masks.append(mask)
clean, com_mask = create_mask.create_mask(sci.replace('.gz',''),processed_data,'_mask.fits',static_mask(proc)[0],mask,saturation(red.header),binning(),rdnoise(raw.header),cr_clean_sigclip(),cr_clean_sigcfrac(),cr_clean_objlim(),log)
processed_data.data = clean
log.info('Calculating 2D background.')
bkg = Background2D(processed_data, (128, 128), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
log.info('Median background: '+str(np.median(bkg.background)))
fits.writeto(sci.replace('/raw/','/red/').replace('.fits','_bkg.fits').replace('.gz',''),bkg.background,overwrite=True)
final = processed_data.subtract(CCDData(bkg.background,unit=u.electron),propagate_uncertainties=True,handle_meta='first_found').divide(red.header['TRUITIME']*red.header['COADDONE']*u.second,propagate_uncertainties=True,handle_meta='first_found')
log.info('Background subtracted and image divided by exposure time.')
final.write(sci.replace('/raw/','/red/').replace('.gz',''),overwrite=True)
processed.append(final)
return processed, masks
def __init__(self, crit_separation, threshold, fwhm, psf_model, fitshape,
sigma=3., ratio=1.0, theta=0.0, sigma_radius=1.5,
sharplo=0.2, sharphi=1.0, roundlo=-1.0, roundhi=1.0,
fitter=LevMarLSQFitter(),
niters=3, aperture_radius=None, extra_output_cols=None):
self.crit_separation = crit_separation
self.threshold = threshold
self.fwhm = fwhm
self.sigma = sigma
self.ratio = ratio
self.theta = theta
self.sigma_radius = sigma_radius
self.sharplo = sharplo
self.sharphi = sharphi
self.roundlo = roundlo
self.roundhi = roundhi
group_maker = DAOGroup(crit_separation=self.crit_separation)
bkg_estimator = MMMBackground(sigma_clip=SigmaClip(sigma=self.sigma))
finder = DAOStarFinder(threshold=self.threshold, fwhm=self.fwhm,
ratio=self.ratio, theta=self.theta,
sigma_radius=self.sigma_radius,
sharplo=self.sharplo, sharphi=self.sharphi,
roundlo=self.roundlo, roundhi=self.roundhi)
super().__init__(group_maker=group_maker, bkg_estimator=bkg_estimator,
psf_model=psf_model, fitshape=fitshape,
finder=finder, fitter=fitter, niters=niters,
aperture_radius=aperture_radius,
extra_output_cols=extra_output_cols)
def create_flat(flat_list,fil,red_path,mbias=None,log=None):
log.info('Processing files for filter: '+fil)
log.info(str(len(flat_list))+' files found.')
flats = []
masks = []
flat_scale = []
for flat in flat_list:
log.info('Loading file: '+flat)
raw = CCDData.read(flat,unit=u.adu)
red = ccdproc.ccd_process(raw, gain=raw.header['SYSGAIN']*u.electron/u.adu, readnoise=rdnoise(raw.header)*u.electron)
log.info('Exposure time of image is '+str(red.header['TRUITIME']*red.header['COADDONE']))
mask = make_source_mask(red, nsigma=3, npixels=5)
bkg = Background2D(red, (20, 20), filter_size=(3, 3),sigma_clip=SigmaClip(sigma=3), bkg_estimator=MeanBackground(), mask=mask, exclude_percentile=80)
masked = np.array(red)
masked[mask] = bkg.background[mask]
log.info('Median flat level: '+str(np.median(masked)))
norm = 1/np.median(masked[224:1824,224:1824])
log.info('Flat normalization: '+str(norm))
flat_scale.append(norm)
flats.append(CCDData(masked,meta=red.header,unit=u.electron))
mflat = ccdproc.combine(flats,method='median',scale=flat_scale,sigma_clip=True)
log.info('Created master flat for filter: '+fil)
mflat.write(red_path+'mflat_'+fil+'.fits',overwrite=True)
log.info('Master flat written to mflat_'+fil+'.fits')
return
def _aper_local_background(self):
"""
Estimate the local background and error using a circular annulus
aperture.
The local backround is the sigma-clipped median value in the
annulus. The background error is the standard error of the
median, sqrt(pi / 2N) * std.
"""
bkg_aper = CircularAnnulus(
self.xypos, self.aperture_params['bkg_aperture_inner_radius'],
self.aperture_params['bkg_aperture_outer_radius'])
bkg_aper_masks = bkg_aper.to_mask(method='center')
sigclip = SigmaClip(sigma=3)
nvalues = []
bkg_median = []
bkg_std = []
for mask in bkg_aper_masks:
bkg_data = mask.multiply(self.model.data.value)
bkg_data_1d = bkg_data[mask.data > 0]
values = sigclip(bkg_data_1d, masked=False)
nvalues.append(values.size)
bkg_median.append(np.median(values))
bkg_std.append(np.std(values))
nvalues = np.array(nvalues)
bkg_median = np.array(bkg_median)
# standard error of the median
bkg_median_err = np.sqrt(np.pi / (2. * nvalues)) * np.array(bkg_std)
bkg_median <<= self.model.data.unit
bkg_median_err <<= self.model.data.unit
return bkg_median, bkg_median_err
def background_rms(data):
# Determines the rms of the background.
sigmaClipper = SigmaClip(sigma=3, maxiters=5)
bck_estimator = SExtractorBackground(data)
bkg = Background2D(data,(50,50),filter_size=(3,3),sigma_clip=sigmaClipper,bkg_estimator=bck_estimator)
print((bkg.background_median, bkg.background_rms_median))
return bkg.background_rms_median
def test_psf_photometry_niters(sigma_psf, sources):
img_shape = (32, 32)
# generate image with read-out noise (Gaussian) and
# background noise (Poisson)
image = (make_gaussian_sources_image(img_shape, sources) +
make_noise_image(img_shape, type='poisson', mean=6.,
random_state=1) +
make_noise_image(img_shape, type='gaussian', mean=0.,
stddev=2., random_state=1))
cp_image = image.copy()
sigma_clip = SigmaClip(sigma=3.)
bkgrms = StdBackgroundRMS(sigma_clip)
std = bkgrms(image)
phot_obj = make_psf_photometry_objs(std, sigma_psf)[1:3]
for iter_phot_obj in phot_obj:
iter_phot_obj.niters = None
result_tab = iter_phot_obj(image)
residual_image = iter_phot_obj.get_residual_image()
assert (result_tab['x_0_unc'] < 1.96 * sigma_psf / np.sqrt(sources['flux'])).all()
assert (result_tab['y_0_unc'] < 1.96 * sigma_psf / np.sqrt(sources['flux'])).all()
assert (result_tab['flux_unc'] < 1.96 * np.sqrt(sources['flux'])).all()
assert_allclose(result_tab['x_fit'], sources['x_mean'], rtol=1e-1)
assert_allclose(result_tab['y_fit'], sources['y_mean'], rtol=1e-1)
assert_allclose(result_tab['flux_fit'], sources['flux'], rtol=1e-1)
assert_array_equal(result_tab['id'], sources['id'])
assert_array_equal(result_tab['group_id'], sources['group_id'])
assert_array_equal(result_tab['iter_detected'], sources['iter_detected'])
assert_allclose(np.mean(residual_image), 0.0, atol=1e1)
# make sure image is note overwritten
assert_array_equal(cp_image, image)
async def __call__(self, image: Image) -> Image:
"""Remove background from image.
Args:
image: Image to remove background from.
Returns:
Image without background.
"""
from photutils.background import Background2D, MedianBackground
# init objects
sigma_clip = SigmaClip(sigma=self.sigma)
bkg_estimator = MedianBackground()
# calculate background
bkg = Background2D(image.data,
self.box_size,
filter_size=self.filter_size,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
# copy image and remove background
img = image.copy()
img.data = img.data - bkg.background
return img
def __init__(self, window_size=15, img_shape=None):
from astropy.stats import SigmaClip
from photutils import Background2D, MedianBackground
self.sigma_clip = SigmaClip(sigma=3., iters=10)
self.bkg_estimator = MedianBackground()
self.B2D = Background2D
def __init__(self, gid, field, flter, nearby_fitting_region, nearby_gids = None):
if not (field == 'S' or field == 'N'):
raise Exception('Field must be S or N.')
if not type(flter) == str:
raise Exception('Filter must be string.')
self.gid = gid
self.field = field
self.flter = flter
self.nearby_fitting_region = nearby_fitting_region
if self.flter == '850':
if self.field == 'N':
self.fitsfile = 'GOODS-N_acsz_sci_sub.fits'
self.fullfield = 'North'
if self.field == 'S':
self.fitsfile = 'goodss_3dhst.v4.0.F850LP_orig_sci.fits'
self.fullfield = 'South'
else:
if self.field =='S':
self.fitsfile = 'goodss_3dhst.v4.0.F'+flter+'W_orig_sci.fits'
self.fullfield = 'South'
if self.field =='N':
self.fitsfile = 'goodsn_3dhst.v4.0.F'+flter+'W_orig_sci.fits'
self.fullfield = 'North'
### CATALOG ###
if (field == 'S') or (field == 'South'):
catalog = '/Users/rosaliaobrien/research/website/catalogs/goodss_3dhst.v4.4.cat'
if (field == 'N') or (field == 'North'):
catalog = '/Users/rosaliaobrien/research/website/catalogs/goodsn_3dhst.v4.4.cat'
self.cat = ascii.read(catalog)
### GET SKY ESTIMATE ###
if not nearby_gids == None:
xmin, xmax, ymin, ymax = self.get_boundaries(nearby_gids)
path = '/Users/rosaliaobrien/research/GALFIT_CLEAR/running_GALFIT/'+self.fullfield+'/'+self.flter+'/'
skydata = fits.open(path+self.fitsfile)[0].data
xmid = (xmin+xmax)/2
ymid = (ymin+ymax)/2
cutout = Cutout2D(skydata, (xmid, ymid), (400, 400), copy=True, mode='partial')
masked_cutout = make_source_mask(cutout.data, snr = 1.5,
npixels=10, dilate_size=11)
cutout.data[masked_cutout] = np.nan
self.scidata = cutout.data
sigma_clip = SigmaClip(sigma=3)
bkg = SExtractorBackground(sigma_clip)
rms = MADStdBackgroundRMS(sigma_clip)
self.sky_value = bkg.calc_background(cutout.data)
self.rms_value = rms.calc_background_rms(cutout.data)
def soss_background(scidata, scimask, bkg_mask=None):
"""Compute a columnwise background for a SOSS observation.
Parameters
----------
scidata : array[float]
The image of the SOSS trace.
scimask : array[bool]
Boolean mask of pixels to be excluded.
bkg_mask : array[bool]
Boolean mask of pixels to be excluded because they are in
the trace, typically constructed with make_profile_mask.
Returns
-------
scidata_bkg : array[float]
Background-subtracted image
col_bkg : array[float]
Column-wise background values
npix_bkg : array[float]
Number of pixels used to calculate each column value in col_bkg
"""
# Check the validity of the input.
data_shape = scidata.shape
if scimask.shape != data_shape:
msg = 'scidata and scimask must have the same shape.'
log.critical(msg)
raise ValueError(msg)
if bkg_mask is not None:
if bkg_mask.shape != data_shape:
msg = 'scidata and bkg_mask must have the same shape.'
log.critical(msg)
raise ValueError(msg)
# Combine the masks and create a masked array.
if bkg_mask is not None:
mask = scimask | bkg_mask
else:
mask = scimask
scidata_masked = np.ma.array(scidata, mask=mask)
# Mask additional pixels using sigma-clipping.
sigclip = SigmaClip(sigma=3, maxiters=None, cenfunc='mean')
scidata_clipped = sigclip(scidata_masked, axis=0)
# Compute the mean for each column and record the number of pixels used.
col_bkg = scidata_clipped.mean(axis=0)
col_bkg = np.where(np.all(scidata_clipped.mask, axis=0), 0., col_bkg)
npix_bkg = (~scidata_clipped.mask).sum(axis=0)
# Background subtract the science data.
scidata_bkg = scidata - col_bkg
return scidata_bkg, col_bkg, npix_bkg
def subtract_background(self,
subtract_min_value=True,
plot_background=False,
ax=None):
"""
A function to subtract the background for the FRD tests
INPUT:
subtract_min_value - subtracts the .min value (no negative numbers)
plot_background - if True, plots the estimated background with the meshes it used.
"""
self.aper = photutils.CircularAperture(positions=(self.x, self.y),
r=self.r)
# get the mask from the aperture
# hack
mask = self.aper.to_mask()[0].to_image(self.data.shape)
# use sigma clipping
sigma_clip = SigmaClip(sigma=3., iters=10)
bkg_estimator = photutils.MedianBackground()
self.bkg = photutils.Background2D(self.data,
self.box_size,
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask,
edge_method="crop")
self.data_background = self.data - self.bkg.background
if subtract_min_value:
self.subtracted_value = self.data_background.min()
print("Subtracted min value:", self.subtracted_value)
self.data_background -= self.subtracted_value
if plot_background:
if ax == None:
self.fig, self.ax = plt.subplots()
else:
self.ax = ax
im = self.ax.imshow(self.bkg.background,
origin='lower',
cmap='Greys_r')
self.ax.set_xlim(0, self.bkg.background.shape[1])
self.ax.set_ylim(0, self.bkg.background.shape[0])
self.ax.set_title("Estimated Background", y=1.02)
self.ax.set_xlabel("X pixels")
self.ax.set_ylabel("Y pixels")
if ax == None:
# Only do this if ax is not supplied
# Don't know how to deal with this without passing the figure explicitly
self.fig.colorbar(im)
self.bkg.plot_meshes(outlines=True, color='#1f77b4', ax=self.ax)
return self.data_background
def create_ENF_features(self, version=0, save=True):
""" documentation """
fname_features = self.ENFpath + "features_v"+str(version)+".fits"
feature_list = []
if version == 0:
self.median_normalize()
if self.datatype == "FFI":
self.flux, comp = self.pca_linregress()
elif version == 1:
from transitleastsquares import transitleastsquares
self.mean_normalize()
if self.datatype == "FFI":
self.flux, comp = self.pca_linregress()
print("Begining Feature Vector Creation Now")
#sigma clip each time you calculate - unsure how better to do this??
sigclip = SigmaClip(sigma=5, maxiters=None, cenfunc='median')
for n in range(len(self.flux)):
times = self.time
ints = self.flux[n]
clipped_inds = np.nonzero(np.ma.getmask(sigclip(ints)))
ints[clipped_inds] = np.nan
delete_index = np.argwhere(np.isnan(ints))
times = np.delete(times, delete_index)
ints = np.delete(ints, delete_index)
try:
feature_vector = enf.featvec(times, ints, v=version)
except ValueError:
print("it did the stupid thing where it freaked out about one light curve and idk why")
if version == 1:
feature_vector = np.nan_to_num(feature_vector, nan=0)
with open(self.ENFpath + 'intermediate_v1_saving.txt', 'a') as f:
f.write('{} {} \n'.format(self.identifiers[n], feature_vector))
feature_list.append(feature_vector)
if version == 0:
if n % 500 == 0:
print(str(n) + " completed")
if self.cadence == "20_s" and n % 20 == 0:
print(str(n) + " completed")
elif version == 1:
print(str(n))
self.features = np.asarray(feature_list)
if save == True:
hdr = fits.Header()
hdr["VERSION"] = version
hdu = fits.PrimaryHDU(feature_list, header=hdr)
hdu.writeto(fname_features)
else:
print("Not saving feature vectors to fits")
return
def run(self, image):
sigma_clip = SigmaClip(sigma=3.)
self.bkg = Background2D(image.data,
box_size=self.box_size,
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=self.bkg_estimator).background
if self.subtract:
image.bkg = self.bkg
image.data = image.data - self.bkg
def background_reduction(List):
Raw_Image_Data = []
Reduced_Image_Data = []
Bkg = []
Bkg_Sigma = []
Median = []
Std = []
Norm = ImageNormalize(stretch=SqrtStretch())
for i in range(len(List)):
Image_File = get_pkg_data_filename(f'RAW_DATA/{List[i]}')
crmask, Raw_Image_Data_1 = astroscrappy.detect_cosmics(
fits.getdata(Image_File, ext=0))
Raw_Image_Data.append(Raw_Image_Data_1)
Mask = (Raw_Image_Data_1 == 0)
Sigma_Clip = SigmaClip(sigma=3.0)
Bkg_Estimator = MedianBackground()
Bkg_ta = Background2D(Raw_Image_Data_1, (25, 25),
filter_size=(3, 3),
sigma_clip=Sigma_Clip,
bkg_estimator=Bkg_Estimator,
mask=Mask)
Bkg.append(Bkg_ta)
Bkg_Sigma_ta = mad_std(Raw_Image_Data_1)
Bkg_Sigma.append(Bkg_Sigma_ta)
Mean, Median_ta, Std_ta = sigma_clipped_stats(Raw_Image_Data_1,
sigma=3.0)
Median.append(Median_ta)
Std.append(Std_ta)
Reduced_Image_Data_to_append = Raw_Image_Data_1 - Bkg_ta.background
Reduced_Image_Data.append(Reduced_Image_Data_to_append)
#plt.figure()
#plt.imshow(Raw_Image_Data_1, cmap = 'gray', origin = 'lower', interpolation = 'bicubic', norm = Norm, vmin = 100, vmax = 1000)
#plt.colorbar()
#plt.figure()
#plt.imshow(Bkg_ta.background, origin='lower', cmap='gray', interpolation = 'bicubic')
#plt.colorbar()
#plt.figure()
#plt.imshow(Reduced_Image_Data_to_append, norm = Norm, origin = 'lower', cmap = 'gray', interpolation = 'bicubic', vmin = 1, vmax = 1000)
#plt.colorbar()
#plt.show()
return Raw_Image_Data, Reduced_Image_Data, Bkg, Bkg_Sigma, Median, Std
def bkgstd(im_data, mask):
"""Estimate the RMS standard deviation of the background in the `im_data`
image.
Arguments
---------
im_data : array_like
Image data
mask : array_like
Mask to apply to image
Returns
-------
np.ndarray
RMS standard deviation of the background image
"""
# use crude image segmentation to find sources above SNR=3, build a
# source mask, and estimate the background RMS
source_mask = make_source_mask(im_data, snr=3, npixels=5,
dilate_size=15, mask=mask)
# combine the bad pixel mask and source mask
rough_mask = np.logical_or(mask, source_mask)
# estimate the background standard deviation
try:
sigma_clip = SigmaClip(sigma=3, maxiters=5) # sigma clipping
except TypeError: # in old astropy, "maxiters" was "iters"
sigma_clip = SigmaClip(sigma=3, iters=5)
try:
bkg = Background2D(im_data, (50,50), filter_size=(5,5),
sigma_clip=sigma_clip,
bkg_estimator=MedianBackground(),
mask=rough_mask)
except ValueError:
e = sys.exc_info()
print("\nWhile attempting background estimation on the "+
"science image, the following error was raised: "+
f"\n{str(e[0])}\n{str(e[1])}\n--> exiting.")
return
return bkg.background_rms
def photutilsky(image, box_size=(50, 50), filter_size=(3, 3)):
""" Estimate sky background using photutils."""
sigma_clip = SigmaClip(sigma=3.)
bkg_estimator = MedianBackground()
bkg = Background2D(image.data,
box_size,
mask=image.mask,
filter_size=filter_size,
sigma_clip=sigma_clip)
return bkg.background
def doSubtractBackground(self, rgb):
sigma_clip = SigmaClip(sigma=3.0)
bkg_estimator = MedianBackground()
for j in range(3):
bkg = Background2D(rgb[j], (50, 50),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
#print("Bkg median: %7.4f, RMS median: %7.4f" % (bkg.background_median, bkg.background_rms_median))
rgb[j] -= bkg.background
imshow(bkg.background, origin='lower', cmap='Greys_r')
async def __call__(self, image: Image) -> Image:
"""Find stars in given image and append catalog.
Args:
image: Image to find stars in.
Returns:
Image with attached catalog.
"""
from astropy.stats import SigmaClip, sigma_clipped_stats
from photutils import Background2D, MedianBackground, DAOStarFinder
# get data
if image.data is None:
log.warning("No data found in image.")
return image
data = image.data.astype(float).copy()
# estimate background
sigma_clip = SigmaClip(sigma=self.bkg_sigma)
bkg_estimator = MedianBackground()
bkg = Background2D(
data,
self.bkg_box_size,
filter_size=self.bkg_filter_size,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=image.mask,
)
data -= bkg.background
# do statistics
mean, median, std = sigma_clipped_stats(data, sigma=3.0)
# find stars
daofind = DAOStarFinder(fwhm=self.fwhm, threshold=self.threshold * std)
loop = asyncio.get_running_loop()
sources = await loop.run_in_executor(None, daofind, data - median)
# rename columns
sources.rename_column("xcentroid", "x")
sources.rename_column("ycentroid", "y")
# match fits conventions
sources["x"] += 1
sources["y"] += 1
# pick columns for catalog
cat = sources["x", "y", "flux", "peak"]
# copy image, set catalog and return it
img = image.copy()
img.catalog = cat
return img
def get_sky_background(img, verbose=True):
from astropy.stats import SigmaClip
from photutils import Background2D, MedianBackground
sigma_clip = SigmaClip(sigma=3.)
bkg = Background2D(img, (100, 100),
filter_size=(5, 5),
sigma_clip=sigma_clip,
bkg_estimator=MedianBackground())
if verbose:
print('Sky background median = {:.3f}, rms = {:.3f} ADU.'.format(
bkg.background_median, bkg.background_rms_median))
return bkg
