def bkg_subtraction(time, flux, scope="tpf", sigma=3):
"""Subtracts background flux from target pixel file.
Parameters
----------
scope : string, "tpf" or "postcard"
If `tpf`, will use data from the target pixel file only to estimate and remove the background.
If `postcard`, will use data from the entire postcard region to estimate and remove the background.
sigma : float
The standard deviation cut used to determine which pixels are representative of the background in each cadence.
"""
tpf_flux_bkg = []
sigma_clip = SigmaClip(sigma=sigma)
# bkg = MMMBackground(sigma_clip=sigma_clip)
bkg = SExtractorBackground(sigma_clip=sigma_clip)
for i in range(len(time)):
bkg_value = bkg.calc_background(flux[i])
tpf_flux_bkg.append(bkg_value)
tpf_flux_bkg = np.array(tpf_flux_bkg)
return tpf_flux_bkg
def __init__(self, gid, field, flter, nearby_fitting_region, nearby_gids = None):
if not (field == 'S' or field == 'N'):
raise Exception('Field must be S or N.')
if not type(flter) == str:
raise Exception('Filter must be string.')
self.gid = gid
self.field = field
self.flter = flter
self.nearby_fitting_region = nearby_fitting_region
if self.flter == '850':
if self.field == 'N':
self.fitsfile = 'GOODS-N_acsz_sci_sub.fits'
self.fullfield = 'North'
if self.field == 'S':
self.fitsfile = 'goodss_3dhst.v4.0.F850LP_orig_sci.fits'
self.fullfield = 'South'
else:
if self.field =='S':
self.fitsfile = 'goodss_3dhst.v4.0.F'+flter+'W_orig_sci.fits'
self.fullfield = 'South'
if self.field =='N':
self.fitsfile = 'goodsn_3dhst.v4.0.F'+flter+'W_orig_sci.fits'
self.fullfield = 'North'
### CATALOG ###
if (field == 'S') or (field == 'South'):
catalog = '/Users/rosaliaobrien/research/website/catalogs/goodss_3dhst.v4.4.cat'
if (field == 'N') or (field == 'North'):
catalog = '/Users/rosaliaobrien/research/website/catalogs/goodsn_3dhst.v4.4.cat'
self.cat = ascii.read(catalog)
### GET SKY ESTIMATE ###
if not nearby_gids == None:
xmin, xmax, ymin, ymax = self.get_boundaries(nearby_gids)
path = '/Users/rosaliaobrien/research/GALFIT_CLEAR/running_GALFIT/'+self.fullfield+'/'+self.flter+'/'
skydata = fits.open(path+self.fitsfile)[0].data
xmid = (xmin+xmax)/2
ymid = (ymin+ymax)/2
cutout = Cutout2D(skydata, (xmid, ymid), (400, 400), copy=True, mode='partial')
masked_cutout = make_source_mask(cutout.data, snr = 1.5,
npixels=10, dilate_size=11)
cutout.data[masked_cutout] = np.nan
self.scidata = cutout.data
sigma_clip = SigmaClip(sigma=3)
bkg = SExtractorBackground(sigma_clip)
rms = MADStdBackgroundRMS(sigma_clip)
self.sky_value = bkg.calc_background(cutout.data)
self.rms_value = rms.calc_background_rms(cutout.data)
def background_rms(data):
# Determines the rms of the background.
sigmaClipper = SigmaClip(sigma=3, maxiters=5)
bck_estimator = SExtractorBackground(data)
bkg = Background2D(data,(50,50),filter_size=(3,3),sigma_clip=sigmaClipper,bkg_estimator=bck_estimator)
print((bkg.background_median, bkg.background_rms_median))
return bkg.background_rms_median
def sub_bkg(self, img):
sigma_clip = SigmaClip(sigma=3.)
bkg_estimator = SExtractorBackground()
mask_0 = make_source_mask(img, nsigma=3, npixels=5, dilate_size=11)
mask_1 = (np.isnan(img))
mask = mask_0 + mask_1
bkg = Background2D(img, (5, 5),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask)
back = bkg.background * ~mask_1
return img - back
def bkg_estimation(filename,
box=(20, 20),
filter_size=(3, 3),
bkg_estimator='SExtractor',
sigma=3.,
sigma_lower=None,
sigma_upper=None,
maxiters=10,
outLevel=1):
imagelist = np.atleast_1d(filename)
for ima in imagelist:
print("\nEstimate background in %s" % ima)
root = os.path.splitext(ima)[0]
hdulist = fits.open(ima)
data = hdulist[0].data
sigma_clip = SigmaClip(sigma=sigma,
sigma_lower=sigma_lower,
sigma_upper=sigma_upper,
maxiters=maxiters)
if bkg_estimator == 'SExtractor':
bkg_estimator = SExtractorBackground()
elif bkg_estimator == 'MMM':
bkg_estimator = MMMBackground()
elif bkg_estimator == 'ModeEstimator':
bkg_estimator = ModeEstimatorBackground()
elif bkg_estimator == 'Median':
bkg_estimator = MedianBackground()
elif bkg_estimator == 'Mean':
bkg_estimator = MeanBackground()
bkg = Background2D(
data,
box,
filter_size=filter_size,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
)
# Create image with background substracted
hdulist[0].data = data - bkg.background
hdulist.writeto(ima, overwrite=True)
if outLevel == 2:
# Create image with 2D background
bkg_file = root + "_bkg_map.fits"
hdulist[0].data = bkg.background
hdulist.writeto(bkg_file, overwrite=True)
def reference(self):
"""
This function ...
:return:
"""
# Inform the user
log.info("Estimating background with photutils ...")
# Plot total mask
if self.config.plot:
plotting.plot_mask(self.total_mask, title="total mask")
integer_aperture_radius = int(math.ceil(self.aperture_radius))
box_shape = (integer_aperture_radius, integer_aperture_radius)
filter_size = (3, 3)
# Estimate the background
sigma_clip = SigmaClip(sigma=3., iters=10)
#bkg_estimator = MedianBackground()
bkg_estimator = SExtractorBackground()
bkg = Background2D(self.frame.data,
box_shape,
filter_size=filter_size,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=self.total_mask.data)
# Statistics
#print("median background", bkg.background_median)
#print("rms background", bkg.background_rms_median)
self.statistics.reference = Map()
self.statistics.reference.median = bkg.background_median
self.statistics.reference.rms = bkg.background_rms_median
# Plot
if self.config.plot:
plotting.plot_box(bkg.background,
title="background from photutils")
# Set the sky
self.reference_sky = Frame(bkg.background)
# Set bkg object
self.photutils_bkg = bkg
# Plot
if self.config.plot:
plotting.plot_box(self.reference_sky, title="reference sky")
def est_bkg(image, pltshow=1):
print("Estimating the background light......")
img = image # check the back grounp
sigma_clip = SigmaClip(sigma=3., iters=10)
bkg_estimator = SExtractorBackground()
mask_0 = make_source_mask(img, snr=2, npixels=5, dilate_size=11)
mask_1 = (np.isnan(img))
mask = mask_0 + mask_1
bkg = Background2D(img, (50, 50),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask)
from matplotlib.colors import LogNorm
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(img, norm=LogNorm(), origin='lower')
#bkg.plot_meshes(outlines=True, color='#1f77b4')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if pltshow == 0:
plt.close()
else:
plt.show()
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(mask, origin='lower')
#bkg.plot_meshes(outlines=True, color='#1f77b4')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if pltshow == 0:
plt.close()
else:
plt.show()
back = bkg.background * ~mask_1
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(back, origin='lower', cmap='Greys_r')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if pltshow == 0:
plt.close()
else:
plt.show()
# img -= back
# pyfits.PrimaryHDU(img).writeto('sub_coadd.fits',overwrite=True)
return back
def sub_bkg(img, plot=True):
from astropy.stats import SigmaClip
from photutils import Background2D, SExtractorBackground
sigma_clip = SigmaClip(sigma=3., iters=10)
bkg_estimator = SExtractorBackground()
from photutils import make_source_mask
mask_0 = make_source_mask(img, snr=2, npixels=5, dilate_size=11)
mask_1 = (np.isnan(img))
mask = mask_0 + mask_1
bkg = Background2D(img, (50, 50),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask)
from matplotlib.colors import LogNorm
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(img, norm=LogNorm(), origin='lower')
#bkg.plot_meshes(outlines=True, color='#1f77b4')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if plot:
plt.show()
else:
plt.close()
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(mask, origin='lower')
#bkg.plot_meshes(outlines=True, color='#1f77b4')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if plot:
plt.show()
else:
plt.close()
back = bkg.background * ~mask_1
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(back, origin='lower', cmap='Greys_r')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if plot:
plt.show()
else:
plt.close()
return img - back, back
def do_photometry(seed):
rng = np.random.default_rng(seed=seed)
def recipe():
return scopesim_grid(N1d=7,
border=100,
perturbation=7,
magnitude=lambda N: rng.uniform(15, 26, N),
custom_subpixel_psf=anisocado_psf,
seed=seed)
img_noisefloor, tab_noisefloor = read_or_generate_image(
f'grid7_pert7_seed{seed}', recipe=recipe, force_generate=False)
guess_table = tab_noisefloor.copy()
guess_table['x_0'] = guess_table['x'].copy()
guess_table['y_0'] = guess_table['y'].copy()
guess_table['x_orig'] = guess_table['x'].copy()
guess_table['y_orig'] = guess_table['y'].copy()
guess_table['flux_0'] = guess_table['f']
guess_table['x_0'] += rng.uniform(-0.1, 0.1, len(guess_table))
guess_table['y_0'] += rng.uniform(-0.1, 0.1, len(guess_table))
fit_stages = [
FitStage(10, 1e-10, 1e-11, np.inf,
all_individual), # first stage: get flux approximately right
FitStage(60, 0.5, 0.5, 10_000, all_individual),
FitStage(130, 0.1, 0.1, 5_000, all_individual),
#FitStage(10, 0.1, 0.1, 1, all_individual)
#FitStage(10, 0.3, 0.3, 1, all_individual), # optimize position, keep flux constant
#FitStage(10, 0.2, 0.2, 5000, all_individual),
#FitStage(30, 0.05, 0.05, 100, all_individual)
]
photometry = IncrementalFitPhotometry(SExtractorBackground(),
anisocado_psf,
max_group_size=1,
group_extension_radius=50,
fit_stages=fit_stages,
use_noise=True)
result_table = photometry.do_photometry(img_noisefloor, guess_table)
result_table['id'] += seed * 10000
return result_table, photometry.residual(result_table)
def grid_photometry_epsf():
fit_stages_grid = [
FitStage(5, 1e-10, 1e-11, np.inf,
all_individual), # first stage: get flux approximately right
FitStage(5, 0.6, 0.6, 10, all_individual),
FitStage(5, 0.3, 0.3, 500_000, all_individual),
#FitStage(30, 0.1, 0.1, 5_000, all_simultaneous)
]
photometry_grid = IncrementalFitPhotometry(SExtractorBackground(),
anisocado_psf,
max_group_size=1,
group_extension_radius=10,
fit_stages=fit_stages_grid,
use_noise=True)
with open(out_dir / 'processing_params_synthetic_grid_epsf.txt', 'w') as f:
f.write(pformat(photometry_grid.__dict__))
return photometry_grid
def fit_background(image, catalog=None, flux_cutoff=8e4):
"""
Estimate background in Full Frame Image.
Parameters:
image (ndarray or string): Either the image as 2D ndarray or a path to FITS or NPY file where to load image from.
catalog (`astropy.table.Table` object): Catalog of stars in the image. Is not yet being used for anything.
flux_cutoff (float): Flux value at which any pixel above will be masked out of the background estimation.
Returns:
ndarray: Estimated background with the same size as the input image.
ndarray: Boolean array specifying which pixels was used to estimate the background (``True`` if pixel was used).
.. codeauthor:: Rasmus Handberg <*****@*****.**>
"""
# Load file:
if isinstance(image, np.ndarray):
img = image
elif isinstance(image, six.string_types):
if image.endswith('.npy'):
img = np.load(image)
else:
img = load_ffi_fits(image)
else:
raise ValueError(
"Input image must be either 2D ndarray or path to file.")
# Create mask
# TODO: Use the known locations of bright stars
mask = ~np.isfinite(img)
mask |= (img > flux_cutoff)
# Estimate the background:
sigma_clip = SigmaClip(sigma=3.0, iters=5)
bkg_estimator = SExtractorBackground()
bkg = Background2D(img, (64, 64),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask,
exclude_percentile=50)
return bkg.background, mask
def fit_background(ffi):
"""
Estimate the background of a Full Frame Image (FFI) using the photutils package.
This method uses the photoutils Background 2D background estimation using a
SExtracktorBackground estimator, a 3-sigma clip to the data, and a masking
of all pixels above 3e5 in flux.
Parameters:
ffi (ndarray): A single TESS Full Frame Image in the form of a 2D array.
Returns:
ndarray: Estimated background with the same size as the input image.
.. codeauthor:: Rasmus Handberg <*****@*****.**>
.. codeauthor:: Oliver James Hall <*****@*****.**>
"""
mask = ~np.isfinite(ffi)
mask |= (ffi > 3e5)
# Estimate the background:
sigma_clip = SigmaClip(sigma=3.0, iters=5) #Sigma clip the data
bkg_estimator = SExtractorBackground() #Call background estimator
bkg = Background2D(ffi, (64, 64), #Estimate background on sigma clipped data
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask,
exclude_percentile=50)
bkg_est_unfilt = bkg.background
#Smoothing the background using a percentile filter
bkg_est = circular_filter(bkg_est_unfilt, diam=15, percentile=50)
return bkg_est
def subtract_sky_ccd_2d(ccd, skysub='SKYSUB', skyval='SKYVAL', box_size=(50,50), filter_size=(3,3), bkg_method='sextractor',
check_skysub=True, max_val=10000, fitsky=None, sigma=3., iters=10, edge_method='crop'):
# Although edge_method='pad' is recommended, it does give errors "total size of new array must be unchanged" for v0.3
if check_skysub and skysub is not None and skysub in ccd.header and ccd.header[skysub]:
return ccd
if not check_skysub and skysub is not None and skysub in ccd.header and ccd.header[skysub]:
print ('WARNING: Image already sky subtracted! Subtracting AGAIN the sky! [set check_skysub = True for checking]')
dsky = {'median': MedianBackground(), 'mean': MeanBackground(), 'sextractor': SExtractorBackground(),
'biweight': BiweightLocationBackground(), 'mode': ModeEstimatorBackground(), 'mm': MMMBackground()}
if not bkg_method in dsky:
print ('WARNING: Background type "%s" NOT available!! Selecting "median" from [%s]' % (bkg_type, ' | '.join(dksy.keys())))
bkg_estimator = dksy['median']
else:
bkg_estimator = dsky[bkg_method]
sigma_clip = SigmaClip(sigma=sigma, iters=iters) if sigma is not None and iters is not None else None
bkg = Background2D(ccd.data, box_size, filter_size=filter_size, sigma_clip=sigma_clip, bkg_estimator=bkg_estimator, edge_method=edge_method)
if max_val is not None and bkg.background_median >= max_val:
if 'FILENAME' in ccd.header:
print ('WARNING: File "%s" has high sky level (%s >= %s). Correction NOT applied' % (ccd.header['FILENAME'], bkg.background_median, max_val))
else:
print ('WARNING: File has high sky level (%s >= %s). Correction NOT applied' % (bkg.background_median, max_val))
return ccd
ccd.data -= bkg.background
ccd.header[skysub] = True
ccd.header[skyval] = bkg.background_median
ccd.header['SKYTYPE'] = '2D'
ccd.header['BKGTYPE'] = bkg_method
ccd.bkg = bkg
if fitsky is not None:
pyfits.writeto(fitsky, bkg.background, clobber=True)
return ccd
def remove_image_background_sextractor(data,
sigma=3.0,
box_size=(50, 50),
filter_size=(3, 3),
positive=False):
sigma_clip = SigmaClip(sigma=sigma)
bkg_estimator = SExtractorBackground()
bkg = Background2D(data,
box_size,
filter_size=filter_size,
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
data_wo_bkg = data - bkg.background
if positive:
data_wo_bkg -= np.min(data_wo_bkg)
if parameters.DEBUG:
fig, ax = plt.subplots(1, 2, figsize=(11, 5))
plot_image_simple(ax[0], bkg.background, scale="lin")
plot_image_simple(ax[1], data_wo_bkg, scale="symlog")
fig.tight_layout()
plt.show()
if parameters.PdfPages:
parameters.PdfPages.savefig()
return data_wo_bkg
def background_rms(data):
"""
Computes the RMS from the image stored in the array data.
Parameters
----------
data : array
Image array.
Returns
-------
rms_median :
List of stacked images
"""
sigmaClipper = SigmaClip(sigma=3, maxiters=5)
bck_estimator = SExtractorBackground(data)
bkg = Background2D(data, (50, 50),
filter_size=(3, 3),
sigma_clip=sigmaClipper,
bkg_estimator=bck_estimator)
print((bkg.background_median, bkg.background_rms_median))
return bkg.background_rms_median
'gausscluster_N2000_mag22_subpixel')
guess_table = prepare_table(tab_gausscluster, perturbation=1)
fit_stages = [
FitStage(1, 1e-11, 1e-11, np.inf, all_individual
), # first stage: get flux approximately right
FitStage(1, 0.6, 0.6, 200_000, brightest_simultaneous(2)),
FitStage(1, 0.6, 0.6, 100_000, not_brightest_individual(2)),
FitStage(1, 0.3, 0.3, 100_000, brightest_simultaneous(4)),
FitStage(1, 0.3, 0.3, 50_000, not_brightest_individual(4)),
FitStage(20, 0.2, 0.2, 20_000, all_individual),
FitStage(20, 0.1, 0.1, 5000, all_simultaneous)
]
photometry = IncrementalFitPhotometry(SExtractorBackground(),
anisocado_psf,
max_group_size=12,
group_extension_radius=20,
fit_stages=fit_stages,
use_noise=True)
with open(out_dir / 'processing_params_synthetic_cluster.txt', 'w') as f:
f.write(pformat(photometry.__dict__))
cluster_result = cached(
lambda: photometry.do_photometry(img_gausscluster, guess_table),
cache_dir / 'synthetic_gausscluster',
rerun=False)
[(np.min(np.abs(i)), np.mean(np.abs(i)), np.max(np.abs(i)))
for i in calc_devation(cluster_result)]
def _compute_background(image, box_size=50, win_size=3, nsigma=5., threshold_flag=None):
"""Use Background2D to determine the background of the input image.
Parameters
----------
image : ndarray
Numpy array of the science extension from the observations FITS file.
box_size : int
Size of box along each axis
win_size : int
Size of 2D filter to apply to the background image
nsigma : float
Number of sigma above background
threshold_flag : float or None
Value from the image which serves as the limit for determining sources.
If None, compute a default value of (background+5*rms(background)).
If threshold < 0.0, use absolute value as scaling factor for default value.
Returns
-------
bkg : 2D ndarray
Background image
bkg_dao_rms : ndarry
Background RMS image
threshold : ndarray
Numpy array representing the background plus RMS
"""
# Report configuration values to log
log.info("")
log.info("Computation of white light image background - Input Parameters")
log.info("Box size: {}".format(box_size))
log.info("Window size: {}".format(win_size))
log.info("NSigma: {}".format(nsigma))
# SExtractorBackground ans StdBackgroundRMS are the defaults
bkg_estimator = SExtractorBackground()
bkgrms_estimator = StdBackgroundRMS()
bkg = None
bkg_dao_rms = None
exclude_percentiles = [10, 25, 50, 75]
for percentile in exclude_percentiles:
log.info("")
log.info("Percentile in use: {}".format(percentile))
try:
bkg = Background2D(image, box_size,
filter_size=win_size,
bkg_estimator=bkg_estimator,
bkgrms_estimator=bkgrms_estimator,
exclude_percentile=percentile,
edge_method="pad")
except Exception:
bkg = None
continue
if bkg is not None:
# Set the bkg_rms at "nsigma" sigma above background
bkg_rms = nsigma * bkg.background_rms
default_threshold = bkg.background + bkg_rms
bkg_rms_mean = bkg.background.mean() + nsigma * bkg_rms.std()
bkg_dao_rms = bkg.background_rms
if threshold_flag is None:
threshold = default_threshold
elif threshold_flag < 0:
threshold = -1 * threshold_flag * default_threshold
log.info("Background threshold set to {} based on {}".format(threshold.max(), default_threshold.max()))
bkg_rms_mean = threshold.max()
else:
bkg_rms_mean = 3. * threshold_flag
threshold = bkg_rms_mean
if bkg_rms_mean < 0:
bkg_rms_mean = 0.
break
# If Background2D does not work at all, define default scalar values for
# the background to be used in source identification
if bkg is None:
bkg_rms_mean = max(0.01, image.min())
bkg_rms = nsigma * bkg_rms_mean
bkg_dao_rms = bkg_rms_mean
threshold = bkg_rms_mean + bkg_rms
# *** FIX: Need to do something for bkg if bkg is None ***
# Report other useful quantities
log.info("")
log.info("Mean background: {}".format(bkg.background.mean()))
log.info("Mean threshold: {}".format(bkg_rms_mean))
log.info("")
log.info("{}".format("=" * 80))
return bkg, bkg_dao_rms, threshold
from photutils import make_source_mask
mask = make_source_mask(data, snr=2, npixels=5, dilate_size=11)
mean, median, std = sigma_clipped_stats(data, sigma=3.0, mask=mask)
print((mean, median, std))
ny, nx = data.shape
y, x = np.mgrid[:ny, :nx]
gradient = x * y / 5000.
data2 = data + gradient
plt.imshow(data2, norm=norm, origin='lower', cmap='Greys_r')
plt.show()
from astropy.stats import SigmaClip
from photutils import Background2D, SExtractorBackground
sigma_clip = SigmaClip(sigma=3., iters=10)
bkg_estimator = SExtractorBackground()
bkg = Background2D(data2, (50, 50),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator)
plt.imshow(data2, norm=norm, origin='lower', cmap='Greys_r')
bkg.plot_meshes(outlines=True, color='#1f77b4')
plt.show()
print(bkg.background_median)
print(bkg.background_rms_median)
plt.imshow(bkg.background, origin='lower', cmap='Greys_r')
plt.show()
#from scipy.ndimage import rotate
#data3 = rotate(data2, -45.)
def flux(nddata, tbl, zeroPoint, gain, radius):
"""
Derive the flux and the magnitude within circular aperture
Parameters
----------
nddata: numpy array
Numpy array where is saved the data and the sky coordinates wcs.
tbl: table
Table where the first column is the id, the second the ra coordinate, the third
is dec coordinate and the four the skycoordinate.
zeroPoint: float
Zero point of your image
gain: float
The gain of your image
radius: float
The radius of your circle in arcsec to derive the flux
Return
------
Table with id, skycoord, flux, flux error, magnitude, magnitude error
"""
# By convention, sextractor
if gain == 0.:
gain = 1000000000000000000000000000.
result = tbl['id', 'skycoord']
result['flux'] = float(np.size(tbl))
result['flux_err'] = float(np.size(tbl))
for i in range(np.size(tbl)):
# Recover the position for each object
position = tbl['skycoord'][i]
if hdr['NAXIS1'] >= 50 and hdr['NAXIS2'] >= 50:
# size of the background map
sizeBkg = 50
# cut the mosaic in stamp to the wcs coordinate of your objects
cutout = Cutout2D(nddata.data,
position, (sizeBkg, sizeBkg),
wcs=nddata.wcs)
# Recover new data and wcs of the stamp
data = cutout.data
wcs = cutout.wcs
else:
# size of the background map
sizeBkg = min(hdr['NAXIS1'], hdr['NAXIS2'])
# Keep data and wcs of the initial image
data = nddata.data
wcs = nddata.wcs
#########################
####### Background ######
#########################
# Mask sources
mask = make_source_mask(data, snr=1, npixels=3, dilate_size=3)
# Derive the background and the rms image
bkg = Background2D(
data,
int(sizeBkg / 10),
filter_size=1,
sigma_clip=None,
bkg_estimator=SExtractorBackground(SigmaClip(sigma=2.5)),
bkgrms_estimator=StdBackgroundRMS(SigmaClip(sigma=2.5)),
exclude_percentile=60,
mask=mask)
###########################
###### Aperture Flux ######
###########################
nddataStamp = NDData(data=data - bkg.background, wcs=wcs)
# Calculate the total error
error = calc_total_error(cutout.data, bkg.background_rms, gain)
# Define a cicularAperture in the wcs position of your objects
apertures = SkyCircularAperture(position, r=radius * u.arcsec)
# Derive the flux and error flux
phot_table = aperture_photometry(nddataStamp, apertures, error=error)
phot_table['aperture_sum'].info.format = '%.8g'
phot_table['aperture_sum_err'].info.format = '%.8g'
# Recover data
result['flux'][i] = phot_table['aperture_sum'][0]
result['flux_err'][i] = phot_table['aperture_sum_err'][0]
###########################
######## Magnitude ########
###########################
# convert flux into magnitude
result['mag'] = -2.5 * np.log10(result['flux']) + zeroPoint
# convert flux error into magnitude error
result['mag_err'] = 1.0857 * (result['flux_err'] / result['flux'])
return result
def extract_spectrogram_background_sextractor(data,
err,
ws=(20, 30),
mask_signal_region=True):
"""
Use photutils library median filter to estimate background behgin the sources.
Parameters
----------
data: array
The 2D array image. The transverse profile is fitted on the y direction for all pixels along the x direction.
err: array
The uncertainties related to the data array.
ws: list
up/down region extension where the sky background is estimated with format [int, int] (default: [20,30])
mask_signal_region: bool
If True, the signal region is masked with np.nan values (default: True)
Returns
-------
bgd_model_func: callable
A 2D function to model the extracted background.
bgd_res: array_like
The background residuals normalized with their uncertainties.
bgd_rms: array_like
The background RMS.
Examples
--------
Build a mock spectrogram with random Poisson noise:
>>> from spectractor.extractor.psf import MoffatGauss
>>> from spectractor.extractor.chromaticpsf import ChromaticPSF
>>> from spectractor import parameters
>>> parameters.DEBUG = True
>>> psf = MoffatGauss()
>>> s0 = ChromaticPSF(psf, Nx=100, Ny=100, saturation=1000)
>>> params = s0.generate_test_poly_params()
>>> saturation = params[-1]
>>> data = s0.build_spectrogram_image(params, mode="1D")
>>> bgd = 10*np.ones_like(data)
>>> data += bgd
>>> data = np.random.poisson(data)
>>> data_errors = np.sqrt(data+1)
Fit the transverse profile:
>>> bgd_model, _, _ = extract_spectrogram_background_sextractor(data, data_errors, ws=[30,50])
"""
Ny, Nx = data.shape
middle = Ny // 2
# mask sources
mask = make_source_mask(data, nsigma=3, npixels=5, dilate_size=11)
# Estimate the background in the two lateral bands together
sigma_clip = SigmaClip(sigma=3.)
bkg_estimator = SExtractorBackground()
if mask_signal_region:
bgd_bands = np.copy(data).astype(float)
bgd_bands[middle - ws[0]:middle + ws[0], :] = np.nan
mask += (np.isnan(bgd_bands))
# windows size in x is set to only 6 pixels to be able to estimate rapid variations of the background on real data
# filter window size is set to window // 2 so 3
# bkg = Background2D(data, ((ws[1] - ws[0]), (ws[1] - ws[0])),
# filter_size=((ws[1] - ws[0]) // 2, (ws[1] - ws[0]) // 2),
# sigma_clip=sigma_clip, bkg_estimator=bkg_estimator,
# mask=mask)
bkg = Background2D(
data, (parameters.PIXWIDTH_BOXSIZE, parameters.PIXWIDTH_BOXSIZE),
filter_size=(parameters.PIXWIDTH_BOXSIZE // 2,
parameters.PIXWIDTH_BOXSIZE // 2),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask)
bgd_model_func = interp2d(np.arange(Nx),
np.arange(Ny),
bkg.background,
kind='linear',
bounds_error=False,
fill_value=None)
bgd_res = ((data - bkg.background) / err)
bgd_res[mask] = np.nan
if parameters.DEBUG:
gs = gridspec.GridSpec(3,
2,
width_ratios=[4, 1],
height_ratios=[1, 1, 1],
wspace=0.1,
hspace=0.04,
right=0.98,
left=0.1,
top=0.98,
bottom=0.1)
fig = plt.figure(figsize=(7, 5))
ax0 = plt.subplot(gs[0, 0])
ax1 = plt.subplot(gs[1, 0])
ax2 = plt.subplot(gs[2, 0])
ax3 = plt.subplot(gs[:, 1])
bgd_bands = np.copy(data).astype(float)
bgd_bands[middle - ws[0]:middle + ws[0], :] = np.nan
bgd_bands[mask] = np.nan
mean = np.nanmean(bgd_bands)
std = np.nanstd(bgd_bands)
cmap = copy.copy(cm.get_cmap())
cmap.set_bad(color='lightgrey')
im = ax0.imshow(bgd_bands,
origin='lower',
aspect="auto",
vmin=mean - 3 * std,
vmax=mean + 3 * std)
v1 = np.linspace(mean - 3 * std, mean + 3 * std, 5, endpoint=True)
cb = plt.colorbar(im, ticks=v1, ax=ax0, label=f'Data units')
cb.ax.set_yticklabels(["{:2.0f}".format(i) for i in v1])
ax0.text(0.05,
0.95,
f'Data background',
color="white",
horizontalalignment='left',
verticalalignment='top',
transform=ax0.transAxes
) # : mean={np.mean(b):.3f}, std={np.std(b):.3f}')
ax0.set_xlabel(parameters.PLOT_XLABEL)
ax0.set_ylabel(parameters.PLOT_YLABEL)
ax0.set_xticks([])
ax1.set_xticks([])
bkg.plot_meshes(outlines=True, color='red', axes=ax1, linewidth=0.5)
b = bkg.background
im = ax1.imshow(b, origin='lower', aspect="auto")
ax1.set_xlabel(parameters.PLOT_XLABEL)
ax1.set_ylabel(parameters.PLOT_YLABEL)
v1 = np.linspace(np.nanmin(b), np.nanmax(b), 5, endpoint=True)
cb1 = plt.colorbar(im, ticks=v1, ax=ax1, label=f'Data units')
cb1.ax.set_yticklabels(["{:2.0f}".format(i) for i in v1])
ax1.text(0.05,
0.95,
f'Fitted background',
color="white",
horizontalalignment='left',
verticalalignment='top',
transform=ax1.transAxes
) # : mean={np.mean(b):.3f}, std={np.std(b):.3f}')
res = (bgd_bands - b) / err
im = ax2.imshow(res, origin='lower', aspect="auto", vmin=-5, vmax=5)
ax2.set_xlabel(parameters.PLOT_XLABEL)
ax2.set_ylabel(parameters.PLOT_YLABEL)
ax2.text(0.05,
0.95,
f'Pull: mean={np.nanmean(res):.3f}, std={np.nanstd(res):.3f}',
color="white",
horizontalalignment='left',
verticalalignment='top',
transform=ax2.transAxes
) # : mean={np.mean(b):.3f}, std={np.std(b):.3f}')
v1 = np.array([-5, -2, 0, 2, 5])
cb2 = plt.colorbar(im, ticks=v1, ax=ax2, label=f'')
cb2.ax.set_yticklabels(["{:1.0f}".format(i) for i in v1])
# ax3.set_title(f'Pull') # mean={np.nanmean(res):.3f}, std={np.nanstd(res):.3f}')
hist_res = res[~np.isnan(res)].flatten()
hist_res = hist_res[hist_res < 5]
hist_res = hist_res[hist_res > -5]
ax3.hist(hist_res, bins=10)
ax3.grid()
ax3.set_yticks([])
# ax3.set_xlim((-5, 5))
# ax3.set_xticks(v1)
ax3.set_xlabel('Pull distribution')
# fig.tight_layout()
if parameters.LSST_SAVEFIGPATH: # pragma: no cover
fig.savefig(
os.path.join(parameters.LSST_SAVEFIGPATH,
'background_extraction.pdf'))
if parameters.DISPLAY: # pragma: no cover
plt.show()
return bgd_model_func, bgd_res, bkg.background_rms
# -*- coding: utf-8 -*- """ Created on Fri Oct 11 15:36:33 2019 @author: dingxu """ from astropy.io import fits from astropy.stats import SigmaClip from photutils import SExtractorBackground fitsname = 'NGC%2011.fts' routename1 = 'E:\\shunbianyuan\\ldf_download\\20190921\\' fitsname1 = routename1+fitsname hduA1 = fits.open(fitsname1) imagedataA1 = hduA1[0].data sigma_clip = SigmaClip(sigma = 3.0) bkg = SExtractorBackground(sigma_clip) bkg_value = bkg.calc_background(imagedataA1) print(bkg_value)
def measure_bkg(img, if_plot=False, nsigma=2, npixels=25, dilate_size=11):
"""
Estimate the 2D background of a image, based on photutils.Background2D, with SExtractorBackground.
Checkout: https://photutils.readthedocs.io/en/stable/background.html
Parameter
--------
parameters (including nsigma, npixels and dilate_size) will be passed to photutils.make_source_mask()
Return
--------
A 2D array type of the background light.
"""
print("Estimating the background light ... ... ...")
from astropy.stats import SigmaClip
from photutils import Background2D, SExtractorBackground
try:
sigma_clip = SigmaClip(sigma=3., maxiters=10)
except (TypeError):
sigma_clip = SigmaClip(sigma=3., iters=10)
# if version.parse(astropy.__version__) >= version.parse("0.4"):
# sigma_clip = SigmaClip(sigma=3., maxiters=10)
# else:
# sigma_clip = SigmaClip(sigma=3., iters=10)
bkg_estimator = SExtractorBackground()
if version.parse(photutils.__version__) > version.parse("0.7"):
mask_0 = make_source_mask(img,
nsigma=nsigma,
npixels=npixels,
dilate_size=dilate_size)
else:
mask_0 = make_source_mask(img,
snr=nsigma,
npixels=npixels,
dilate_size=dilate_size)
mask_1 = (np.isnan(img))
mask = mask_0 + mask_1
box_s = int(len(img) / 40)
if box_s < 10:
box_s = 10
bkg = Background2D(img, (box_s, box_s),
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=bkg_estimator,
mask=mask)
from matplotlib.colors import LogNorm
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(img, norm=LogNorm(), origin='lower')
#bkg.plot_meshes(outlines=True, color='#1f77b4')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if if_plot:
plt.show()
else:
plt.close()
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(mask, origin='lower')
#bkg.plot_meshes(outlines=True, color='#1f77b4')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if if_plot:
plt.show()
else:
plt.close()
bkg_light = bkg.background * ~mask_1
fig = plt.figure(figsize=(15, 15))
ax = fig.add_subplot(1, 1, 1)
ax.imshow(bkg_light, origin='lower', cmap='Greys_r')
ax.xaxis.set_visible(False)
ax.yaxis.set_visible(False)
if if_plot:
plt.show()
else:
plt.close()
return bkg_light
def stack(cutout, radiusStack, gain, zeroPoint, wavelenght):
"""
Print out pdf with stamp stacking and plot graph magnitude
in function of wavelenght
Parameters
---------_
cutout: list
array of images N filt x N sources
radiusStack: float
radius in pixel
zeroPoint: float
Zero point of your image
gain: float
The gain of your image
wavelenght: float
wavelenght of each filters
"""
# define the resulting stacked image as a cutout with the same dimension of the input cutouts
o = 0
# Create pdf where are storing stack stamp and graph
pdfOut = PdfPages(dirpath + 'stacking.pdf')
fig, axs = plt.subplots(1, len(filters), figsize=(5, 5))
# Saving magnitude and error
mag = [[0.0 for i in range(len(filters))] for j in range(2)]
for i in cutout: # i is a list of galaxies
print('Photometry ' + filters[o])
# Assuming that the shape is the same for all galaxies
#this is now a tuple, e.g. (25,25), so the code will work also for rectangular stamps
sizeStack = i[0].shape
#this is now a tuple, e.g. (25,25), so the code will work also for rectangular stamps
print(sizeStack) #just for testing
stackImg = np.zeros(sizeStack)
#LOOP over pixels: (I changed a lot here)
# fait tous les pixel x
for x in range(sizeStack[0]):
# fait tous les pixel y
for y in range(sizeStack[1]):
# for the given pixels, collect the flux from all stamps into a list
pxl = [] #use an empty list, but it can be also np.zeros()
for stamp in i:
pxl.append(stamp.data[x, y])
# caluclate the median:
stackImg[x, y] = np.median(pxl)
axs[o].set_title(filters[o], fontsize=5, pad=2.5)
axs[o].get_xaxis().set_visible(False)
axs[o].get_yaxis().set_visible(False)
mappa = axs[o].imshow(stackImg,
cmap='afmhot',
origin='lower',
interpolation='nearest')
zrange = zscale.zscale(stackImg)
mappa.set_clim(zrange)
# Mask sources
mask = make_source_mask(stackImg, snr=1, npixels=3, dilate_size=3)
# Derive the background and the rms image
bkg = Background2D(
stackImg,
int(sizeStack[0] / 10),
filter_size=1,
sigma_clip=None,
bkg_estimator=SExtractorBackground(SigmaClip(sigma=2.5)),
bkgrms_estimator=StdBackgroundRMS(SigmaClip(sigma=2.5)),
exclude_percentile=90,
mask=mask)
if gain[o] == 0.:
gain[o] = 1000000000000000000000000000.
# Calculate the total error
error = calc_total_error(stackImg, bkg.background_rms, gain[o])
# Define a cicularAperture in the wcs position of your objects
apertures = CircularAperture(
(int(sizeStack[0] / 2), int(sizeStack[1] / 2)), r=radiusStack[o])
# Derive the flux and error flux
photometry = aperture_photometry(stackImg - bkg.background,
apertures,
error=error)
# Saving magnitude and error
mag[0][o] = -2.5 * np.log10(
photometry['aperture_sum'][0]) + zeroPoint[o]
mag[1][o] = 1.0857 * (photometry['aperture_sum_err'][0] /
photometry['aperture_sum'][0])
o = o + 1
plt.savefig(pdfOut, format='pdf')
# Plot
plt.clf()
y = mag[0]
x = wavelenght
plt.plot(x, y, 'ro')
xlimL = wavelenght[0] - 1000
xlimR = wavelenght[len(wavelenght) - 1] + 1000
plt.xlim(xlimL, xlimR)
plt.ylim(32, 22)
plt.errorbar(wavelenght,
mag[0],
yerr=mag[1],
fmt='none',
capsize=10,
ecolor='blue',
zorder=1)
plt.xlabel('wavelenght(Å)')
plt.ylabel('magnitude')
plt.title('')
plt.savefig(pdfOut, format='pdf')
pdfOut.close()
np.savetxt(dirpath + 'fluxStack.txt',
mag,
header='Ligne1: mag, Ligne2: mag_err')
return
def write_intro(self, gids, outputfile, sigma_image, use_mask = False):
use_mask = True
# print('Use mask?', use_mask, gid)
# print(gid)
outputfilefits = outputfile+'.fits'
# if use_mask == False:
xmin, xmax, ymin, ymax = self.get_boundaries(gids)
# zeropoints taken from https://iopscience.iop.org/article/10.1088/0067-0049/214/2/24/pdf
if self.flter == '850':
if self.field == 'N':
inputpsf = 'goodsn_3dhst.v4.0.F850LP_psf.fits'
psffine = 'none'
zeropoint_og = 24.871
if self.field == 'S':
zeropoint_og = 24.871
inputpsf = 'goodss_3dhst.v4.0.F850LP_psf.fits'
psffine = 'none'
elif self.flter == '105':
if self.field =='S':
zeropoint_og = 23.972
inputpsf = 'none'
psffine = 'none'
else:
if self.flter == '160':
zeropoint_og = 25.946
if self.flter == '125':
zeropoint_og = 26.230
if self.flter == '606':
zeropoint_og = 26.511
if self.flter == '775':
zeropoint_og = 25.671
if self.field =='S':
inputpsf = 'goodss_3dhst.v4.0.F'+self.flter+'W_psf.fits'
psffine = 'goodss_3dhst.v4.0.F'+self.flter+'W_orig_wht.fits'
if self.field =='N':
inputpsf = 'goodsn_3dhst.v4.0.F'+self.flter+'W_psf.fits'
psffine = 'goodsn_3dhst.v4.0.F'+self.flter+'W_orig_wht.fits'
if use_mask == True:
# print('Using mask!', gid)
segmap = str(self.gid)+'_segm.fits'
# if os.path.exists('/Users/rosaliaobrien/research/GALFIT_CLEAR/running_GALFIT/segm_maps/'+segmap) == False:
print('Creating segmentation map...')
self.make_segm()
# Move segmentation map to working directory
os.rename('/Users/rosaliaobrien/research/GALFIT_CLEAR/running_GALFIT/segm_maps/'+segmap, segmap)
elif use_mask == False:
# print('Not using mask!', gid)
segmap = 'none'
zeropoint=zeropoint_og
if sigma_image == None:
sigma_image = 'none'
text = '''================================================================================\n\
# IMAGE and GALFIT CONTROL PARAMETERS\n\
A) {0} # Input data image (FITS file)\n\
B) {1} # Output data image block\n\
C) {9} # Sigma image name (made from data if blank or "none") \n\
D) {2} # Input PSF image and (optional) diffusion kernel\n
E) none # PSF fine sampling factor relative to data\n
F) {10} # Bad pixel mask (FITS image or ASCII coord list)\n\
G) none # File with parameter constraints (ASCII file) \n\
H) {4} {5} {6} {7} # Image region to fit (xmin xmax ymin ymax)\n\
I) 200 200 # Size of the convolution box (x y)\n\
J) {8} # Magnitude photometric zeropoint \n\
K) 0.06 0.06 # Plate scale (dx dy) [arcsec per pixel]\n\
O) regular # Display type (regular, curses, both)\n\
P) 0 # Options: 0=normal run; 1,2=make model/imgblock & quit
\n
'''.format(self.fitsfile, outputfilefits, inputpsf, psffine, xmin, xmax, ymin, ymax, zeropoint, sigma_image, segmap)
path = '/Users/rosaliaobrien/research/GALFIT_CLEAR/running_GALFIT/'+self.fullfield+'/'+self.flter+'/'
skydata = fits.open(path+self.fitsfile)[0].data
xmid = (xmin+xmax)/2
ymid = (ymin+ymax)/2
cutout = Cutout2D(skydata, (xmid, ymid), (400, 400), copy=True, mode='partial')
masked_cutout = make_source_mask(cutout.data, snr = 1.5,
npixels=10, dilate_size=11)
cutout.data[masked_cutout] = np.nan
self.scidata = cutout.data
sigma_clip = SigmaClip(sigma=3)
bkg = SExtractorBackground(sigma_clip)
rms = MADStdBackgroundRMS(sigma_clip)
self.sky_value = bkg.calc_background(cutout.data)
self.rms_value = rms.calc_background_rms(cutout.data)
###PLOT MASKED IMAGE AND SEGMENTATION MAP###
if use_mask == False: #Plot 1 subplot if not using segmentation map
num_plots_x = 1
num_plots_y = 1
else:
num_plots_x = 2
num_plots_y = 1
fig, ax = plt.subplots(num_plots_y, num_plots_x, figsize = (5,2))
ax_og = ax #Make 'original' axis in case you splot segmentation map
if use_mask == True:
ax = ax[0]
im = ax.imshow(self.scidata, vmin = -100, vmax = 100)
plt.colorbar(im, ax = ax)
ax.set_title('gid: {}\nSky: {:.2f}\nRMS: {:.2f}'.format(self.gid, self.sky_value, self.rms_value), size=9)
ax.set_xticklabels([])
ax.set_yticklabels([])
if use_mask == False:
plt.show()
########################
if use_mask == True:
ax = ax_og
### PLOT SEGMAP ###
segmdata = fits.open(segmap)[0].data
xmid = (xmin+xmax)/2
ymid = (ymin+ymax)/2
cutout = Cutout2D(segmdata, (xmid, ymid), (400, 400), copy=True, mode='partial')
ax[1].imshow(cutout.data)
ax[1].set_title('segmentation map'.format(self.gid, self.sky_value, self.rms_value), size=9)
ax[1].set_xticklabels([])
ax[1].set_yticklabels([])
plt.show()
########################
return text
def estimate_background(image, method='1D', sigma=3, diag=False):
'''Background estimation.
Generate an estimated background image and median background rms from a given input image.
Parameters
----------
image: array
2D array containing the image values, no units
method: string
'2D' or '1D'. 2D is suitable for large images. 1D is more appropriate for small images, especially those with uniform background levels.
sigma: float
Sigma clip level for background estimation. Default is 3. For 1D background estimation in small frames, sigma should be 2.
For 2D background estimation, sigma should usually be 3 or 4.
Returns
-------
bkg_image: array
2D array the same size as image of the estimated background
bkg_rms: float
rms median of the background
Example
-------
>>> np.random.seed(0)
>>> im = np.ones((10,10)) + np.random.uniform(size=(10,10))
>>> im *= u.ph
>>> im[5,5] += 5 * u.ph
>>> bkg_im, bkg_med = estimate_background(im, method='2D', sigma=4)
>>> np.allclose([bkg_im[0,0].value, bkg_med.value],
... [1.45771779, 0.288040735])
True
>>> bkg_im_1d, bkg_med_1d = estimate_background(im, method='1D', sigma=4)
>>> np.allclose([bkg_im_1d[0,0].value, bkg_med_1d.value],
... [1.4686512016477016, 0.28804073501451516])
True
'''
if method == '2D':
from photutils import Background2D, SExtractorBackground
from astropy.stats import SigmaClip
# Define estimator (we're using the default SExtractorBackground estimator from photutils)
bkg_estimator = SExtractorBackground()
# Want the box size to be ~10 pixels or thereabouts
boxes0 = np.int(1 if image.shape[0] <= 5 else np.round(image.shape[0] /
10))
boxes1 = np.int(1 if image.shape[1] <= 5 else np.round(image.shape[1] /
10))
bkg = Background2D(
image.value, (image.shape[0] // boxes0, image.shape[1] // boxes1),
bkg_estimator=bkg_estimator,
sigma_clip=SigmaClip(sigma=sigma))
bkg_image = bkg.background * image.unit
bkg_rms_median = bkg.background_rms_median * image.unit
if diag:
print("Image shape:", image.shape)
print("Boxes per axis: {}, {}".format(boxes0, boxes1))
print("Box size: {}, {}".format(image.shape[0] // boxes0,
image.shape[1] // boxes1))
elif method == '1D':
from astropy.stats import sigma_clipped_stats
bkg_image = np.zeros(np.shape(image)) * image.unit
# Get mean, median and std using a sigmaclip of 1:
bkg_mean, bkg_median, bkg_rms_median = sigma_clipped_stats(image,
sigma=sigma,
maxiters=10)
bkg_image[:] = bkg_median
return bkg_image, bkg_rms_median
def make_lc(tic, ra=None, dec=None, process=None):
print('Calculando Camara y CCD')
_, _, _, _, cam, ccd, _, _, _ = ts2p(0, ra, dec, trySector=args.Sector)
idx = (cam[0] - 1) * 4 + (ccd[0] - 1)
print('Leyendo Header')
h5 = h5s[idx]
q = h5['data'][3] == 0
ffi = np.array(
glob.glob('/raid/TESS/FFI/s%04d/tess*-s%04d-%d-%d-*ffic.fits' %
(args.Sector, args.Sector, cam, ccd)))[q][0]
hdr = fits.getheader(ffi, 1)
w = WCS(hdr)
x, y = w.all_world2pix(ra, dec, 0)
print('Leyendo hdf5')
allhdus = FFICut(h5, y, x, args.size)
hdus = allhdus.hdu
qual = hdus[1].data['QUALITY'] == 0
time = hdus[1].data['TIME'][q]
flux = hdus[1].data['FLUX'][q]
errs = hdus[1].data['FLUX_ERR'][q]
bkgs = np.zeros(len(flux))
print('Calculando Background')
for i, f in enumerate(flux):
sigma_clip = SigmaClip(sigma=1)
bkg = SExtractorBackground(sigma_clip=sigma_clip)
bkgs[i] = bkg.calc_background(f)
#DBSCAN Aperture
x = x - int(x) + args.size // 2
y = y - int(y) + args.size // 2
if not args.circ:
daps = [
generate_aperture(flux - bkgs[:, None, None], n=i)
for i in [1, 2, 3, 4, 5]
]
dap = np.array([select_aperture(d, x, y) for d in daps])
else:
XX, YY = np.ogrid[:args.size, :args.size]
dap = [
np.sqrt((XX - y)**2 + (YY - x)**2) < i
for i in np.arange(1, 3.1, 0.5)
]
#Aperture photometry
lcfl = np.einsum('ijk,ljk->li', flux - bkgs[:, None, None], dap)
lcer = np.sqrt(np.einsum('ijk,ljk->li', np.square(errs), dap))
#Lightkurves
lkf = [
TessLightCurve(time=time, flux=lcfl[i], flux_err=lcer[i])
for i in range(len(lcfl))
]
#Select best
cdpp = [lk.estimate_cdpp() for lk in lkf]
bidx = np.argmin(cdpp)
lkf = lkf[bidx]
#Save light curve
inst = np.repeat('TESS', len(time))
output = np.transpose([time, lkf.flux, lkf.flux_err, inst])
np.savetxt('TIC%s.dat' % tic, output, fmt='%s')
print('LC READY!')
#Save JPG preview
stamp = flux - bkgs[:, None, None]
fig, ax = plt.subplots(figsize=[4, 4])
fig.patch.set_visible(False)
stamp = np.log10(np.nanmedian(stamp[::10], axis=0))
stamp[np.isnan(stamp)] = np.nanmedian(stamp)
ax.matshow(stamp, cmap='gist_gray', aspect='equal')
xm, ym = pixel_border(dap[bidx])
for xi, yi in zip(xm, ym):
ax.plot(xi, yi, color='lime', lw=1.25)
ax.plot(x, y, '.r')
plt.axis('off')
fig.subplots_adjust(left=0, right=1, top=1, bottom=0, wspace=0, hspace=0)
fig.savefig('img/TIC%s.png' % tic, dpi=72)
plt.close(fig)
del (fig, ax)
return 1
w = WCS(hdr)
x, y = w.all_world2pix(ra[i], dec[i], 0)
allhdus = FFICut(h5, y, x, args.size)
hdus = allhdus.hdu
qual = hdus[1].data['QUALITY'] == 0
time = hdus[1].data['TIME'][q]
flux = hdus[1].data['FLUX'][q]
errs = hdus[1].data['FLUX_ERR'][q]
bkgs = np.zeros(len(flux))
#print('Calculando Background')
for i, f in enumerate(flux):
sigma_clip = SigmaClip(sigma=1)
bkg = SExtractorBackground(sigma_clip=sigma_clip)
bkgs[i] = bkg.calc_background(f)
#DBSCAN Aperture
x = x - int(x) + flux.shape[1] // 2
y = y - int(y) + flux.shape[2] // 2
if not args.circ:
daps = [
generate_aperture(flux - bkgs[:, None, None], n=i)
for i in [1, 2, 3, 4, 5]
]
dap = np.array([select_aperture(d, x, y) for d in daps])
else:
XX, YY = np.ogrid[:flux.shape[1], :flux.shape[2]]
dap = [
color_print('CCD Row: ', 'lightcyan', str(row+x), 'default', '\tCCD Column: ', 'lightcyan', str(column+y), 'default')
#Background and data
ma = hdus[1].data['QUALITY'] == 0
time = hdus[1].data['TIME'][ma]
flux = hdus[1].data['FLUX'][ma]
errs = hdus[1].data['FLUX_ERR'][ma]
bkgs = np.zeros(len(flux))
berr = np.zeros(len(flux))
#Constant background, bks != 0
if not args.backdrop:
for i,f in enumerate(flux):
sigma_clip = SigmaClip(sigma=1)
bkg = SExtractorBackground(sigma_clip=sigma_clip)
bkgs[i] = bkg.calc_background(f) if args.manualap is None else bkg.calc_background(f[~theap])
mad_bkg = mad_std(f)
berr[i] = (3*1.253 - 2)*mad_bkg/np.sqrt(f.size)
#Manual aperture
if args.manualap is not None:
apix2, apix1 = np.genfromtxt(args.manualap, unpack=True).astype(int)
theap = np.zeros(flux[0].shape).astype(bool)
theap[apix1, apix2] = True
#PRF from lightkurve
#if args.prf:
def rm_bkg(source,
syntax,
xc=0,
yc=0):
import logging
import warnings
import numpy as np
from astropy.stats import SigmaClip
from photutils import Background2D, MedianBackground,MADStdBackgroundRMS, BkgIDWInterpolator
from autophot.packages.aperture import ap_phot
from astropy.modeling import models, fitting
from photutils import CircularAperture
from photutils import SExtractorBackground
logger = logging.getLogger(__name__)
# try:
# source_size = syntax['image_radius']
# except:
source_size = 2*syntax['fwhm']
try:
if syntax['psf_bkg_surface'] and not syntax['psf_bkg_poly']:
sigma_clip = SigmaClip(sigma=syntax['lim_SNR'])
bkg_estimator = MADStdBackgroundRMS()
positions = [source.shape[0]/2,source.shape[1]/2]
aperture = CircularAperture(positions, r=source_size)
masks = aperture.to_mask(method='center')
mask_array = masks.to_image(shape=((source.shape[0], source.shape[1])))
bkg = Background2D(source,
box_size = (3, 3),
mask = mask_array,
filter_size=(3, 3),
sigma_clip=sigma_clip,
bkg_estimator=SExtractorBackground(),
interpolator= BkgIDWInterpolator(),
# edge_method = 'pad'
)
surface = bkg.background
# bkg_median = np.nanmedian(bkg.background_median)
source_bkg_free = source - surface
elif syntax['psf_bkg_poly']:
surface_function_init = models.Polynomial2D(degree=syntax['psf_bkg_poly_degree'])
fit_surface = fitting.LevMarLSQFitter()
x = np.arange(0,source.shape[0])
y = np.arange(0,source.shape[0])
xx,yy= np.meshgrid(x,y)
positions = [source.shape[0]/2,source.shape[1]/2]
aperture = CircularAperture(positions, r=source_size)
masks = aperture.to_mask(method='center')
mask_array = masks.to_image(shape=((source.shape[0], source.shape[1])))
source[mask_array.astype(bool)] == np.nan
with warnings.catch_warnings():
# Ignore model linearity warning from the fitter
warnings.simplefilter('ignore')
surface_fit = fit_surface(surface_function_init, xx, yy, source)
surface = surface_fit(xx,yy)
# bkg_median = np.nanmedian(surface)
source_bkg_free = source - surface
elif syntax['psf_bkg_local']:
pos = list(zip([xc],[yc]))
ap,bkg = ap_phot(pos,
source,
radius = syntax['ap_size'] * syntax['fwhm'],
r_in = syntax['r_in_size'] * syntax['fwhm'],
r_out = syntax['r_out_size'] * syntax['fwhm'])
surface = np.ones(source.shape) * bkg
source_bkg_free = source - surface
except Exception as e:
logger.exception(e)
return source_bkg_free,surface
def GetData(TIC,
indir,
outdir,
cadence='lc',
S=None,
fits=None,
aperture_name=None,
max_pixels=75,
saturation_tolerance=-0.1,
bad_bits=[
1, 2, 4, 8, 16, 32, 64, 128, 136, 160, 164, 168, 176, 180, 256,
512, 1024, 2048
],
**kwargs):
if S is None:
raise Exception
sector = ast.literal_eval(S)
tpf = os.path.join(indir, fits)
with pyfits.open(tpf) as f:
qdata = f[1].data
tpf_aperture = None
tpf_big_aperture = None,
apertures = {'tpf': tpf_aperture, 'tpf_big': tpf_big_aperture}
fitsheader = [
pyfits.getheader(tpf, 0).cards,
pyfits.getheader(tpf, 1).cards,
pyfits.getheader(tpf, 2).cards
]
cadn = np.array(qdata.field('CADENCENO'), dtype='int32')
time = np.array(qdata.field('TIME'), dtype='float64')
fpix = np.array(qdata.field('FLUX'), dtype='float64')
fpix_err = np.array(qdata.field('FLUX_ERR'), dtype='float64')
qual = np.array(qdata.field('QUALITY'), dtype=int)
naninds = np.where(np.isnan(time))
time = Interpolate(np.arange(0, len(time)), naninds, time)
pc1 = np.array(qdata.field('POS_CORR1'), dtype='float64')
pc2 = np.array(qdata.field('POS_CORR2'), dtype='float64')
if not np.all(np.isnan(pc1)) and not np.all(np.isnan(pc2)):
pc1 = Interpolate(time, np.where(np.isnan(pc1)), pc1)
pc2 = Interpolate(time, np.where(np.isnan(pc2)), pc2)
else:
pc1 = None
pc2 = None
f97 = np.zeros((fpix.shape[1], fpix.shape[2]))
for i in range(fpix.shape[1]):
for j in range(fpix.shape[2]):
tmp = np.delete(fpix[:, i, j], np.where(np.isnan(fpix[:, i, j])))
if len(tmp):
f = SavGol(tmp)
med = np.nanmedian(f)
MAD = 1.4826 * np.nanmedian(np.abs(f - med))
bad = np.where((f > med + 10. * MAD)
| (f < med - 10. * MAD))[0]
np.delete(tmp, bad)
i97 = int(0.975 * len(tmp))
tmp = tmp[np.argsort(tmp)[i97]]
f97[i, j] = tmp
sav_ap_name = kwargs.get('sav_aper_name', 'aperture')
Nmax = np.copy(max_pixels)
co = 0
while Nmax >= max_pixels:
ticmag = fitsheader[0]['TESSMAG'][1]
if ticmag > 6.5:
ss = 1.5
elif 6.5 >= ticmag > 9.:
ss = 1.3
elif 9. >= ticmag > 12.:
ss = 1.2
elif ticmag < 12.:
ss = 1.1
aperture, Nmax = A3d(fpix, N_pixels_max=max_pixels - 1, sigma=ss)
if np.sum(aperture) == 0.0:
sys.exit("WARNING: Aperture can't be created, try another TIC")
elif np.sum(aperture) == 1.0:
ii = np.where(aperture == 1.)
try:
aperture[ii[0] + 1, ii[1]] = 1.
except:
pass
try:
aperture[ii[0] - 1, ii[1]] = 1.
except:
pass
try:
aperture[ii[0], ii[1] + 1] = 1.
except:
pass
try:
aperture[ii[0], ii[1] - 1] = 1.
except:
pass
if fpix.shape[1] < 12 and fpix.shape[2] < 12: break
if Nmax >= max_pixels:
nanind = np.where(np.isnan(fpix_err) == 1)
fpix_err[nanind] = 0.
bin_row = int(np.ceil(fpix.shape[1] / 12.))
res_row = fpix.shape[1] % bin_row
if res_row != 0:
add_row = bin_row - res_row
fpix_row = np.ndarray(shape=(fpix.shape[0],
fpix.shape[1] + add_row,
fpix.shape[2]))
fpix_err_row = np.ndarray(shape=(fpix.shape[0],
fpix.shape[1] + add_row,
fpix.shape[2]))
for fi, fp in enumerate(fpix):
SexBkg = SExtractorBackground()
fp[fp <= 0.] = 0.0
try:
fp_bkg = np.ones(
fpix.shape[2]) * SexBkg.calc_background(fp)
except:
fp_bkg = np.zeros(fpix.shape[2])
for ad in np.arange(add_row):
fp = np.vstack((fp, fp_bkg))
fpix_row[fi] = fp
for fi, fe in enumerate(fpix_err):
for ad in np.arange(add_row):
fe = np.vstack((fe, np.zeros(fpix_err.shape[2])))
fpix_err_row[fi] = fe
fpix = np.copy(fpix_row)
fpix_err = np.copy(fpix_err_row)
bin_col = int(np.ceil(fpix.shape[2] / 12.))
res_col = fpix.shape[2] % bin_col
if res_col != 0:
add_col = bin_col - res_col
fpix_col = np.ndarray(shape=(fpix.shape[0], fpix.shape[1],
fpix.shape[2] + add_col))
fpix_err_col = np.ndarray(shape=(fpix.shape[0], fpix.shape[1],
fpix.shape[2] + add_col))
for fi, fp in enumerate(fpix):
SexBkg = SExtractorBackground()
fp[fp <= 0.] = 0.0
try:
fp_bkg = np.ones(
fpix.shape[1]) * SexBkg.calc_background(fp)
except:
fp_bkg = np.zeros(fpix.shape[1])
fp_bkg = fp_bkg.reshape(fp_bkg.shape[0], 1)
for ad in np.arange(add_col):
fp = np.hstack((fp, fp_bkg))
fpix_col[fi] = fp
for fi, fe in enumerate(fpix_err):
for ad in np.arange(add_col):
fe = np.hstack(
(fe, np.zeros(shape=(fpix.shape[1], 1))))
fpix = np.copy(fpix_col)
fpix_err = np.copy(fpix_err_col)
fpix_bin = np.ndarray(shape=(fpix.shape[0],
fpix[i].shape[0] // bin_row,
fpix[i].shape[1] // bin_col))
for i, f in enumerate(fpix_bin):
fpix_bin[i] = fpix[i].reshape(fpix[i].shape[0] // bin_row,
bin_row,
fpix[i].shape[1] // bin_col,
bin_col).sum(3).sum(1)
fpix = np.copy(fpix_bin)
fpix_err_bin = np.ndarray(shape=(fpix_err.shape[0],
fpix_err[0].shape[0] // bin_row,
fpix_err[0].shape[1] // bin_col))
for i, f in enumerate(fpix_err_bin):
fpix_err_bin[i] = fpix_err[i].reshape(
fpix_err[i].shape[0] // bin_row, bin_row,
fpix_err[i].shape[1] // bin_col, bin_col).sum(3).sum(1)
fpix_err = np.copy(fpix_err_bin)
co += 1
apertures.update({'automatic': aperture})
pixel_images = [fpix[0], fpix[len(fpix) // 2], fpix[len(fpix) - 1]]
if np.sum(aperture) > max_pixels:
keys = list(apertures.keys())
npix = np.array([np.sum(apertures[k]) for k in keys])
aperture_name = keys[np.argmax(npix * (npix <= max_pixels))]
aperture = apertures[aperture_name]
aperture[np.isnan(fpix[0])] = 0
if np.sum(aperture) > max_pixels:
raise Exception(
'No apertures available with fewer than {} pixels. Aborting...'
.format(max_pixels))
aperture[np.isnan(fpix[0])] = 0
ap = np.where(aperture == 1)
fpix2D = np.array([f[ap] for f in fpix], dtype='float64')
fpix_err2D = np.array([p[ap] for p in fpix_err], dtype='float64')
binds = np.where(aperture != 1)
if len(binds[0]) > 0 and sector not in [9, 10, 11]:
bkg = np.nanmedian(np.array([f[binds] for f in fpix], dtype='float64'),
axis=1)
bkg_err = 1.253 * np.nanmedian(np.array([e[binds] for e in fpix_err],
dtype='float64'),
axis=1) / np.sqrt(len(binds[0]))
bkg = bkg.reshape(-1, 1)
bkg_err = bkg_err.reshape(-1, 1)
elif len(binds[0]) > 0 and sector in [9, 10, 11]:
bkg = np.zeros_like(fpix2D)
bkg_err = np.zeros_like(fpix2D)
else:
bkg = np.zeros_like(fpix2D)
bkg_err = np.zeros_like(fpix2D)
fpix = fpix2D - bkg
fpix_err = np.sqrt(fpix_err2D**2 + bkg_err**2)
flux = np.sum(fpix, axis=1)
ferr = np.sqrt(np.sum(fpix_err**2, axis=1))
nanmask = np.where(np.isnan(flux) | (flux == 0))[0]
badmask = []
for b in bad_bits:
badmask += list(np.where(qual == b)[0])
tmpmask = np.array(list(set(np.concatenate([badmask, nanmask]))))
t = np.delete(time, tmpmask)
f = np.delete(flux, tmpmask)
f = SavGol(f)
med = np.nanmedian(f)
MAD = 1.4826 * np.nanmedian(np.abs(f - med))
bad = np.where((f > med + 10. * MAD) | (f < med - 10. * MAD))[0]
outmask = [np.argmax(time == t[i]) for i in bad]
fpix = Interpolate(time, nanmask, fpix)
fpix_err = Interpolate(time, nanmask, fpix_err)
data = dict(ID=TIC,
sector=sector,
cadn=cadn,
time=time,
fpix=fpix,
fpix_err=fpix_err,
nanmask=np.sort(nanmask),
badmask=np.sort(badmask),
outmask=np.sort(outmask),
aperture=aperture,
apertures=apertures,
pixel_images=pixel_images,
bkg=bkg,
qual=qual,
pc1=pc1,
pc2=pc2,
fitsheader=fitsheader,
tessmag=fitsheader[0]['TESSMAG'][1],
radius=fitsheader[0]['RADIUS'][1],
ra=fitsheader[0]['RA_OBJ'][1],
dec=fitsheader[0]['DEC_OBJ'][1],
pmra=fitsheader[0]['PMRA'][1],
pmdec=fitsheader[0]['PMDEC'][1],
teff=fitsheader[0]['TEFF'][1],
pmtotal=fitsheader[0]['PMTOTAL'][1],
logg=fitsheader[0]['LOGG'][1],
feh=fitsheader[0]['MH'][1])
return data
