def simple_spectrum(obs):
"""Simple estimate of spectrum
Parameters
----------
obs : shadow.Observation
Input observation
Returns
-------
m_spat : np.ndarray
Array of the mean spectrum
g_spat : np.ndarray
Array of the gradient of the spectrum
"""
#Cut out
dat = np.copy(
(obs.sci /
obs.flat))[:, obs.spatial.reshape(-1)][:, :,
obs.spectral.reshape(-1)]
err = np.copy(
(obs.err /
obs.flat))[:, obs.spatial.reshape(-1)][:, :,
obs.spectral.reshape(-1)]
dat /= obs.vsr_mean
err /= obs.vsr_mean
m_spec = np.average(dat[~obs.in_transit][(~obs.in_transit).sum() // 2],
weights=1 /
err[~obs.in_transit][(~obs.in_transit).sum() // 2],
axis=0)
m_spec /= np.mean(m_spec)
x = np.linspace(0.5, -0.5, dat.shape[2])
x_p = np.linspace(-0.5, 0.5, dat.shape[2] * 10)
l = lk.LightCurve(x, m_spec)
l = l.normalize()
r = lk.RegressionCorrector(l)
dm = lk.designmatrix.create_spline_matrix(
x, knots=list(np.linspace(-0.5, 0.5, int(
dat.shape[2] / 1.5))[1:-1])).append_constant()
dm_p = lk.designmatrix.create_spline_matrix(
x_p, knots=list(np.linspace(-0.5, 0.5, int(
dat.shape[2] / 1.5))[1:-1])).append_constant()
_ = r.correct(dm, sigma=1e10)
model = dm_p.X.dot(r.coefficients)
g_spec = np.gradient(model, x_p)[::10]
m_spec = (np.atleast_3d(m_spec) *
np.ones(dat.shape).transpose([0, 2, 1])).transpose([0, 2, 1])
return m_spec, g_spec
def find_number_of_PCs(results, regressors, lc):
'''
Purpose:
Find the number of principal components to use in the PCA method
'''
# Parameters / Criteria
npc = 7
nbins = 40 # number of bins to divide the each PC
threshold_variance = 1e-4 # 5 # -> 500% # THIS IS NOT USED.
dm = lk.DesignMatrix(regressors,
name='regressors').pca(npc).append_constant()
rc = lk.RegressionCorrector(lc)
lc_regressed = rc.correct(dm)
# Find the median of the moving variance for each PCs
boxsize = np.floor(dm.values[:, 0].size / nbins).astype(int)
pcs_variances = []
for ipc in range(npc):
pcs_variances.append(
pd.Series(dm.values[:, ipc]).rolling(boxsize).var().median())
# pcs_variances = [ pd.Series(dm.values[:,ipc]).rolling(boxsize).var().median() for ipc in range(npc) ]
pcs_variances = np.array(pcs_variances)
relative_change_variances = np.abs(
np.diff(pcs_variances) / pcs_variances[:-1]) # < THIS IS NOT USED.
# Select PCs with variance above threshold value
ind_bad_pcs = np.argwhere(pcs_variances > threshold_variance)
if ind_bad_pcs.size > 0:
# Find index first PC that exceeds threshold value. This is the new npc
new_npc = ind_bad_pcs[0].item()
else:
print(
f'No PCs with variance>{threshold_variance}. All {npc} PCs used.')
new_npc = npc
# Store to results
results['pca_all'] = {'coef':rc.coefficients,\
'pc':[dm.values[:,i] for i in range(dm.rank)],\
'dm':dm,\
'rc':rc,\
'npc':npc,\
'npc_used':new_npc,\
'pc_variances':pcs_variances,\
'threshold_variance':threshold_variance,\
'nbins':nbins}
return new_npc, dm, rc
def build(self, object_info: MissionObjectInfo, sherlock_dir, caches_root_dir):
mission_id = object_info.mission_id()
sherlock_id = object_info.sherlock_id()
logging.info("Retrieving star catalog info...")
mission, mission_prefix, id = super().parse_object_id(mission_id)
if mission_prefix not in self.star_catalogs:
raise ValueError("Wrong object id " + mission_id)
cadence = object_info.cadence if object_info.cadence is not None else "short"
author = object_info.author if object_info.author is not None else self.authors[mission]
star_info = starinfo.StarInfo(sherlock_id, *self.star_catalogs[mission_prefix].catalog_info(id))
logging.info("Downloading lightcurve files...")
sectors = None if object_info.sectors == 'all' or mission != constants.MISSION_TESS else object_info.sectors
campaigns = None if object_info.sectors == 'all' or mission != constants.MISSION_K2 else object_info.sectors
quarters = None if object_info.sectors == 'all' or mission != constants.MISSION_KEPLER else object_info.sectors
tokens = sectors if sectors is not None else campaigns if campaigns is not None else quarters
tokens = tokens if tokens is not None else "all"
apertures = {}
tpf_search_results = lk.search_targetpixelfile(str(mission_id))
for tpf_search_result in tpf_search_results:
logging.info("There is data for Mission: %s, Year %.0f, Author: %s, ExpTime: %.0f",
tpf_search_result.mission[0], tpf_search_result.year[0], tpf_search_result.author[0],
tpf_search_result.exptime[0].value)
tpfs_dir = sherlock_dir + "/tpfs/"
if not os.path.exists(tpfs_dir):
os.mkdir(tpfs_dir)
if object_info.apertures is None:
lcf_search_results = lk.search_lightcurve(str(mission_id), mission=mission, cadence=cadence,
sector=sectors, quarter=quarters,
campaign=campaigns, author=author)
lcf = lcf_search_results.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR)
tpfs = lk.search_targetpixelfile(str(mission_id), mission=mission, cadence=cadence,
sector=sectors, quarter=quarters,
campaign=campaigns, author=author)\
.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR,
cutout_size=(CUTOUT_SIZE, CUTOUT_SIZE))
if lcf is None:
raise ObjectProcessingError("The target " + str(mission_id) + " is not available for the author " + author +
", cadence " + str(cadence) + "s and sectors " + str(tokens))
lc_data = self.extract_lc_data(lcf)
lc = None
matching_objects = []
for tpf in tpfs:
shutil.copy(tpf.path, tpfs_dir + os.path.basename(tpf.path))
if mission_prefix == self.MISSION_ID_KEPLER:
sector = tpf.quarter
elif mission_prefix == self.MISSION_ID_TESS:
sector = tpf.sector
if mission_prefix == self.MISSION_ID_KEPLER_2:
sector = tpf.campaign
apertures[sector] = ApertureExtractor.from_boolean_mask(tpf.pipeline_mask, tpf.column, tpf.row)
for i in range(0, len(lcf.data)):
if lcf.data[i].label == mission_id:
if lc is None:
lc = lcf.data[i].normalize()
else:
lc = lc.append(lcf.data[i].normalize())
else:
matching_objects.append(lcf.data[i].label)
matching_objects = set(matching_objects)
if len(matching_objects) > 0:
logging.warning("================================================")
logging.warning("TICS IN THE SAME PIXEL: " + str(matching_objects))
logging.warning("================================================")
if lc is None:
tokens = sectors if sectors is not None else campaigns if campaigns is not None else quarters
tokens = tokens if tokens is not None else "all"
raise ObjectProcessingError("The target " + str(mission_id) + " is not available for the author " + author +
", cadence " + str(cadence) + "s and sectors " + str(tokens))
lc = lc.remove_nans()
transits_min_count = self.__calculate_transits_min_count(len(lcf))
if mission_prefix == self.MISSION_ID_KEPLER:
sectors = [lcfile.quarter for lcfile in lcf]
elif mission_prefix == self.MISSION_ID_TESS:
sectors = [file.sector for file in lcf]
elif mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
sectors = [lcfile.campaign for lcfile in lcf]
lc = lc.to_corrector("sff").correct(windows=20)
source = "tpf"
else:
logging.info("Using user apertures!")
tpf_search_results = lk.search_targetpixelfile(str(mission_id), mission=mission, cadence=cadence,
sector=sectors, quarter=quarters, campaign=campaigns,
author=author)
tpfs = tpf_search_results.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR,
cutout_size=(CUTOUT_SIZE, CUTOUT_SIZE))
source = "tpf"
apertures = object_info.apertures
lc = None
for tpf in tpfs:
shutil.copy(tpf.path, tpfs_dir + os.path.basename(tpf.path))
if mission_prefix == self.MISSION_ID_KEPLER:
sector = tpf.quarter
elif mission_prefix == self.MISSION_ID_TESS:
sector = tpf.sector
elif mission_prefix == self.MISSION_ID_KEPLER_2:
sector = tpf.campaign
boolean_aperture = ApertureExtractor.from_pixels_to_boolean_mask(apertures[sector], tpf.column, tpf.row,
CUTOUT_SIZE, CUTOUT_SIZE)
tpf.plot(aperture_mask=boolean_aperture, mask_color='red')
plt.savefig(sherlock_dir + "/fov/Aperture_[" + str(sector) + "].png")
plt.close()
if mission_prefix == self.MISSION_ID_KEPLER:
corrector = lk.KeplerCBVCorrector(tpf)
corrector.plot_cbvs([1, 2, 3, 4, 5, 6, 7])
raw_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture).remove_nans()
plt.savefig(sherlock_dir + "/Corrector_components[" + str(sector) + "].png")
plt.close()
it_lc = corrector.correct([1, 2, 3, 4, 5])
ax = raw_lc.plot(color='C3', label='SAP Flux', linestyle='-')
it_lc.plot(ax=ax, color='C2', label='CBV Corrected SAP Flux', linestyle='-')
plt.savefig(sherlock_dir + "/Raw_vs_CBVcorrected_lc[" + str(sector) + "].png")
plt.close()
elif mission_prefix == self.MISSION_ID_KEPLER_2:
raw_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture).remove_nans()
it_lc = raw_lc.to_corrector("sff").correct(windows=20)
ax = raw_lc.plot(color='C3', label='SAP Flux', linestyle='-')
it_lc.plot(ax=ax, color='C2', label='CBV Corrected SAP Flux', linestyle='-')
plt.savefig(sherlock_dir + "/Raw_vs_SFFcorrected_lc[" + str(sector) + "].png")
plt.close()
elif mission_prefix == self.MISSION_ID_TESS:
temp_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture)
where_are_NaNs = np.isnan(temp_lc.flux)
temp_lc = temp_lc[np.where(~where_are_NaNs)]
regressors = tpf.flux[np.argwhere(~where_are_NaNs), ~boolean_aperture]
temp_token_lc = [temp_lc[i: i + 2000] for i in range(0, len(temp_lc), 2000)]
regressors_token = [regressors[i: i + 2000] for i in range(0, len(regressors), 2000)]
it_lc = None
raw_it_lc = None
item_index = 0
for temp_token_lc_item in temp_token_lc:
regressors_token_item = regressors_token[item_index]
design_matrix = lk.DesignMatrix(regressors_token_item, name='regressors').pca(5).append_constant()
corr_lc = lk.RegressionCorrector(temp_token_lc_item).correct(design_matrix)
if it_lc is None:
it_lc = corr_lc
raw_it_lc = temp_token_lc_item
else:
it_lc = it_lc.append(corr_lc)
raw_it_lc = raw_it_lc.append(temp_token_lc_item)
item_index = item_index + 1
ax = raw_it_lc.plot(label='Raw light curve')
it_lc.plot(ax=ax, label='Corrected light curve')
plt.savefig(sherlock_dir + "/Raw_vs_DMcorrected_lc[" + str(sector) + "].png")
plt.close()
if lc is None:
lc = it_lc.normalize()
else:
lc = lc.append(it_lc.normalize())
lc = lc.remove_nans()
lc.plot(label="Normalized light curve")
plt.savefig(sherlock_dir + "/Normalized_lc[" + str(sector) + "].png")
plt.close()
transits_min_count = self.__calculate_transits_min_count(len(tpfs))
if mission_prefix == self.MISSION_ID_KEPLER or mission_id == self.MISSION_ID_KEPLER_2:
sectors = [lcfile.quarter for lcfile in tpfs]
elif mission_prefix == self.MISSION_ID_TESS:
sectors = [file.sector for file in tpfs]
if mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
sectors = [lcfile.campaign for lcfile in tpfs]
sectors = None if sectors is None else np.unique(sectors)
lc_data = None
return LcBuild(lc, lc_data, star_info, transits_min_count, cadence, None, sectors, source, apertures)
def make_custom_lc(
self,
sector=None,
sap_mask=None,
aper_radius=None,
percentile=None,
threshold_sigma=None,
use_pld=True,
pixel_components=3,
spline_n_knots=100,
spline_degree=3,
background_mask=None,
pca_nterms=5,
with_offset=True,
):
"""
create a custom lightcurve with background subtraction, based on this tutorial:
https://docs.lightkurve.org/tutorials/04-how-to-remove-tess-scattered-light-using-regressioncorrector.html
Parameters
----------
sector : int or str
specific sector or all
aper_radius: int
aperture mask radius
percentile: float
aperture mask percentile
threshold_sigma: float
aperture mask threshold [sigma]
pca_nterms : int
number of pca terms to use
Returns
-------
corrected_lc : lightkurve object
"""
if self.verbose:
print("Using lightcurve with custom aperture.")
sector = sector if sector is not None else self.sector
sap_mask = sap_mask if sap_mask else self.sap_mask
aper_radius = aper_radius if aper_radius else self.aper_radius
percentile = percentile if percentile else self.percentile
threshold_sigma = (threshold_sigma
if threshold_sigma else self.threshold_sigma)
if self.tpf is None:
tpf, tpf_info = self.get_tpf(sector=sector, return_df=True)
else:
if self.tpf.sector == sector:
tpf = self.tpf
else:
tpf, tpf_info = self.get_tpf(sector=sector, return_df=True)
# Make an aperture mask and a raw light curve
self.aper_mask = parse_aperture_mask(
tpf,
sap_mask=sap_mask,
aper_radius=aper_radius,
percentile=percentile,
threshold_sigma=threshold_sigma,
verbose=False,
)
raw_lc = tpf.to_lightcurve(method="aperture",
aperture_mask=self.aper_mask)
# remove nans
idx = (np.isnan(raw_lc.time)
| np.isnan(raw_lc.flux)
| np.isnan(raw_lc.flux_err))
self.tpf = tpf[~idx]
self.raw_lc = raw_lc[~idx]
if use_pld:
if self.verbose:
print("Removing scattered light + applying PLD")
pld = lk.TessPLDCorrector(self.tpf, aperture_mask=self.aper_mask)
if background_mask is None:
background_mask = ~self.aper_mask
corrected_lc = pld.correct(
pixel_components=pixel_components,
spline_n_knots=spline_n_knots,
spline_degree=spline_degree,
background_mask=background_mask,
)
self.corrector = pld
else:
if self.verbose:
print("Removing scattered light")
# Make a design matrix and pass it to a linear regression corrector
regressors = tpf.flux[~idx][:, ~self.aper_mask]
dm = (lk.DesignMatrix(
regressors, name="pixels").pca(pca_nterms).append_constant())
# Regression Corrector Object
rc = lk.RegressionCorrector(self.raw_lc)
self.corrector = rc
corrected_lc = rc.correct(dm)
# Optional: Remove the scattered light, allowing for the large offset from scattered light
if with_offset:
corrected_lc = (self.raw_lc - rc.model_lc +
np.percentile(rc.model_lc.flux, q=5))
lc = corrected_lc.normalize()
self.lc_custom = lc
# compute Contamination
if self.gaia_sources is None:
gaia_sources = self.query_gaia_dr2_catalog(radius=120,
verbose=False)
else:
gaia_sources = self.gaia_sources
fluxes = get_fluxes_within_mask(self.tpf, self.aper_mask, gaia_sources)
self.contratio = sum(fluxes) - 1
if self.tic_params is None:
_ = self.query_tic_catalog(return_nearest_xmatch=True)
tic_contratio = self.tic_params.contratio
dcontratio = abs(tic_contratio - self.contratio)
if (tic_contratio is not None) & (dcontratio > 0.5):
print(f"contratio: {self.contratio:.2f} (TIC={tic_contratio:.2f})")
# add method
lc.detrend = lambda: detrend(lc)
return lc
def make_custom_lc(
self,
sector=None,
tpf_size=None,
sap_mask=None,
aper_radius=None,
percentile=None,
threshold_sigma=None,
use_pld=True,
pixel_components=3,
spline_n_knots=100,
spline_degree=3,
background_mask=None,
pca_nterms=5,
with_offset=True,
):
"""
create a custom lightcurve based on this tutorial:
https://docs.lightkurve.org/tutorials/04-how-to-remove-tess-scattered-light-using-regressioncorrector.html
Parameters
----------
sector : int or str
specific sector or all
cutout_size : tuple
tpf cutout size
aper_radius: int
aperture mask radius
percentile: float
aperture mask percentile
threshold_sigma: float
aperture mask threshold [sigma]
method : float
PLD (default)
Returns
-------
corrected_lc : lightkurve object
"""
if self.verbose:
print("Using lightcurve with custom aperture.")
sector = sector if sector is not None else self.sector
sap_mask = sap_mask if sap_mask else self.sap_mask
aper_radius = aper_radius if aper_radius else self.aper_radius
percentile = percentile if percentile else self.percentile
threshold_sigma = (threshold_sigma
if threshold_sigma else self.threshold_sigma)
cutout_size = tpf_size if tpf_size else self.cutout_size
tpf_tesscut = self.get_tpf_tesscut(sector=sector,
cutout_size=cutout_size)
self.aper_mask = parse_aperture_mask(
tpf_tesscut,
sap_mask=sap_mask,
aper_radius=aper_radius,
percentile=percentile,
threshold_sigma=threshold_sigma,
verbose=False,
)
raw_lc = tpf_tesscut.to_lightcurve(method="aperture",
aperture_mask=self.aper_mask)
# remove nans
idx = (np.isnan(raw_lc.time)
| np.isnan(raw_lc.flux)
| np.isnan(raw_lc.flux_err))
self.tpf_tesscut = tpf_tesscut[~idx]
self.lc_custom_raw = raw_lc[~idx]
if use_pld:
if self.verbose:
print("Removing scattered light + applying PLD")
pld = lk.TessPLDCorrector(self.tpf_tesscut,
aperture_mask=self.aper_mask)
if background_mask is None:
background_mask = ~self.aper_mask
corrected_lc = pld.correct(
pixel_components=pixel_components,
spline_n_knots=spline_n_knots,
spline_degree=spline_degree,
background_mask=background_mask,
)
self.corrector = pld
else:
if self.verbose:
print("Removing scattered light")
# Make a design matrix and pass it to a linear regression corrector
regressors = tpf_tesscut.flux[~idx][:, ~self.aper_mask]
dm = (lk.DesignMatrix(
regressors,
name="regressors").pca(nterms=pca_nterms).append_constant())
rc = lk.RegressionCorrector(raw_lc)
self.corrector = rc
corrected_lc = rc.correct(dm)
# Optional: Remove the scattered light, allowing for the large offset from scattered light
if with_offset:
corrected_lc = (raw_lc - rc.model_lc +
np.percentile(rc.model_lc.flux, q=5))
lc = corrected_lc.normalize()
self.lc_custom = lc
# compute Contamination
if self.gaia_sources is None:
gaia_sources = self.query_gaia_dr2_catalog(radius=120)
else:
gaia_sources = self.gaia_sources
fluxes = get_fluxes_within_mask(self.tpf_tesscut, self.aper_mask,
gaia_sources)
self.contratio = sum(fluxes) - 1
# add method
lc.detrend = lambda: detrend(lc)
return lc
def simple_vsr(obs, gradient=False):
"""Simple estimate of Variable Scan rate
Parameters
----------
obs : shadow.Observation
Input observation
Returns
-------
m_spat : np.ndarray
Array of the mean variable scan rate
g_spat : np.ndarray
Array of the gradient of the variable scan rate
"""
#Cut out
dat = np.copy(
(obs.sci /
obs.flat))[:, obs.spatial.reshape(-1)][:, :,
obs.spectral.reshape(-1)]
err = np.copy(
(obs.err /
obs.flat))[:, obs.spatial.reshape(-1)][:, :,
obs.spectral.reshape(-1)]
# Divide through by average spectrum
avg = np.atleast_3d(np.average(dat, weights=1 / err,
axis=1)).transpose([0, 2, 1])
dat /= avg
err /= avg
# ff = np.average(dat[~obs.in_transit], weights=(1/err)[~obs.in_transit], axis=0)
# ff = np.atleast_3d(ff).transpose([2, 0, 1]) * np.ones(obs.data.shape)
# ff[obs.cosmic_rays] = 1
# ff[obs.error/obs.data > 0.1] = 1
# dat /= ff
# err /= ff
x = np.linspace(0.5, -0.5, dat.shape[1])
x_p = np.linspace(-0.5, 0.5, dat.shape[1] * 10)
# Mean spatial model
m_spat = np.zeros((dat.shape[0], dat.shape[1]))
if gradient:
# Gradient of spatial model
g_spat = np.zeros((dat.shape[0], dat.shape[1]))
for idx in tqdm(range(dat.shape[0]), desc='Building Simple VSR'):
l = lk.LightCurve(x, np.average(dat[idx], weights=1 / err[idx],
axis=1)).normalize()
m_spat[idx, :] = l.flux
if gradient:
r = lk.RegressionCorrector(l)
dm = lk.designmatrix.create_sparse_spline_matrix(
x,
knots=list(
np.linspace(-0.5, 0.5, int(dat.shape[1] /
2))[1:-1])).append_constant()
dm_p = lk.designmatrix.create_sparse_spline_matrix(
x_p,
knots=list(
np.linspace(-0.5, 0.5, int(dat.shape[1] /
2))[1:-1])).append_constant()
_ = r.correct(dm, sigma=1e10)
model = dm_p.X.dot(r.coefficients)
g_spat[idx, :] = np.gradient(model, x_p)[::10]
m_spat = np.atleast_3d(m_spat) * np.ones(dat.shape)
#g_spat = np.atleast_3d(g_spat) * np.ones(dat.shape)
if gradient:
return m_spat, g_spat
return m_spat
def get_LCs(CLUSTERS,phot_type):
NUMBER_PCA_COMPONENTS = 5 #no of vectors for PCA
#initialize the masks
star_mask=np.empty([len(tpfs),cutout_size,cutout_size],dtype='bool')
if phot_type == 'pixel': # if pixel photometry is requested, define a different variable
radius_mask=np.empty([len(tpfs),cutout_size,cutout_size],dtype='bool')
sky_mask=np.empty([len(tpfs),cutout_size,cutout_size],dtype='bool')
frame = tpfs[0].shape[0]//2
# obtain the aperture and the background masks with help of cluster radius and percentile (85%)
if phot_type == 'pixel':
radius_mask[0],sky_mask[0] = circle_aperture(tpfs[0][frame].flux,tpfs[0][frame].flux, \
Complete_Clusters[Complete_Clusters['NAME'] == CLUSTERS[0]]['CORE_RADIUS'].values[0],85)
elif phot_type == 'ensemble':
star_mask[0],sky_mask[0] = circle_aperture(tpfs[0][frame].flux,tpfs[0][frame].flux, \
Complete_Clusters[Complete_Clusters['NAME'] == CLUSTERS[0]]['CORE_RADIUS'].values[0],85)
# if pixel photometry, iterate over each pixel in the radius mask and download LCs
if phot_type == 'pixel':
for i in range(len(np.where(radius_mask[0])[0])):
star_mask=np.zeros([len(tpfs),cutout_size,cutout_size],dtype='bool')
pixel_loc=np.where(radius_mask[0])[0][i],np.where(radius_mask[0])[1][i]
star_mask[0][pixel_loc]=True
lightcurves = [tpfs[i].to_lightcurve(aperture_mask=star_mask[i]) for i in range(len(tpfs))];
for i in range(len(lightcurves)):
lightcurves[i]=lightcurves[i][lightcurves[i].flux_err > 0]
#PCA correction
regressors = [tpfs[i].flux[:, sky_mask[i]] for i in range(len(tpfs))]# The regressor is the inverse of the aperture
#Design a matrix for linear regression
dms = [lk.DesignMatrix(r, name='regressors').pca(NUMBER_PCA_COMPONENTS).append_constant() for r in regressors]
#Remove noise using linear regression against a DesignMatrix.
correctors = [lk.RegressionCorrector(lc) for lc in lightcurves]
correctedLCs = [correctors[i].correct(dms[i]) for i in range(len(correctors))]#this is how to subtract the background noise from the star pixels
#polyomial correction. to remove long period systematic trends that survived the PCA correction
correctedLCs_poly,poly_terms = polynomial_correction(correctedLCs[0])
#save the pixel corrected LCs
correctedLCs_poly.to_csv(CLUSTERS[0]+' pixels/core_pixels/corrected_LCs_CORE_RADIUS_pixel_no.{0}'.format(pixel_loc))
# is ensemble photometry, download LC for entire aperture
elif phot_type == 'ensemble':
lightcurves = [tpfs[i].to_lightcurve(aperture_mask=star_mask[i]) for i in range(len(tpfs))];
for i in range(len(lightcurves)):
lightcurves[i]=lightcurves[i][lightcurves[i].flux_err > 0]
regressors = [tpfs[i].flux[:, sky_mask[i]] for i in range(len(tpfs))]# The regressor is the inverse of the aperture
#Design a matrix for linear regression
dms = [lk.DesignMatrix(r, name='regressors').pca(NUMBER_PCA_COMPONENTS).append_constant() for r in regressors]
#Remove noise using linear regression against a DesignMatrix.
correctors = [lk.RegressionCorrector(lc) for lc in lightcurves]
correctedLCs = [correctors[i].correct(dms[i]) for i in range(len(correctors))]
#polynomial correction
correctedLCs_poly,poly_terms = polynomial_correction(correctedLCs[0])
#calculate the lomb-scargle periodogram for the ensemble corrected LC for a range frequencies
t = correctedLCs_poly[correctedLCs_poly.quality == 0].time
dy = correctedLCs_poly[correctedLCs_poly.quality == 0].flux_err
y = correctedLCs_poly[correctedLCs_poly.quality == 0].flux
omega=np.arange(0.05,11,0.01)
P_LS = LombScargle(t, y, dy=dy).power(omega)# LS power
# max_power=np.max(P_LS)
# freq_at_max_power=omega[np.argmax(P_LS)]
return omega, P_LS #return frequencies and LS power
def extract_light_curve(fits_filename, outputdir, return_msg=True):
# Output name
output = Path(fits_filename.stem + '_corrected.pickled')
output = outputdir / output
# Parameters and Criteria:
sigma_clipping = 5 # To be applied after the detrending of the light curve
# Structure the data to be saved
results = {
'tic': None,
'sector': None,
'ra': None,
'dec': None,
'headers': None, # Headers from the original FITS file
'fit': None, # Result from fit
'neighbours_all': None, # All neighbours stars info
'neighbours_used': None, # Used neighbours stars info
'target': None, # Target star info
'aperture_threshold': None, # HDU to store tabular information
'pca_all': None, # PCA results
'pca_used': None, # PCA results
'centroids': None, # Centroids results
'excluded_intervals': None, # Excluded intervals in days
'lc_raw': None, # Light curves
'lc_raw_nonan': None, # Light curves
'lc_trend': None, # Light curves
'lc_regressed': None, # Light curves
'lc_regressed_notoutlier': None, # Light curves
'median_image': None,
'masks': None,
'tag': None
}
# Save information from header from original FITS file
HDUL, ext = [], 0
while True:
try:
HDUL.append(
fits.getheader(fits_filename.as_posix(), ext=ext).tostring())
ext += 1
except IndexError:
break
except Exception as e:
print('Unexpected exception when reading headers from FITS: ', e)
break
results['headers'] = HDUL
# Load the TESS taret pixel file
try:
tpf = lk.TessTargetPixelFile(fits_filename.as_posix())
except Exception as e:
# Save results
err_msg = f'"lightkurve.TessTargetPixelFile()" could not open file {fits_filename.as_posix()}. Exception: {e}'
results['tag'] = err_msg
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
tic = str(tpf.get_keyword('ticid'))
sector = str(tpf.get_keyword('sector'))
target_ra = np.float(tpf.ra)
target_dec = np.float(tpf.dec)
# Store to results
results['tic'], results['sector'] = tic, sector
results['ra'], results['dec'] = target_ra, target_dec
# Initialize messages
id_msg = f'TIC {tic} Sector {sector}: Skipped: '
OK_msg = f'TIC {tic} Sector {sector}: OK'
# Calculate the median image
with warnings.catch_warnings():
warnings.simplefilter('ignore')
median_image = np.nanmedian(tpf.flux, axis=0)
# Store to results
results['median_image'] = median_image
# Estimate of aperture mask and background mask
ap_mask_threshold, bkg_mask_threshold = 5, 3
ap_mask = threshold_mask(median_image,
threshold=ap_mask_threshold,
reference_pixel='center')
ap_bkg = ~threshold_mask(
median_image, threshold=bkg_mask_threshold, reference_pixel=None)
# Exclude NaN values outside the camera
ap_bkg &= ~np.isnan(median_image)
# Estimate the median flux background
median_bkg_flux = np.median(median_image[ap_bkg])
# Store to results
results['masks'] = {'aperture':ap_mask,\
'background':ap_bkg}
# Check validity of aperture mask
OK_ap_mask, err_msg = check_aperture_mask(ap_mask, id_msg)
# If aperture is not good, exit program with corresponding message
if not OK_ap_mask:
# Save results
results['tag'] = err_msg
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
# Refine aperture
try:
WCS = tpf.wcs
except IndexError:
# Save results
err_msg = id_msg + 'No WCS info in header'
results['tag'] = err_msg
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
ap_mask,\
target_coord_pixel, target_tmag,\
nb_coords_pixel, nb_tmags,\
err_msg = refine_aperture(results, tic, target_ra, target_dec, WCS,\
ap_mask, ap_mask_threshold, median_image, prepend_err_msg=id_msg)
# If not satisfactory aperture mask
if ap_mask is None:
# Save results
results['tag'] = err_msg
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
# Variation in time of aperture's center of mass
centroid_col, centroid_row = tpf.estimate_centroids(aperture_mask=ap_mask,
method='quadratic')
centroid_col -= tpf.column
centroid_row -= tpf.row
sqrt_col2_row2 = np.sqrt(centroid_col**2 + centroid_row**2)
# Store to results
results['centroids'] = {'col':centroid_col,\
'row':centroid_row,\
'sqrt_col2_row2':sqrt_col2_row2,\
'time':tpf.time}
# Fit the image and find the contamination fraction within the aperture mask
fitted_image, err_msg = contamination(results, median_image,ap_mask,\
target_coord_pixel, target_tmag,\
nb_coords_pixel, nb_tmags,\
tpf.wcs,median_bkg_flux,
prepend_err_msg=id_msg)
if fitted_image is None:
# Save results
results['tag'] = err_msg
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
# Generate the raw light curve
lc_raw = tpf.to_lightcurve(aperture_mask=ap_mask, method='aperture')
# Store to results
results['lc_raw'] = {'flux':lc_raw.flux,\
'time':lc_raw.time}
# Find the indices of the quality mask that created the light curve
ind = np.argwhere(tpf.quality_mask == True)
# Masks with True value the light curve times with null or NaN flux
mask = lc_raw.flux == 0
mask |= lc_raw.flux == np.nan
# Set to False the indices to be masked (ignored).
# Note that we take a subset from `ind` because the masks where defined from the light curve
tpf.quality_mask[ind[mask]] = False
# Exclude intervals previously decided
exclude_interval(tpf, sector, results)
# Generate the light curve
lc = tpf.to_lightcurve(aperture_mask=ap_mask, method='aperture')
# Store to results
results['lc_raw_nonan'] = {'flux':lc.flux,\
'time':lc.time}
# Make a design matrix and pass it to a linear regression corrector
regressors = tpf.flux[:, ap_bkg]
# Number of PCs to use
npc, dm, rc = find_number_of_PCs(results, regressors, lc)
if npc == 0:
# Save results
results['tag'] = id_msg + 'None PC used, no detrended done.'
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
try:
# Detrend light curve using PCA
dm = lk.DesignMatrix(regressors,
name='regressors').pca(npc).append_constant()
rc = lk.RegressionCorrector(lc)
lc_regressed = rc.correct(dm)
lc_trend = rc.diagnostic_lightcurves['regressors']
# Sigma clipping the remove outliers
lc_regressed_no_outliers, lc_mask_regressed_outliers = lc_regressed.remove_outliers(
return_mask=True, sigma=sigma_clipping)
# Store to results
results['lc_trend'] = {'flux':lc_trend.flux,\
'time':lc_trend.time}
results['lc_regressed'] = {'flux':lc_regressed.flux,\
'time':lc_regressed.time,\
'outlier_mask':lc_mask_regressed_outliers,\
'sigma_clipping':sigma_clipping}
results['lc_regressed_notoutlier'] = {'flux':lc_regressed_no_outliers.flux,\
'time':lc_regressed_no_outliers.time}
results['pca_used'] = {'coef':rc.coefficients,\
'pc':[dm.values[:,i] for i in range(dm.rank)],\
'dm':dm,\
'rc':rc,\
'npc':npc}
# Save results
results['tag'] = 'OK'
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return OK_msg
return
except Exception as e:
print('!!!!!!!!!!!!!!!!!!!!!!!!!')
print(f' Sector {sector}.')
print('EXCEPTION:', e)
print('!!!!!!!!!!!!!!!!!!!!!!!!!')
if return_msg: return id_msg + '::' + repr(e) + '::' + str(e)
return
lkf = lkf[bidx]
color_print('\nAperture chosen: ', 'lightcyan', str(bidx+1) + 'px radius' if args.circ==0 else 'No. ' + str(bidx), 'default',
'\tNumber of pixels inside: ', 'lightcyan', str(dap[bidx].sum()), 'default')
#PCA
if args.pca:
import lightkurve
regressors = flux[:, ~dap[bidx]]
dm = lightkurve.DesignMatrix(regressors, name='regressors')
dm = dm.pca(5)
dm = dm.append_constant()
corrector = lightkurve.RegressionCorrector(lkf)
corr_flux = corrector.correct(dm)
lkf = lkf - corrector.model_lc + np.percentile(corrector.model_lc.flux, 5)
#lkf.flux = lkf.flux - corrector.model_lc + np.percentile(corrector.model_lc.flux, 5)
#PLD?
elif args.pld:
'''
if iP is None:
mask = np.ones(time.size).astype(bool)
else:
mask = mask_planet(time, it0, iP, dur=idur)
flux_pld -= bkgs[:,None,None]
pldflsum = np.nansum(flux_pld, axis=0)
pldthr = mad_std(pldflsum)
def calculate_contamination(
tpfs,
period,
t0,
duration,
sigma=5,
plot=True,
cbvs=True,
sff=False,
windows=20,
bins=5,
spline_period=1,
**kwargs,
):
"""Calculate the contamination for a target
Parameters
----------
period : float
Period of transiting object in days
t0 : float
Transit midpoint of transiting object in days
duration : float
Duration of transit in days
sigma : float
The significance level at which to create an aperture for the contaminanting source.
If the apertures are large, try increasing sigma. If the apertures are small,
or contaminante fails, you could try (slightly) lowering sigma.
plot: bool
If True, will generate a figure
cbvs : bool
If True, will use Kepler/TESS CBVs to detrend. Default is True
sff : bool
If True, will use the SFF method to correct variability. Default is False.
spline_period : float
The period of a spline to fit. For short period variability,
set this value to a smaller number. Default is 0.75 days.
Returns
-------
result : list of dict
List of dictionaries containing the contamination properties
If plot is True, will show a figure, and will put the
matplotlib.pyplot.figure object into the result dictionary.
"""
if isinstance(tpfs, (list)):
tpfs = TPFC(tpfs)
elif isinstance(tpfs, TPF):
tpfs = TPFC([tpfs])
elif not isinstance(tpfs, TPFC):
raise ValueError(
"please pass `lk.TargetPixelFile` or `lk.TargetPixelFileCollection`"
)
# remove nans
for idx, tpf in enumerate(tpfs):
aper = tpf.pipeline_mask
if not (aper.any()):
aper = tpf.create_threshold_mask()
mask = (np.abs(
(tpf.pos_corr1)) < 5) & ((np.gradient(tpf.pos_corr2)) < 5)
mask &= np.nan_to_num(tpf.to_lightcurve(aperture_mask=aper).flux) != 0
if tpf.mission.lower() in ["k2", "ktwo"]:
# For k2 we have to get rid of some serious outliers...
r, c = tpf.estimate_centroids("all")
r, c = np.nan_to_num(r.value), np.nan_to_num(c.value)
arclength = np.hypot(r - np.min(r), c - np.min(c))
reg = lk.RegressionCorrector(
lk.LightCurve(time=tpf.time, flux=np.gradient(arclength)))
dm = lk.correctors.designmatrix.create_sparse_spline_matrix(
tpf.time.value,
n_knots=int((tpf.time[-1].value - tpf.time[0].value) // 1),
).append_constant()
_ = reg.correct(dm, sigma=3)
# If there aren't too many outliers
if (reg.outlier_mask.sum()) / len(reg.outlier_mask) < 0.2:
mask &= ~reg.outlier_mask
tpfs[idx] = tpfs[idx][mask]
results = []
for tpf in tqdm(tpfs, desc="Modeling TPFs"):
aper = tpf.pipeline_mask
if not (aper.any()):
aper = tpf.create_threshold_mask()
lc = tpf.to_lightcurve(aperture_mask=aper).normalize()
if sff:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
s = lk.correctors.SFFCorrector(lc)
_ = s.correct(windows=windows,
bins=bins,
timescale=spline_period)
sff_dm = s.dmc
bls = lc.to_periodogram("bls",
period=[period, period],
duration=duration)
t_mask = bls.get_transit_mask(period=period,
transit_time=t0,
duration=duration)
# Correct light curve
if cbvs:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
cbv_array = (lk.correctors.CBVCorrector(
tpf.to_lightcurve(aperture_mask=aper),
interpolate_cbvs=True,
extrapolate_cbvs=True,
).cbvs[0].to_designmatrix().X[:, :4])
else:
cbv_array = None
r1, c1 = tpf.estimate_centroids(aperture_mask=aper)
r1 -= np.median(r1)
c1 -= np.median(c1)
X = build_X(
lc.time.jd,
r1.value,
c1.value,
cbvs=cbv_array,
windows=windows,
bins=bins,
spline_period=spline_period,
sff=sff,
**kwargs,
)
X = X[:, np.asarray(X.sum(axis=0))[0] != 0]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
dm1 = lk.SparseDesignMatrix(
X,
name="X",
prior_mu=np.hstack([np.zeros(X.shape[1] - 1), 0]),
prior_sigma=np.hstack([np.ones(X.shape[1] - 1) * 1e2, 0.1]),
)
r = lk.RegressionCorrector(lc.copy())
target = r.correct(dm1, cadence_mask=~t_mask)
stellar_lc = r.diagnostic_lightcurves["X"].flux
if t_mask.sum() == 0:
results.append(
_package_results(
tpf,
target=target,
contaminator=target * 0,
aper=aper,
contaminant_aper=aper * False,
transit_pixels=np.zeros(aper.shape),
transit_pixels_err=np.inf * np.ones(aper.shape),
period=period,
t0=t0,
duration=duration,
plot=plot,
))
continue
# Find a transit model
bls = target.to_periodogram("bls",
period=[period, period],
duration=duration)
t_model = (~bls.get_transit_mask(period=period,
transit_time=t0,
duration=duration)).astype(float) - 1
depth = bls.compute_stats(period=period,
transit_time=t0,
duration=duration)["depth"][0]
X = build_X(
lc.time.jd,
r1.value,
c1.value,
flux=stellar_lc,
t_model=t_model,
cbvs=cbv_array,
windows=windows,
bins=bins,
spline_period=spline_period,
sff=sff,
**kwargs,
)
X = X[:, np.asarray(X.sum(axis=0))[0] != 0]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
dm = lk.SparseDesignMatrix(
X,
name="X",
prior_mu=np.hstack([np.zeros(X.shape[1] - 1), depth]),
prior_sigma=np.hstack([np.ones(X.shape[1] - 1) * 1e4, 0.1]),
)
dm_no_transit = lk.SparseDesignMatrix(
X[:, :-1],
name="X",
prior_mu=np.hstack([np.zeros(X.shape[1] - 1)]),
prior_sigma=np.hstack([np.ones(X.shape[1] - 1) * 1e4]),
)
model = np.zeros(tpf.flux.shape)
model_err = np.zeros(tpf.flux.shape)
# Hard coded saturation limit. Probably not ideal.
saturated = np.percentile(np.nan_to_num(tpf.flux.value), 95,
axis=0) > 1.5e5
if saturated.any():
dsat = np.gradient(saturated.astype(float), axis=0)
if (~np.any(dsat == 0.5)) | (~np.any(dsat == -0.5)):
raise ValueError(
"Too close to a saturation column that isn't fully captured."
)
saturated |= np.abs(dsat) != 0
pixels = tpf.flux.value.copy()
pixels_err = tpf.flux_err.value.copy()
transit_pixels = np.zeros(tpf.flux.shape[1:])
transit_pixels_err = np.zeros(tpf.flux.shape[1:]) * np.inf
chi_ratio = np.zeros(tpf.flux.shape[1:]) * np.inf
for jdx, s in enumerate(saturated.T):
if any(s):
l = np.where(s)[0][s.sum() // 2]
pixels[:, s, jdx] = np.nan
pixels[:, l, jdx] = tpf.flux.value[:, s, jdx].sum(axis=(1))
pixels_err[:, l, jdx] = ((tpf.flux_err.value[:, s, jdx]**
2).sum(axis=(1))**0.5) / s.sum()
for idx in range(tpf.shape[1]):
for jdx in range(tpf.shape[2]):
if np.nansum(pixels[:, idx, jdx]) == 0:
continue
# If we wanted a box smooth...
# lb1, lb2 = np.max([idx - 1, 0]), np.max([jdx - 1, 0])
# ub1, ub2 = np.min([idx + 2, pixels.shape[1]]), np.min(
# [jdx + 2, pixels.shape[2]]
# )
#
# r.lc.flux = np.nansum(
# pixels[:, lb1:ub1][:, :, lb2:ub2],
# axis=(1, 2),
# )
# r.lc.flux_err = (
# np.nansum((pixels_err[:, lb1:ub1][:, :, lb2:ub2] ** 2), axis=(1, 2))
# ** 0.5
# )
# r.lc.flux_err /= np.nanmedian(r.lc.flux)
# r.lc.flux /= np.nanmedian(r.lc.flux)
# clc = r.correct(dm)
r.lc.flux = pixels[:, idx, jdx] / np.median(pixels[:, idx,
jdx])
r.lc.flux_err = pixels_err[:, idx, jdx] / np.median(
pixels[:, idx, jdx])
if sff:
# Do a first multiplicative correction
_ = r.correct(sff_dm, cadence_mask=~t_mask)
mlc = r.model_lc + np.median(r.lc.flux)
mlc /= np.percentile(mlc.flux, 95)
r.lc /= mlc
r.lc = r.lc / np.median(r.lc.flux)
clc = r.correct(dm)
transit_pixels[idx, jdx] = r.coefficients[-1]
sigma_w_inv = X.T.dot(X / r.lc.flux_err[:, None]**2) + np.diag(
1 / dm.prior_sigma**2)
transit_pixels_err[idx, jdx] = (np.asarray(
np.linalg.inv(sigma_w_inv)).diagonal()[-1]**0.5)
for jdx, s in enumerate(saturated.T):
if any(s):
l = np.where(s)[0][s.sum() // 2]
transit_pixels[s, jdx] = transit_pixels[l, jdx]
transit_pixels_err[s, jdx] = transit_pixels_err[l, jdx]
with warnings.catch_warnings():
warnings.simplefilter("ignore")
contaminant_aper = create_threshold_mask(
transit_pixels / transit_pixels_err, sigma)
if tpf.mission.lower() in ["k2", "ktwo"]:
# Ktwo is noisy so we have to make the aperture a bit bigger.
contaminant_aper |= (np.asarray(
np.gradient(contaminant_aper.astype(float))) != 0).any(axis=0)
contaminated_lc = tpf.to_lightcurve(
aperture_mask=contaminant_aper).normalize()
r.lc = contaminated_lc
contaminator = r.correct(dm1, cadence_mask=~t_mask)
results.append(
_package_results(
tpf,
target=target,
contaminator=contaminator,
aper=aper,
contaminant_aper=contaminant_aper,
transit_pixels=transit_pixels,
transit_pixels_err=transit_pixels_err,
period=period,
t0=t0,
duration=duration,
plot=plot,
))
return results
def __init__(self,
targetpixelfile,
gaia=True,
magnitude_limit=18,
frequencies=[],
frequnit=u.uHz,
principle_components=5,
aperture=None,
**kwargs):
#Defining an aperture that will be used in plotting and making empty 2-d arrays of the correct size for masks
if targetpixelfile.pipeline_mask.any() == False:
self.aperture = aperture
else:
self.aperture = targetpixelfile.pipeline_mask
self.tpf = targetpixelfile
# Make a design matrix and pass it to a linear regression corrector
self.raw_lc = self.tpf.to_lightcurve(aperture_mask=self.aperture)
self.dm = lk.DesignMatrix(
self.tpf.flux[:, ~self.tpf.create_threshold_mask()],
name='regressors').pca(principle_components)
rc = lk.RegressionCorrector(self.raw_lc)
corrected_lc = rc.correct(self.dm.append_constant())
corrected_lc[np.where(corrected_lc.quality == 0)]
self.corrected_lc = corrected_lc.remove_outliers()
self.frequency_list = np.asarray((frequencies * frequnit).to(1 / u.d))
self.principle_components = principle_components
def Obtain_Initial_Phase(tpf, corrected_lc, frequency_list):
flux = corrected_lc.flux.value
times = corrected_lc.time.value - np.mean(corrected_lc.time.value)
pg = corrected_lc.to_periodogram(frequency=np.append(
[0.0001], frequency_list),
ls_method='slow')
initial_flux = np.asarray(pg.power[1:])
initial_phase = np.zeros(len(frequency_list))
def lc_model(time, amp, freq, phase):
return amp * np.sin(2 * np.pi * freq * time + phase)
def background_model(time, height):
return np.ones(len(time)) * height
for j in np.arange(len(frequency_list)):
for i in np.arange(len(frequency_list)):
if (i == 0):
model = lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model += lm.Model(background_model,
independent_vars=['time'])
else:
model += lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model.set_param_hint('f{0:d}phase'.format(i),
min=-np.pi,
max=np.pi,
value=initial_phase[i],
vary=False)
model.set_param_hint('f{0:d}amp'.format(i),
value=initial_flux[i],
vary=False)
model.set_param_hint('height',
value=np.mean(flux),
vary=False)
model.set_param_hint('f{0:d}freq'.format(i),
value=frequency_list[i],
vary=False)
params = model.make_params()
params['f{0:d}phase'.format(j)].set(vary=True)
params['f{0:d}phase'.format(j)].set(value=initial_phase[j])
params['f{0:d}phase'.format(j)].set(brute_step=np.pi / 10)
result = model.fit(corrected_lc.flux.value,
params,
time=times,
method='brute')
initial_phase[j] = result.best_values['f{0:d}phase'.format(j)]
return initial_phase
self.initial_phases = Obtain_Initial_Phase(self.tpf, self.corrected_lc,
self.frequency_list)
def Obtain_Final_Phase(tpf, corrected_lc, frequency_list,
initial_phases):
flux = corrected_lc.flux.value
times = corrected_lc.time.value - np.mean(corrected_lc.time.value)
pg = corrected_lc.to_periodogram(frequency=np.append(
[0.0001], frequency_list),
ls_method='slow')
initial_flux = np.asarray(pg.power[1:])
def lc_model(time, amp, freq, phase):
return amp * np.sin(2 * np.pi * freq * time + phase)
def background_model(time, height):
return np.ones(len(time)) * height
for i in np.arange(len(frequency_list)):
if (i == 0):
model = lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model += lm.Model(background_model,
independent_vars=['time'])
else:
model += lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model.set_param_hint('f{0:d}phase'.format(i),
min=-np.pi,
max=np.pi,
value=initial_phases[i],
vary=True)
model.set_param_hint('f{0:d}amp'.format(i),
value=initial_flux[i],
vary=True)
model.set_param_hint('height', value=np.mean(flux), vary=True)
model.set_param_hint('f{0:d}freq'.format(i),
value=frequency_list[i],
vary=False)
params = model.make_params()
result = model.fit(corrected_lc.flux.value, params, time=times)
final_phases = [
result.best_values['f{0:d}phase'.format(j)]
for j in np.arange(len(frequency_list))
]
return final_phases
self.final_phases = Obtain_Final_Phase(self.tpf, self.corrected_lc,
self.frequency_list,
self.initial_phases)
def Obtain_Final_Fit(tpf, corrected_lc, frequency_list, final_phases):
flux = corrected_lc.flux.value
times = corrected_lc.time.value - np.mean(corrected_lc.time.value)
pg = corrected_lc.to_periodogram(frequency=np.append(
[0.0001], frequency_list),
ls_method='slow')
initial_flux = np.asarray(pg.power[1:])
def lc_model(time, amp, freq, phase):
return amp * np.sin(2 * np.pi * freq * time + phase)
def background_model(time, height):
return np.ones(len(time)) * height
for i in np.arange(len(frequency_list)):
if (i == 0):
model = lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model += lm.Model(background_model,
independent_vars=['time'])
else:
model += lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model.set_param_hint('f{0:d}phase'.format(i),
value=final_phases[i],
vary=False)
model.set_param_hint('f{0:d}amp'.format(i),
value=initial_flux[i],
vary=True)
model.set_param_hint('height', value=np.mean(flux), vary=True)
model.set_param_hint('f{0:d}freq'.format(i),
value=frequency_list[i],
vary=False)
params = model.make_params()
result = model.fit(corrected_lc.flux.value, params, time=times)
return result
heats = []
heats_error = []
#Iterating through columns of pixels
for i in np.arange(0, len(self.aperture)):
#Iterating through rows of pixels
for j in np.arange(0, len(self.aperture[0])):
#Making an empty 2-d array
mask = np.zeros((len(self.aperture), len(self.aperture[0])),
dtype=bool)
#Iterating to isolate pixel by pixel to get light curves
mask[i][j] = True
#Getting the light curve for a pixel and excluding any flagged data
lightcurve = self.tpf.to_lightcurve(aperture_mask=mask)
rcc = lk.RegressionCorrector(lightcurve)
lc = rcc.correct(self.dm.append_constant())
#lc = lc[np.where(lc.quality == 0)]
#lc = lc.remove_outliers()
bestfit = Obtain_Final_Fit(self.tpf, lc, self.frequency_list,
self.final_phases)
heat = np.asarray([
bestfit.best_values['f{0:d}amp'.format(n)]
for n in np.arange(len(self.frequency_list))
])
#heat = bestfit.best_values['f0amp']# / bestfit.params['f0amp'].stderr
heat_error = np.asarray([
bestfit.params['f{0:d}amp'.format(n)].stderr
for n in np.arange(len(self.frequency_list))
])
#Extending the list of fitting data for each pixel
heats.extend([heat])
heats_error.extend([heat_error])
#Taking the final list and turning it into a 2-d numpy array with the same dimensions of the full postage stamp
#heats = np.reshape(np.asarray(heats),(len(self.aperture),len(self.aperture[0])))
#heats_error = np.reshape(np.asarray(heats_error),(len(self.aperture),len(self.aperture[0])))
heats = np.asarray(heats)
heats_error = np.asarray(heats_error)
#Defining self.periodogram as this 2-d array of periodogram data
self.heatmap = heats.T
self.heatmap_error = heats_error.T
self.timeserieslength = (self.tpf.time.max() -
self.tpf.time.min()).value
self.gaiadata = None
if (gaia == True):
"""Make the Gaia Figure Elements"""
# Get the positions of the Gaia sources
c1 = SkyCoord(self.tpf.ra, self.tpf.dec, frame='icrs', unit='deg')
# Use pixel scale for query size
pix_scale = 4.0 # arcseconds / pixel for Kepler, default
if self.tpf.mission == 'TESS':
pix_scale = 21.0
# We are querying with a diameter as the radius, overfilling by 2x.
from astroquery.vizier import Vizier
Vizier.ROW_LIMIT = -1
result = Vizier.query_region(
c1,
catalog=["I/345/gaia2"],
radius=Angle(np.max(self.tpf.shape[1:]) * pix_scale, "arcsec"))
no_targets_found_message = ValueError(
'Either no sources were found in the query region '
'or Vizier is unavailable')
too_few_found_message = ValueError(
'No sources found brighter than {:0.1f}'.format(
magnitude_limit))
if result is None:
raise no_targets_found_message
elif len(result) == 0:
raise too_few_found_message
result = result["I/345/gaia2"].to_pandas()
result = result[result.Gmag < magnitude_limit]
if len(result) == 0:
raise no_targets_found_message
year = ((self.tpf.time[0].jd - 2457206.375) * u.day).to(u.year)
pmra = ((np.nan_to_num(np.asarray(result.pmRA)) *
u.milliarcsecond / u.year) * year).to(u.deg).value
pmdec = ((np.nan_to_num(np.asarray(result.pmDE)) *
u.milliarcsecond / u.year) * year).to(u.deg).value
result.RA_ICRS += pmra
result.DE_ICRS += pmdec
radecs = np.vstack([result['RA_ICRS'], result['DE_ICRS']]).T
coords = self.tpf.wcs.all_world2pix(radecs, 0)
# Gently size the points by their Gaia magnitude
sizes = 64.0 / 2**(result['Gmag'] / 5.0)
one_over_parallax = 1.0 / (result['Plx'] / 1000.)
source = dict(ra=result['RA_ICRS'],
dec=result['DE_ICRS'],
source=result['Source'].astype(str),
Gmag=result['Gmag'],
plx=result['Plx'],
one_over_plx=one_over_parallax,
x=coords[:, 0],
y=coords[:, 1],
size=sizes)
self.gaiadata = source
class frequency_heatmap:
def __init__(self, tpf, heats, heats_error, frequencies,
gaia_data):
self.heat_stamp = heats
self.gaiadata = gaia_data
self.heatmap_error = heats_error
self.size = tpf.pipeline_mask.shape
self.frequencies = frequencies
def centroid(self):
#Residuals to minimize relative to the error bars
def residual(params, amp, amperr):
x = params['x']
y = params['y']
sigma = params['sigma']
xpix, ypix = np.meshgrid(np.arange(self.size[0]),
np.arange(self.size[1]))
res = []
for i in np.arange(len(self.frequencies)):
height = params['height{0:d}'.format(i)]
model = height * np.exp(-(((x - xpix) / sigma)**2 +
((y - ypix) / sigma)**2) / 2)
res.extend([(amp[i].reshape(self.size) - model) /
amperr[i].reshape(self.size)])
return np.asarray(res)
#Set starting values to converge from
self.heatmap_error[np.where(
self.heatmap_error == None)] = np.nan
composite_heatmap = self.heat_stamp.sum(axis=0).reshape(
self.size) / ((np.nansum(
self.heatmap_error**2, axis=0))**(1 / 2)).reshape(
self.size) #issue with numpy using sqrt?
c = np.where(composite_heatmap == composite_heatmap.max())
params = Parameters()
for i in np.arange(len(frequencies)):
params.add('height{0:d}'.format(i),
value=np.max(self.heat_stamp[i]))
params.add('x', value=c[1][0])
params.add('y', value=c[0][0])
params.add('sigma', value=1)
#Do the fit
minner = Minimizer(residual,
params,
fcn_args=(self.heat_stamp,
self.heatmap_error))
self.result = minner.minimize()
fit = self.result.params.valuesdict()
self.x = fit['x']
self.y = fit['y']
def star_list(self):
gaia_data = self.gaiadata
no_gaia_data_message = ValueError(
'No gaia data initialized in PixelMapPeriodogram class')
if gaia_data == None:
raise no_gaia_data_message
else:
distances = np.square(self.x - gaia_data['x']) + np.square(
self.y - gaia_data['y'])
closest_star_mask = np.where(
np.square(self.x - gaia_data['x']) +
np.square(self.y - gaia_data['y']) == (
np.square(self.x - gaia_data['x']) +
np.square(self.y - gaia_data['y'])).min())
stars = dict(
ra=np.asarray(gaia_data['ra']),
dec=np.asarray(gaia_data['dec']),
source=np.asarray(gaia_data['source']),
x=np.asarray(gaia_data['x']),
y=np.asarray(gaia_data['y']),
distance=distances,
probability=2 * stats.norm.sf(
distances,
scale=np.sqrt(self.result.params['x'].stderr**2 +
self.result.params['y'].stderr**2))
) #I believe mutiply by 2 since we wont have a negative distance
starlist = pd.DataFrame.from_dict(stars)
self.stars = starlist.sort_values(by=[r'distance'])
fh = frequency_heatmap(self.tpf, self.heatmap, self.heatmap_error,
self.frequency_list, self.gaiadata)
fh.centroid()
fh.star_list()
self.centroid = [fh.x, fh.y]
self.heatmap = self.heatmap.sum(axis=0).reshape(
self.aperture.shape[0], self.aperture.shape[1]) / np.sqrt(
(self.heatmap_error**2).sum(axis=0)).reshape(
self.aperture.shape[0], self.aperture.shape[1])
self.starfit = fh.stars.reset_index()
self.result = fh.result
def build(self, object_info, sherlock_dir):
mission_id = object_info.mission_id()
sherlock_id = object_info.sherlock_id()
quarters = None
sectors = None
logging.info("Retrieving star catalog info...")
mission, mission_prefix, id = super().parse_object_id(mission_id)
if mission_prefix not in self.star_catalogs:
raise ValueError("Wrong object id " + mission_id)
star_info = starinfo.StarInfo(
sherlock_id, *self.star_catalogs[mission_prefix].catalog_info(id))
logging.info("Downloading lightcurve files...")
sectors = None if object_info.sectors == 'all' or mission != "TESS" else object_info.sectors
quarters = None if object_info.sectors == 'all' or mission != "K2" else object_info.sectors
campaigns = None if object_info.sectors == 'all' or mission != "Kepler" else object_info.sectors
if object_info.aperture_file is None:
lcf = lk.search_lightcurve(str(mission_id), mission=mission, cadence="short",
sector=sectors, quarter=quarters,
campaign=campaigns, author=self.authors[mission])\
.download_all()
if lcf is None:
raise ObjectProcessingError(
"Light curve not found for object id " + mission_id)
lc = None
matching_objects = []
for i in range(0, len(lcf.PDCSAP_FLUX)):
if lcf.PDCSAP_FLUX[i].label == mission_id:
if lc is None:
lc = lcf.PDCSAP_FLUX[i].normalize()
else:
lc = lc.append(lcf.PDCSAP_FLUX[i].normalize())
else:
matching_objects.append(lcf.PDCSAP_FLUX[i].label)
if len(matching_objects) > 0:
logging.warning(
"================================================")
logging.warning("TICS IN THE SAME PIXEL: " +
str(matching_objects))
logging.warning(
"================================================")
lc = lc.remove_nans()
transits_min_count = self.__calculate_transits_min_count(len(lcf))
if mission_prefix == self.MISSION_ID_KEPLER or mission_id == self.MISSION_ID_KEPLER_2:
quarters = [lcfile.quarter for lcfile in lcf]
elif mission_prefix == self.MISSION_ID_TESS:
sectors = [file.sector for file in lcf]
if mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
quarters = [lcfile.campaign for lcfile in lcf]
lc = lc.to_corrector("sff").correct(windows=20)
return lc, star_info, transits_min_count, np.unique(
sectors), np.unique(quarters)
else:
logging.info("Using user apertures!")
tpfs = lk.search_targetpixelfile(
str(mission_id),
mission=mission,
cadence="short",
sector=sectors,
quarter=quarters,
campaign=campaigns,
author=authors[mission]).download_all()
apertures = {}
if isinstance(object_info.aperture_file, str):
aperture = []
with open(object_info.aperture_file, 'r') as fd:
reader = csv.reader(fd)
for row in reader:
aperture.append(row)
aperture = np.array(aperture)
for tpf in tpfs:
if mission_prefix == self.MISSION_ID_KEPLER:
apertures[tpf.quarter] = aperture
elif mission_prefix == self.MISSION_ID_TESS:
apertures[tpf.sector] = aperture
elif mission_prefix == self.MISSION_ID_KEPLER_2:
apertures[tpf.campaign] = aperture
else:
for sector, aperture_file in object_info.aperture_file.items():
aperture = []
with open(aperture_file, 'r') as fd:
reader = csv.reader(fd)
for row in reader:
aperture.append(row)
aperture = np.array(aperture)
apertures[sector] = aperture
lc = None
for tpf in tpfs:
if mission_prefix == self.MISSION_ID_KEPLER:
sector = tpf.quarter
elif mission_prefix == self.MISSION_ID_TESS:
sector = tpf.sector
elif mission_prefix == self.MISSION_ID_KEPLER_2:
sector = tpf.campaign
aperture = apertures[sector].astype(bool)
tpf.plot(aperture_mask=aperture, mask_color='red')
plt.savefig(sherlock_dir + "/Aperture_[" + str(sector) +
"].png")
plt.close()
if mission_prefix == self.MISSION_ID_KEPLER:
corrector = lk.KeplerCBVCorrector(tpf)
corrector.plot_cbvs([1, 2, 3, 4, 5, 6, 7])
raw_lc = tpf.to_lightcurve(
aperture_mask=aperture).remove_nans()
plt.savefig(sherlock_dir + "/Corrector_components[" +
str(sector) + "].png")
plt.close()
it_lc = corrector.correct([1, 2, 3, 4, 5])
ax = raw_lc.plot(color='C3',
label='SAP Flux',
linestyle='-')
it_lc.plot(ax=ax,
color='C2',
label='CBV Corrected SAP Flux',
linestyle='-')
plt.savefig(sherlock_dir + "/Raw_vs_CBVcorrected_lc[" +
str(sector) + "].png")
plt.close()
elif mission_prefix == self.MISSION_ID_KEPLER_2:
raw_lc = tpf.to_lightcurve(
aperture_mask=aperture).remove_nans()
it_lc = raw_lc.to_corrector("sff").correct(windows=20)
ax = raw_lc.plot(color='C3',
label='SAP Flux',
linestyle='-')
it_lc.plot(ax=ax,
color='C2',
label='CBV Corrected SAP Flux',
linestyle='-')
plt.savefig(sherlock_dir + "/Raw_vs_SFFcorrected_lc[" +
str(sector) + "].png")
plt.close()
elif mission_prefix == self.MISSION_ID_TESS:
temp_lc = tpf.to_lightcurve(aperture_mask=aperture)
where_are_NaNs = np.isnan(temp_lc.flux)
temp_lc = temp_lc[np.where(~where_are_NaNs)]
regressors = tpf.flux[np.argwhere(~where_are_NaNs),
~aperture]
temp_token_lc = [
temp_lc[i:i + 2000]
for i in range(0, len(temp_lc), 2000)
]
regressors_token = [
regressors[i:i + 2000]
for i in range(0, len(regressors), 2000)
]
it_lc = None
raw_it_lc = None
item_index = 0
for temp_token_lc_item in temp_token_lc:
regressors_token_item = regressors_token[item_index]
design_matrix = lk.DesignMatrix(
regressors_token_item,
name='regressors').pca(5).append_constant()
corr_lc = lk.RegressionCorrector(
temp_token_lc_item).correct(design_matrix)
if it_lc is None:
it_lc = corr_lc
raw_it_lc = temp_token_lc_item
else:
it_lc = it_lc.append(corr_lc)
raw_it_lc = raw_it_lc.append(temp_token_lc_item)
item_index = item_index + 1
ax = raw_it_lc.plot(label='Raw light curve')
it_lc.plot(ax=ax, label='Corrected light curve')
plt.savefig(sherlock_dir + "/Raw_vs_DMcorrected_lc[" +
str(sector) + "].png")
plt.close()
if lc is None:
lc = it_lc.normalize()
else:
lc = lc.append(it_lc.normalize())
lc = lc.remove_nans()
lc.plot(label="Normalized light curve")
plt.savefig(sherlock_dir + "/Normalized_lc[" + str(sector) +
"].png")
plt.close()
transits_min_count = self.__calculate_transits_min_count(len(tpfs))
if mission_prefix == self.MISSION_ID_KEPLER or mission_id == self.MISSION_ID_KEPLER_2:
quarters = [lcfile.quarter for lcfile in tpfs]
elif mission_prefix == self.MISSION_ID_TESS:
sectors = [file.sector for file in tpfs]
if mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
quarters = [lcfile.campaign for lcfile in tpfs]
return lc, star_info, transits_min_count, np.unique(
sectors), np.unique(quarters)
