def find_number_of_PCs(results, regressors, lc):
'''
Purpose:
Find the number of principal components to use in the PCA method
'''
# Parameters / Criteria
npc = 7
nbins = 40 # number of bins to divide the each PC
threshold_variance = 1e-4 # 5 # -> 500% # THIS IS NOT USED.
dm = lk.DesignMatrix(regressors,
name='regressors').pca(npc).append_constant()
rc = lk.RegressionCorrector(lc)
lc_regressed = rc.correct(dm)
# Find the median of the moving variance for each PCs
boxsize = np.floor(dm.values[:, 0].size / nbins).astype(int)
pcs_variances = []
for ipc in range(npc):
pcs_variances.append(
pd.Series(dm.values[:, ipc]).rolling(boxsize).var().median())
# pcs_variances = [ pd.Series(dm.values[:,ipc]).rolling(boxsize).var().median() for ipc in range(npc) ]
pcs_variances = np.array(pcs_variances)
relative_change_variances = np.abs(
np.diff(pcs_variances) / pcs_variances[:-1]) # < THIS IS NOT USED.
# Select PCs with variance above threshold value
ind_bad_pcs = np.argwhere(pcs_variances > threshold_variance)
if ind_bad_pcs.size > 0:
# Find index first PC that exceeds threshold value. This is the new npc
new_npc = ind_bad_pcs[0].item()
else:
print(
f'No PCs with variance>{threshold_variance}. All {npc} PCs used.')
new_npc = npc
# Store to results
results['pca_all'] = {'coef':rc.coefficients,\
'pc':[dm.values[:,i] for i in range(dm.rank)],\
'dm':dm,\
'rc':rc,\
'npc':npc,\
'npc_used':new_npc,\
'pc_variances':pcs_variances,\
'threshold_variance':threshold_variance,\
'nbins':nbins}
return new_npc, dm, rc
def build(self, object_info: MissionObjectInfo, sherlock_dir, caches_root_dir):
mission_id = object_info.mission_id()
sherlock_id = object_info.sherlock_id()
logging.info("Retrieving star catalog info...")
mission, mission_prefix, id = super().parse_object_id(mission_id)
if mission_prefix not in self.star_catalogs:
raise ValueError("Wrong object id " + mission_id)
cadence = object_info.cadence if object_info.cadence is not None else "short"
author = object_info.author if object_info.author is not None else self.authors[mission]
star_info = starinfo.StarInfo(sherlock_id, *self.star_catalogs[mission_prefix].catalog_info(id))
logging.info("Downloading lightcurve files...")
sectors = None if object_info.sectors == 'all' or mission != constants.MISSION_TESS else object_info.sectors
campaigns = None if object_info.sectors == 'all' or mission != constants.MISSION_K2 else object_info.sectors
quarters = None if object_info.sectors == 'all' or mission != constants.MISSION_KEPLER else object_info.sectors
tokens = sectors if sectors is not None else campaigns if campaigns is not None else quarters
tokens = tokens if tokens is not None else "all"
apertures = {}
tpf_search_results = lk.search_targetpixelfile(str(mission_id))
for tpf_search_result in tpf_search_results:
logging.info("There is data for Mission: %s, Year %.0f, Author: %s, ExpTime: %.0f",
tpf_search_result.mission[0], tpf_search_result.year[0], tpf_search_result.author[0],
tpf_search_result.exptime[0].value)
tpfs_dir = sherlock_dir + "/tpfs/"
if not os.path.exists(tpfs_dir):
os.mkdir(tpfs_dir)
if object_info.apertures is None:
lcf_search_results = lk.search_lightcurve(str(mission_id), mission=mission, cadence=cadence,
sector=sectors, quarter=quarters,
campaign=campaigns, author=author)
lcf = lcf_search_results.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR)
tpfs = lk.search_targetpixelfile(str(mission_id), mission=mission, cadence=cadence,
sector=sectors, quarter=quarters,
campaign=campaigns, author=author)\
.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR,
cutout_size=(CUTOUT_SIZE, CUTOUT_SIZE))
if lcf is None:
raise ObjectProcessingError("The target " + str(mission_id) + " is not available for the author " + author +
", cadence " + str(cadence) + "s and sectors " + str(tokens))
lc_data = self.extract_lc_data(lcf)
lc = None
matching_objects = []
for tpf in tpfs:
shutil.copy(tpf.path, tpfs_dir + os.path.basename(tpf.path))
if mission_prefix == self.MISSION_ID_KEPLER:
sector = tpf.quarter
elif mission_prefix == self.MISSION_ID_TESS:
sector = tpf.sector
if mission_prefix == self.MISSION_ID_KEPLER_2:
sector = tpf.campaign
apertures[sector] = ApertureExtractor.from_boolean_mask(tpf.pipeline_mask, tpf.column, tpf.row)
for i in range(0, len(lcf.data)):
if lcf.data[i].label == mission_id:
if lc is None:
lc = lcf.data[i].normalize()
else:
lc = lc.append(lcf.data[i].normalize())
else:
matching_objects.append(lcf.data[i].label)
matching_objects = set(matching_objects)
if len(matching_objects) > 0:
logging.warning("================================================")
logging.warning("TICS IN THE SAME PIXEL: " + str(matching_objects))
logging.warning("================================================")
if lc is None:
tokens = sectors if sectors is not None else campaigns if campaigns is not None else quarters
tokens = tokens if tokens is not None else "all"
raise ObjectProcessingError("The target " + str(mission_id) + " is not available for the author " + author +
", cadence " + str(cadence) + "s and sectors " + str(tokens))
lc = lc.remove_nans()
transits_min_count = self.__calculate_transits_min_count(len(lcf))
if mission_prefix == self.MISSION_ID_KEPLER:
sectors = [lcfile.quarter for lcfile in lcf]
elif mission_prefix == self.MISSION_ID_TESS:
sectors = [file.sector for file in lcf]
elif mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
sectors = [lcfile.campaign for lcfile in lcf]
lc = lc.to_corrector("sff").correct(windows=20)
source = "tpf"
else:
logging.info("Using user apertures!")
tpf_search_results = lk.search_targetpixelfile(str(mission_id), mission=mission, cadence=cadence,
sector=sectors, quarter=quarters, campaign=campaigns,
author=author)
tpfs = tpf_search_results.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR,
cutout_size=(CUTOUT_SIZE, CUTOUT_SIZE))
source = "tpf"
apertures = object_info.apertures
lc = None
for tpf in tpfs:
shutil.copy(tpf.path, tpfs_dir + os.path.basename(tpf.path))
if mission_prefix == self.MISSION_ID_KEPLER:
sector = tpf.quarter
elif mission_prefix == self.MISSION_ID_TESS:
sector = tpf.sector
elif mission_prefix == self.MISSION_ID_KEPLER_2:
sector = tpf.campaign
boolean_aperture = ApertureExtractor.from_pixels_to_boolean_mask(apertures[sector], tpf.column, tpf.row,
CUTOUT_SIZE, CUTOUT_SIZE)
tpf.plot(aperture_mask=boolean_aperture, mask_color='red')
plt.savefig(sherlock_dir + "/fov/Aperture_[" + str(sector) + "].png")
plt.close()
if mission_prefix == self.MISSION_ID_KEPLER:
corrector = lk.KeplerCBVCorrector(tpf)
corrector.plot_cbvs([1, 2, 3, 4, 5, 6, 7])
raw_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture).remove_nans()
plt.savefig(sherlock_dir + "/Corrector_components[" + str(sector) + "].png")
plt.close()
it_lc = corrector.correct([1, 2, 3, 4, 5])
ax = raw_lc.plot(color='C3', label='SAP Flux', linestyle='-')
it_lc.plot(ax=ax, color='C2', label='CBV Corrected SAP Flux', linestyle='-')
plt.savefig(sherlock_dir + "/Raw_vs_CBVcorrected_lc[" + str(sector) + "].png")
plt.close()
elif mission_prefix == self.MISSION_ID_KEPLER_2:
raw_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture).remove_nans()
it_lc = raw_lc.to_corrector("sff").correct(windows=20)
ax = raw_lc.plot(color='C3', label='SAP Flux', linestyle='-')
it_lc.plot(ax=ax, color='C2', label='CBV Corrected SAP Flux', linestyle='-')
plt.savefig(sherlock_dir + "/Raw_vs_SFFcorrected_lc[" + str(sector) + "].png")
plt.close()
elif mission_prefix == self.MISSION_ID_TESS:
temp_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture)
where_are_NaNs = np.isnan(temp_lc.flux)
temp_lc = temp_lc[np.where(~where_are_NaNs)]
regressors = tpf.flux[np.argwhere(~where_are_NaNs), ~boolean_aperture]
temp_token_lc = [temp_lc[i: i + 2000] for i in range(0, len(temp_lc), 2000)]
regressors_token = [regressors[i: i + 2000] for i in range(0, len(regressors), 2000)]
it_lc = None
raw_it_lc = None
item_index = 0
for temp_token_lc_item in temp_token_lc:
regressors_token_item = regressors_token[item_index]
design_matrix = lk.DesignMatrix(regressors_token_item, name='regressors').pca(5).append_constant()
corr_lc = lk.RegressionCorrector(temp_token_lc_item).correct(design_matrix)
if it_lc is None:
it_lc = corr_lc
raw_it_lc = temp_token_lc_item
else:
it_lc = it_lc.append(corr_lc)
raw_it_lc = raw_it_lc.append(temp_token_lc_item)
item_index = item_index + 1
ax = raw_it_lc.plot(label='Raw light curve')
it_lc.plot(ax=ax, label='Corrected light curve')
plt.savefig(sherlock_dir + "/Raw_vs_DMcorrected_lc[" + str(sector) + "].png")
plt.close()
if lc is None:
lc = it_lc.normalize()
else:
lc = lc.append(it_lc.normalize())
lc = lc.remove_nans()
lc.plot(label="Normalized light curve")
plt.savefig(sherlock_dir + "/Normalized_lc[" + str(sector) + "].png")
plt.close()
transits_min_count = self.__calculate_transits_min_count(len(tpfs))
if mission_prefix == self.MISSION_ID_KEPLER or mission_id == self.MISSION_ID_KEPLER_2:
sectors = [lcfile.quarter for lcfile in tpfs]
elif mission_prefix == self.MISSION_ID_TESS:
sectors = [file.sector for file in tpfs]
if mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
sectors = [lcfile.campaign for lcfile in tpfs]
sectors = None if sectors is None else np.unique(sectors)
lc_data = None
return LcBuild(lc, lc_data, star_info, transits_min_count, cadence, None, sectors, source, apertures)
def make_custom_lc(
self,
sector=None,
sap_mask=None,
aper_radius=None,
percentile=None,
threshold_sigma=None,
use_pld=True,
pixel_components=3,
spline_n_knots=100,
spline_degree=3,
background_mask=None,
pca_nterms=5,
with_offset=True,
):
"""
create a custom lightcurve with background subtraction, based on this tutorial:
https://docs.lightkurve.org/tutorials/04-how-to-remove-tess-scattered-light-using-regressioncorrector.html
Parameters
----------
sector : int or str
specific sector or all
aper_radius: int
aperture mask radius
percentile: float
aperture mask percentile
threshold_sigma: float
aperture mask threshold [sigma]
pca_nterms : int
number of pca terms to use
Returns
-------
corrected_lc : lightkurve object
"""
if self.verbose:
print("Using lightcurve with custom aperture.")
sector = sector if sector is not None else self.sector
sap_mask = sap_mask if sap_mask else self.sap_mask
aper_radius = aper_radius if aper_radius else self.aper_radius
percentile = percentile if percentile else self.percentile
threshold_sigma = (threshold_sigma
if threshold_sigma else self.threshold_sigma)
if self.tpf is None:
tpf, tpf_info = self.get_tpf(sector=sector, return_df=True)
else:
if self.tpf.sector == sector:
tpf = self.tpf
else:
tpf, tpf_info = self.get_tpf(sector=sector, return_df=True)
# Make an aperture mask and a raw light curve
self.aper_mask = parse_aperture_mask(
tpf,
sap_mask=sap_mask,
aper_radius=aper_radius,
percentile=percentile,
threshold_sigma=threshold_sigma,
verbose=False,
)
raw_lc = tpf.to_lightcurve(method="aperture",
aperture_mask=self.aper_mask)
# remove nans
idx = (np.isnan(raw_lc.time)
| np.isnan(raw_lc.flux)
| np.isnan(raw_lc.flux_err))
self.tpf = tpf[~idx]
self.raw_lc = raw_lc[~idx]
if use_pld:
if self.verbose:
print("Removing scattered light + applying PLD")
pld = lk.TessPLDCorrector(self.tpf, aperture_mask=self.aper_mask)
if background_mask is None:
background_mask = ~self.aper_mask
corrected_lc = pld.correct(
pixel_components=pixel_components,
spline_n_knots=spline_n_knots,
spline_degree=spline_degree,
background_mask=background_mask,
)
self.corrector = pld
else:
if self.verbose:
print("Removing scattered light")
# Make a design matrix and pass it to a linear regression corrector
regressors = tpf.flux[~idx][:, ~self.aper_mask]
dm = (lk.DesignMatrix(
regressors, name="pixels").pca(pca_nterms).append_constant())
# Regression Corrector Object
rc = lk.RegressionCorrector(self.raw_lc)
self.corrector = rc
corrected_lc = rc.correct(dm)
# Optional: Remove the scattered light, allowing for the large offset from scattered light
if with_offset:
corrected_lc = (self.raw_lc - rc.model_lc +
np.percentile(rc.model_lc.flux, q=5))
lc = corrected_lc.normalize()
self.lc_custom = lc
# compute Contamination
if self.gaia_sources is None:
gaia_sources = self.query_gaia_dr2_catalog(radius=120,
verbose=False)
else:
gaia_sources = self.gaia_sources
fluxes = get_fluxes_within_mask(self.tpf, self.aper_mask, gaia_sources)
self.contratio = sum(fluxes) - 1
if self.tic_params is None:
_ = self.query_tic_catalog(return_nearest_xmatch=True)
tic_contratio = self.tic_params.contratio
dcontratio = abs(tic_contratio - self.contratio)
if (tic_contratio is not None) & (dcontratio > 0.5):
print(f"contratio: {self.contratio:.2f} (TIC={tic_contratio:.2f})")
# add method
lc.detrend = lambda: detrend(lc)
return lc
def make_custom_lc(
self,
sector=None,
tpf_size=None,
sap_mask=None,
aper_radius=None,
percentile=None,
threshold_sigma=None,
use_pld=True,
pixel_components=3,
spline_n_knots=100,
spline_degree=3,
background_mask=None,
pca_nterms=5,
with_offset=True,
):
"""
create a custom lightcurve based on this tutorial:
https://docs.lightkurve.org/tutorials/04-how-to-remove-tess-scattered-light-using-regressioncorrector.html
Parameters
----------
sector : int or str
specific sector or all
cutout_size : tuple
tpf cutout size
aper_radius: int
aperture mask radius
percentile: float
aperture mask percentile
threshold_sigma: float
aperture mask threshold [sigma]
method : float
PLD (default)
Returns
-------
corrected_lc : lightkurve object
"""
if self.verbose:
print("Using lightcurve with custom aperture.")
sector = sector if sector is not None else self.sector
sap_mask = sap_mask if sap_mask else self.sap_mask
aper_radius = aper_radius if aper_radius else self.aper_radius
percentile = percentile if percentile else self.percentile
threshold_sigma = (threshold_sigma
if threshold_sigma else self.threshold_sigma)
cutout_size = tpf_size if tpf_size else self.cutout_size
tpf_tesscut = self.get_tpf_tesscut(sector=sector,
cutout_size=cutout_size)
self.aper_mask = parse_aperture_mask(
tpf_tesscut,
sap_mask=sap_mask,
aper_radius=aper_radius,
percentile=percentile,
threshold_sigma=threshold_sigma,
verbose=False,
)
raw_lc = tpf_tesscut.to_lightcurve(method="aperture",
aperture_mask=self.aper_mask)
# remove nans
idx = (np.isnan(raw_lc.time)
| np.isnan(raw_lc.flux)
| np.isnan(raw_lc.flux_err))
self.tpf_tesscut = tpf_tesscut[~idx]
self.lc_custom_raw = raw_lc[~idx]
if use_pld:
if self.verbose:
print("Removing scattered light + applying PLD")
pld = lk.TessPLDCorrector(self.tpf_tesscut,
aperture_mask=self.aper_mask)
if background_mask is None:
background_mask = ~self.aper_mask
corrected_lc = pld.correct(
pixel_components=pixel_components,
spline_n_knots=spline_n_knots,
spline_degree=spline_degree,
background_mask=background_mask,
)
self.corrector = pld
else:
if self.verbose:
print("Removing scattered light")
# Make a design matrix and pass it to a linear regression corrector
regressors = tpf_tesscut.flux[~idx][:, ~self.aper_mask]
dm = (lk.DesignMatrix(
regressors,
name="regressors").pca(nterms=pca_nterms).append_constant())
rc = lk.RegressionCorrector(raw_lc)
self.corrector = rc
corrected_lc = rc.correct(dm)
# Optional: Remove the scattered light, allowing for the large offset from scattered light
if with_offset:
corrected_lc = (raw_lc - rc.model_lc +
np.percentile(rc.model_lc.flux, q=5))
lc = corrected_lc.normalize()
self.lc_custom = lc
# compute Contamination
if self.gaia_sources is None:
gaia_sources = self.query_gaia_dr2_catalog(radius=120)
else:
gaia_sources = self.gaia_sources
fluxes = get_fluxes_within_mask(self.tpf_tesscut, self.aper_mask,
gaia_sources)
self.contratio = sum(fluxes) - 1
# add method
lc.detrend = lambda: detrend(lc)
return lc
def get_LCs(CLUSTERS,phot_type):
NUMBER_PCA_COMPONENTS = 5 #no of vectors for PCA
#initialize the masks
star_mask=np.empty([len(tpfs),cutout_size,cutout_size],dtype='bool')
if phot_type == 'pixel': # if pixel photometry is requested, define a different variable
radius_mask=np.empty([len(tpfs),cutout_size,cutout_size],dtype='bool')
sky_mask=np.empty([len(tpfs),cutout_size,cutout_size],dtype='bool')
frame = tpfs[0].shape[0]//2
# obtain the aperture and the background masks with help of cluster radius and percentile (85%)
if phot_type == 'pixel':
radius_mask[0],sky_mask[0] = circle_aperture(tpfs[0][frame].flux,tpfs[0][frame].flux, \
Complete_Clusters[Complete_Clusters['NAME'] == CLUSTERS[0]]['CORE_RADIUS'].values[0],85)
elif phot_type == 'ensemble':
star_mask[0],sky_mask[0] = circle_aperture(tpfs[0][frame].flux,tpfs[0][frame].flux, \
Complete_Clusters[Complete_Clusters['NAME'] == CLUSTERS[0]]['CORE_RADIUS'].values[0],85)
# if pixel photometry, iterate over each pixel in the radius mask and download LCs
if phot_type == 'pixel':
for i in range(len(np.where(radius_mask[0])[0])):
star_mask=np.zeros([len(tpfs),cutout_size,cutout_size],dtype='bool')
pixel_loc=np.where(radius_mask[0])[0][i],np.where(radius_mask[0])[1][i]
star_mask[0][pixel_loc]=True
lightcurves = [tpfs[i].to_lightcurve(aperture_mask=star_mask[i]) for i in range(len(tpfs))];
for i in range(len(lightcurves)):
lightcurves[i]=lightcurves[i][lightcurves[i].flux_err > 0]
#PCA correction
regressors = [tpfs[i].flux[:, sky_mask[i]] for i in range(len(tpfs))]# The regressor is the inverse of the aperture
#Design a matrix for linear regression
dms = [lk.DesignMatrix(r, name='regressors').pca(NUMBER_PCA_COMPONENTS).append_constant() for r in regressors]
#Remove noise using linear regression against a DesignMatrix.
correctors = [lk.RegressionCorrector(lc) for lc in lightcurves]
correctedLCs = [correctors[i].correct(dms[i]) for i in range(len(correctors))]#this is how to subtract the background noise from the star pixels
#polyomial correction. to remove long period systematic trends that survived the PCA correction
correctedLCs_poly,poly_terms = polynomial_correction(correctedLCs[0])
#save the pixel corrected LCs
correctedLCs_poly.to_csv(CLUSTERS[0]+' pixels/core_pixels/corrected_LCs_CORE_RADIUS_pixel_no.{0}'.format(pixel_loc))
# is ensemble photometry, download LC for entire aperture
elif phot_type == 'ensemble':
lightcurves = [tpfs[i].to_lightcurve(aperture_mask=star_mask[i]) for i in range(len(tpfs))];
for i in range(len(lightcurves)):
lightcurves[i]=lightcurves[i][lightcurves[i].flux_err > 0]
regressors = [tpfs[i].flux[:, sky_mask[i]] for i in range(len(tpfs))]# The regressor is the inverse of the aperture
#Design a matrix for linear regression
dms = [lk.DesignMatrix(r, name='regressors').pca(NUMBER_PCA_COMPONENTS).append_constant() for r in regressors]
#Remove noise using linear regression against a DesignMatrix.
correctors = [lk.RegressionCorrector(lc) for lc in lightcurves]
correctedLCs = [correctors[i].correct(dms[i]) for i in range(len(correctors))]
#polynomial correction
correctedLCs_poly,poly_terms = polynomial_correction(correctedLCs[0])
#calculate the lomb-scargle periodogram for the ensemble corrected LC for a range frequencies
t = correctedLCs_poly[correctedLCs_poly.quality == 0].time
dy = correctedLCs_poly[correctedLCs_poly.quality == 0].flux_err
y = correctedLCs_poly[correctedLCs_poly.quality == 0].flux
omega=np.arange(0.05,11,0.01)
P_LS = LombScargle(t, y, dy=dy).power(omega)# LS power
# max_power=np.max(P_LS)
# freq_at_max_power=omega[np.argmax(P_LS)]
return omega, P_LS #return frequencies and LS power
def extract_light_curve(fits_filename, outputdir, return_msg=True):
# Output name
output = Path(fits_filename.stem + '_corrected.pickled')
output = outputdir / output
# Parameters and Criteria:
sigma_clipping = 5 # To be applied after the detrending of the light curve
# Structure the data to be saved
results = {
'tic': None,
'sector': None,
'ra': None,
'dec': None,
'headers': None, # Headers from the original FITS file
'fit': None, # Result from fit
'neighbours_all': None, # All neighbours stars info
'neighbours_used': None, # Used neighbours stars info
'target': None, # Target star info
'aperture_threshold': None, # HDU to store tabular information
'pca_all': None, # PCA results
'pca_used': None, # PCA results
'centroids': None, # Centroids results
'excluded_intervals': None, # Excluded intervals in days
'lc_raw': None, # Light curves
'lc_raw_nonan': None, # Light curves
'lc_trend': None, # Light curves
'lc_regressed': None, # Light curves
'lc_regressed_notoutlier': None, # Light curves
'median_image': None,
'masks': None,
'tag': None
}
# Save information from header from original FITS file
HDUL, ext = [], 0
while True:
try:
HDUL.append(
fits.getheader(fits_filename.as_posix(), ext=ext).tostring())
ext += 1
except IndexError:
break
except Exception as e:
print('Unexpected exception when reading headers from FITS: ', e)
break
results['headers'] = HDUL
# Load the TESS taret pixel file
try:
tpf = lk.TessTargetPixelFile(fits_filename.as_posix())
except Exception as e:
# Save results
err_msg = f'"lightkurve.TessTargetPixelFile()" could not open file {fits_filename.as_posix()}. Exception: {e}'
results['tag'] = err_msg
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
tic = str(tpf.get_keyword('ticid'))
sector = str(tpf.get_keyword('sector'))
target_ra = np.float(tpf.ra)
target_dec = np.float(tpf.dec)
# Store to results
results['tic'], results['sector'] = tic, sector
results['ra'], results['dec'] = target_ra, target_dec
# Initialize messages
id_msg = f'TIC {tic} Sector {sector}: Skipped: '
OK_msg = f'TIC {tic} Sector {sector}: OK'
# Calculate the median image
with warnings.catch_warnings():
warnings.simplefilter('ignore')
median_image = np.nanmedian(tpf.flux, axis=0)
# Store to results
results['median_image'] = median_image
# Estimate of aperture mask and background mask
ap_mask_threshold, bkg_mask_threshold = 5, 3
ap_mask = threshold_mask(median_image,
threshold=ap_mask_threshold,
reference_pixel='center')
ap_bkg = ~threshold_mask(
median_image, threshold=bkg_mask_threshold, reference_pixel=None)
# Exclude NaN values outside the camera
ap_bkg &= ~np.isnan(median_image)
# Estimate the median flux background
median_bkg_flux = np.median(median_image[ap_bkg])
# Store to results
results['masks'] = {'aperture':ap_mask,\
'background':ap_bkg}
# Check validity of aperture mask
OK_ap_mask, err_msg = check_aperture_mask(ap_mask, id_msg)
# If aperture is not good, exit program with corresponding message
if not OK_ap_mask:
# Save results
results['tag'] = err_msg
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
# Refine aperture
try:
WCS = tpf.wcs
except IndexError:
# Save results
err_msg = id_msg + 'No WCS info in header'
results['tag'] = err_msg
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
ap_mask,\
target_coord_pixel, target_tmag,\
nb_coords_pixel, nb_tmags,\
err_msg = refine_aperture(results, tic, target_ra, target_dec, WCS,\
ap_mask, ap_mask_threshold, median_image, prepend_err_msg=id_msg)
# If not satisfactory aperture mask
if ap_mask is None:
# Save results
results['tag'] = err_msg
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
# Variation in time of aperture's center of mass
centroid_col, centroid_row = tpf.estimate_centroids(aperture_mask=ap_mask,
method='quadratic')
centroid_col -= tpf.column
centroid_row -= tpf.row
sqrt_col2_row2 = np.sqrt(centroid_col**2 + centroid_row**2)
# Store to results
results['centroids'] = {'col':centroid_col,\
'row':centroid_row,\
'sqrt_col2_row2':sqrt_col2_row2,\
'time':tpf.time}
# Fit the image and find the contamination fraction within the aperture mask
fitted_image, err_msg = contamination(results, median_image,ap_mask,\
target_coord_pixel, target_tmag,\
nb_coords_pixel, nb_tmags,\
tpf.wcs,median_bkg_flux,
prepend_err_msg=id_msg)
if fitted_image is None:
# Save results
results['tag'] = err_msg
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
# Generate the raw light curve
lc_raw = tpf.to_lightcurve(aperture_mask=ap_mask, method='aperture')
# Store to results
results['lc_raw'] = {'flux':lc_raw.flux,\
'time':lc_raw.time}
# Find the indices of the quality mask that created the light curve
ind = np.argwhere(tpf.quality_mask == True)
# Masks with True value the light curve times with null or NaN flux
mask = lc_raw.flux == 0
mask |= lc_raw.flux == np.nan
# Set to False the indices to be masked (ignored).
# Note that we take a subset from `ind` because the masks where defined from the light curve
tpf.quality_mask[ind[mask]] = False
# Exclude intervals previously decided
exclude_interval(tpf, sector, results)
# Generate the light curve
lc = tpf.to_lightcurve(aperture_mask=ap_mask, method='aperture')
# Store to results
results['lc_raw_nonan'] = {'flux':lc.flux,\
'time':lc.time}
# Make a design matrix and pass it to a linear regression corrector
regressors = tpf.flux[:, ap_bkg]
# Number of PCs to use
npc, dm, rc = find_number_of_PCs(results, regressors, lc)
if npc == 0:
# Save results
results['tag'] = id_msg + 'None PC used, no detrended done.'
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return err_msg
return
try:
# Detrend light curve using PCA
dm = lk.DesignMatrix(regressors,
name='regressors').pca(npc).append_constant()
rc = lk.RegressionCorrector(lc)
lc_regressed = rc.correct(dm)
lc_trend = rc.diagnostic_lightcurves['regressors']
# Sigma clipping the remove outliers
lc_regressed_no_outliers, lc_mask_regressed_outliers = lc_regressed.remove_outliers(
return_mask=True, sigma=sigma_clipping)
# Store to results
results['lc_trend'] = {'flux':lc_trend.flux,\
'time':lc_trend.time}
results['lc_regressed'] = {'flux':lc_regressed.flux,\
'time':lc_regressed.time,\
'outlier_mask':lc_mask_regressed_outliers,\
'sigma_clipping':sigma_clipping}
results['lc_regressed_notoutlier'] = {'flux':lc_regressed_no_outliers.flux,\
'time':lc_regressed_no_outliers.time}
results['pca_used'] = {'coef':rc.coefficients,\
'pc':[dm.values[:,i] for i in range(dm.rank)],\
'dm':dm,\
'rc':rc,\
'npc':npc}
# Save results
results['tag'] = 'OK'
picklefile = open(output.as_posix(), 'wb')
pickle.dump(results, picklefile)
picklefile.close()
if return_msg: return OK_msg
return
except Exception as e:
print('!!!!!!!!!!!!!!!!!!!!!!!!!')
print(f' Sector {sector}.')
print('EXCEPTION:', e)
print('!!!!!!!!!!!!!!!!!!!!!!!!!')
if return_msg: return id_msg + '::' + repr(e) + '::' + str(e)
return
#Select best
cdpp = [lk.estimate_cdpp() for lk in lkf]
bidx = np.argmin(cdpp)
lkf = lkf[bidx]
color_print('\nAperture chosen: ', 'lightcyan', str(bidx+1) + 'px radius' if args.circ==0 else 'No. ' + str(bidx), 'default',
'\tNumber of pixels inside: ', 'lightcyan', str(dap[bidx].sum()), 'default')
#PCA
if args.pca:
import lightkurve
regressors = flux[:, ~dap[bidx]]
dm = lightkurve.DesignMatrix(regressors, name='regressors')
dm = dm.pca(5)
dm = dm.append_constant()
corrector = lightkurve.RegressionCorrector(lkf)
corr_flux = corrector.correct(dm)
lkf = lkf - corrector.model_lc + np.percentile(corrector.model_lc.flux, 5)
#lkf.flux = lkf.flux - corrector.model_lc + np.percentile(corrector.model_lc.flux, 5)
#PLD?
elif args.pld:
'''
if iP is None:
mask = np.ones(time.size).astype(bool)
else:
def build_X(
time,
pos_corr1,
pos_corr2,
flux=None,
t_model=None,
background=False,
cbvs=None,
spline=True,
spline_period=0.75,
sff=False,
windows=20,
bins=15,
):
"""Build a design matrix to model pixel in target pixel files
Parameters
----------
tpf : lightkurve.TargetPixelFile
Input target pixel file to make the design matrix for
flux : np.ndarray
The SAP flux to use for creating the design matrix
t_model: None or np.ndarray
The transit model, if None no transit model will be used in the design matrix
cbvs: None or np.ndarray
Cotrending Basis vectors. If None will not be used in design matrix
spline: bool
Whether to use a B-Spline in time
spline_period: float
If using a spline, what time period the knots should be spaced at
Returns
-------
SA : scipy.sparse.csr_matrix
The design matrix to use to detrend the input TPF
"""
r, c = np.nan_to_num(pos_corr1), np.nan_to_num(pos_corr2)
r[np.abs(r) > 10] = 0
c[np.abs(r) > 10] = 0
breaks = np.where((np.diff(time) > (np.median(np.diff(time)) * 20)))[0] - 1
breaks = breaks[breaks > 0]
if sff:
lc = lk.KeplerLightCurve(time=time,
flux=time**0,
centroid_col=pos_corr1,
centroid_row=pos_corr2)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
s = lk.correctors.SFFCorrector(lc)
_ = s.correct(windows=windows, bins=bins, timescale=spline_period)
centroids = s.dmc["sff"].X
else:
if np.nansum(r) == 0:
ts0 = np.asarray(
[np.in1d(time, t) for t in np.array_split(time, breaks)])
ts1 = np.asarray([
np.in1d(time, t) * (time - t.mean()) / (t[-1] - t[0])
for t in np.array_split(time, breaks)
])
time_array = np.vstack([ts0, ts1, ts1**2]).T
centroids = np.copy(time_array)
else:
centroids = np.nan_to_num(
np.vstack([
r**idx * c**jdx for idx in np.arange(0, 4)
for jdx in range(0, 4)
]).T)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
centroids = lk.correctors.DesignMatrix(centroids).split(
list(breaks)).X
A = csr_matrix(np.copy(centroids))
if cbvs is not None:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
A = hstack([
A,
lk.DesignMatrix(np.nan_to_num(cbvs)).split(list(breaks)).X
]).tocsr()
if spline:
spline_dm = create_sparse_spline_matrix(
time,
n_knots=int(np.max([4,
int((time[-1] - time[0]) // spline_period)
]))).X
A = hstack([csr_matrix(A), spline_dm]).tocsr()
if flux is None:
if t_model is not None:
return hstack([A, np.atleast_2d(t_model).T]).tocsr()
return A
# SA = A.multiply(np.atleast_2d(flux).T).tocsr()
if t_model is not None:
SA = hstack([A, np.atleast_2d(t_model).T]).tocsr()
else:
SA = A.tocsr()
return SA
def build_X(tpf,
flux,
t_model=None,
background=False,
cbvs=None,
spline=True,
spline_period=2,
sff=False):
"""Build a design matrix to use in the model"""
r, c = np.nan_to_num(tpf.pos_corr1), np.nan_to_num(tpf.pos_corr2)
r[np.abs(r) > 10] = 0
c[np.abs(r) > 10] = 0
if sff:
r = lk.SFFCorrector(lk.LightCurve(tpf.time, flux))
_ = r.correct()
centroids = r.X['sff']
else:
breaks = np.where((np.diff(tpf.time) > (0.0202 * 10)))[0] - 1
breaks = breaks[breaks > 0]
if r.sum() == 0:
ts0 = np.asarray([
np.in1d(tpf.time, t) for t in np.array_split(tpf.time, breaks)
])
ts1 = np.asarray([
np.in1d(tpf.time, t) * (tpf.time - t.mean()) / (t[-1] - t[0])
for t in np.array_split(tpf.time, breaks)
])
centroids = np.vstack([ts0, ts1, ts1**2]).T
else:
rs0 = np.asarray([
np.in1d(tpf.time, t) for t in np.array_split(tpf.time, breaks)
])
rs1 = np.asarray([
np.in1d(tpf.time, t) * (r - r[np.in1d(tpf.time, t)].mean())
for t in np.array_split(tpf.time, breaks)
])
cs1 = np.asarray([
np.in1d(tpf.time, t) * (c - c[np.in1d(tpf.time, t)].mean())
for t in np.array_split(tpf.time, breaks)
])
centroids = np.vstack([
rs1, cs1, rs1 * cs1, rs1**2, cs1**2, rs1**2 * cs1,
rs1 * cs1**2, rs1**2 * cs1**2, rs1**3 * cs1**3,
rs1**3 * cs1**2, rs1**3 * cs1, rs1**3, cs1**3, cs1**3 * rs1,
cs1**3 * rs1**2
]).T
A = np.copy(centroids)
if cbvs is not None:
A = np.hstack([A, np.nan_to_num(cbvs)])
if background:
bkg = lk.DesignMatrix(
tpf.flux[:, ~tpf.create_threshold_mask()]).pca(3).values
A = np.hstack([A, bkg])
if spline:
spline_dm = lk.correctors.designmatrix.create_spline_matrix(
tpf.time,
n_knots=np.max(
[4, int(
(tpf.time[-1] - tpf.time[0]) // spline_period)])).values
A = np.hstack([A, spline_dm])
SA = np.atleast_2d(flux).T * A
if t_model is not None:
SA = np.hstack([
SA,
np.atleast_2d(np.ones(len(tpf.time))).T,
np.atleast_2d(t_model).T
])
else:
SA = np.hstack([SA, np.atleast_2d(np.ones(len(tpf.time))).T])
return csr_matrix(SA)
def __init__(self,
targetpixelfile,
gaia=True,
magnitude_limit=18,
frequencies=[],
frequnit=u.uHz,
principle_components=5,
aperture=None,
**kwargs):
#Defining an aperture that will be used in plotting and making empty 2-d arrays of the correct size for masks
if targetpixelfile.pipeline_mask.any() == False:
self.aperture = aperture
else:
self.aperture = targetpixelfile.pipeline_mask
self.tpf = targetpixelfile
# Make a design matrix and pass it to a linear regression corrector
self.raw_lc = self.tpf.to_lightcurve(aperture_mask=self.aperture)
self.dm = lk.DesignMatrix(
self.tpf.flux[:, ~self.tpf.create_threshold_mask()],
name='regressors').pca(principle_components)
rc = lk.RegressionCorrector(self.raw_lc)
corrected_lc = rc.correct(self.dm.append_constant())
corrected_lc[np.where(corrected_lc.quality == 0)]
self.corrected_lc = corrected_lc.remove_outliers()
self.frequency_list = np.asarray((frequencies * frequnit).to(1 / u.d))
self.principle_components = principle_components
def Obtain_Initial_Phase(tpf, corrected_lc, frequency_list):
flux = corrected_lc.flux.value
times = corrected_lc.time.value - np.mean(corrected_lc.time.value)
pg = corrected_lc.to_periodogram(frequency=np.append(
[0.0001], frequency_list),
ls_method='slow')
initial_flux = np.asarray(pg.power[1:])
initial_phase = np.zeros(len(frequency_list))
def lc_model(time, amp, freq, phase):
return amp * np.sin(2 * np.pi * freq * time + phase)
def background_model(time, height):
return np.ones(len(time)) * height
for j in np.arange(len(frequency_list)):
for i in np.arange(len(frequency_list)):
if (i == 0):
model = lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model += lm.Model(background_model,
independent_vars=['time'])
else:
model += lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model.set_param_hint('f{0:d}phase'.format(i),
min=-np.pi,
max=np.pi,
value=initial_phase[i],
vary=False)
model.set_param_hint('f{0:d}amp'.format(i),
value=initial_flux[i],
vary=False)
model.set_param_hint('height',
value=np.mean(flux),
vary=False)
model.set_param_hint('f{0:d}freq'.format(i),
value=frequency_list[i],
vary=False)
params = model.make_params()
params['f{0:d}phase'.format(j)].set(vary=True)
params['f{0:d}phase'.format(j)].set(value=initial_phase[j])
params['f{0:d}phase'.format(j)].set(brute_step=np.pi / 10)
result = model.fit(corrected_lc.flux.value,
params,
time=times,
method='brute')
initial_phase[j] = result.best_values['f{0:d}phase'.format(j)]
return initial_phase
self.initial_phases = Obtain_Initial_Phase(self.tpf, self.corrected_lc,
self.frequency_list)
def Obtain_Final_Phase(tpf, corrected_lc, frequency_list,
initial_phases):
flux = corrected_lc.flux.value
times = corrected_lc.time.value - np.mean(corrected_lc.time.value)
pg = corrected_lc.to_periodogram(frequency=np.append(
[0.0001], frequency_list),
ls_method='slow')
initial_flux = np.asarray(pg.power[1:])
def lc_model(time, amp, freq, phase):
return amp * np.sin(2 * np.pi * freq * time + phase)
def background_model(time, height):
return np.ones(len(time)) * height
for i in np.arange(len(frequency_list)):
if (i == 0):
model = lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model += lm.Model(background_model,
independent_vars=['time'])
else:
model += lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model.set_param_hint('f{0:d}phase'.format(i),
min=-np.pi,
max=np.pi,
value=initial_phases[i],
vary=True)
model.set_param_hint('f{0:d}amp'.format(i),
value=initial_flux[i],
vary=True)
model.set_param_hint('height', value=np.mean(flux), vary=True)
model.set_param_hint('f{0:d}freq'.format(i),
value=frequency_list[i],
vary=False)
params = model.make_params()
result = model.fit(corrected_lc.flux.value, params, time=times)
final_phases = [
result.best_values['f{0:d}phase'.format(j)]
for j in np.arange(len(frequency_list))
]
return final_phases
self.final_phases = Obtain_Final_Phase(self.tpf, self.corrected_lc,
self.frequency_list,
self.initial_phases)
def Obtain_Final_Fit(tpf, corrected_lc, frequency_list, final_phases):
flux = corrected_lc.flux.value
times = corrected_lc.time.value - np.mean(corrected_lc.time.value)
pg = corrected_lc.to_periodogram(frequency=np.append(
[0.0001], frequency_list),
ls_method='slow')
initial_flux = np.asarray(pg.power[1:])
def lc_model(time, amp, freq, phase):
return amp * np.sin(2 * np.pi * freq * time + phase)
def background_model(time, height):
return np.ones(len(time)) * height
for i in np.arange(len(frequency_list)):
if (i == 0):
model = lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model += lm.Model(background_model,
independent_vars=['time'])
else:
model += lm.Model(lc_model,
independent_vars=['time'],
prefix='f{0:d}'.format(i))
model.set_param_hint('f{0:d}phase'.format(i),
value=final_phases[i],
vary=False)
model.set_param_hint('f{0:d}amp'.format(i),
value=initial_flux[i],
vary=True)
model.set_param_hint('height', value=np.mean(flux), vary=True)
model.set_param_hint('f{0:d}freq'.format(i),
value=frequency_list[i],
vary=False)
params = model.make_params()
result = model.fit(corrected_lc.flux.value, params, time=times)
return result
heats = []
heats_error = []
#Iterating through columns of pixels
for i in np.arange(0, len(self.aperture)):
#Iterating through rows of pixels
for j in np.arange(0, len(self.aperture[0])):
#Making an empty 2-d array
mask = np.zeros((len(self.aperture), len(self.aperture[0])),
dtype=bool)
#Iterating to isolate pixel by pixel to get light curves
mask[i][j] = True
#Getting the light curve for a pixel and excluding any flagged data
lightcurve = self.tpf.to_lightcurve(aperture_mask=mask)
rcc = lk.RegressionCorrector(lightcurve)
lc = rcc.correct(self.dm.append_constant())
#lc = lc[np.where(lc.quality == 0)]
#lc = lc.remove_outliers()
bestfit = Obtain_Final_Fit(self.tpf, lc, self.frequency_list,
self.final_phases)
heat = np.asarray([
bestfit.best_values['f{0:d}amp'.format(n)]
for n in np.arange(len(self.frequency_list))
])
#heat = bestfit.best_values['f0amp']# / bestfit.params['f0amp'].stderr
heat_error = np.asarray([
bestfit.params['f{0:d}amp'.format(n)].stderr
for n in np.arange(len(self.frequency_list))
])
#Extending the list of fitting data for each pixel
heats.extend([heat])
heats_error.extend([heat_error])
#Taking the final list and turning it into a 2-d numpy array with the same dimensions of the full postage stamp
#heats = np.reshape(np.asarray(heats),(len(self.aperture),len(self.aperture[0])))
#heats_error = np.reshape(np.asarray(heats_error),(len(self.aperture),len(self.aperture[0])))
heats = np.asarray(heats)
heats_error = np.asarray(heats_error)
#Defining self.periodogram as this 2-d array of periodogram data
self.heatmap = heats.T
self.heatmap_error = heats_error.T
self.timeserieslength = (self.tpf.time.max() -
self.tpf.time.min()).value
self.gaiadata = None
if (gaia == True):
"""Make the Gaia Figure Elements"""
# Get the positions of the Gaia sources
c1 = SkyCoord(self.tpf.ra, self.tpf.dec, frame='icrs', unit='deg')
# Use pixel scale for query size
pix_scale = 4.0 # arcseconds / pixel for Kepler, default
if self.tpf.mission == 'TESS':
pix_scale = 21.0
# We are querying with a diameter as the radius, overfilling by 2x.
from astroquery.vizier import Vizier
Vizier.ROW_LIMIT = -1
result = Vizier.query_region(
c1,
catalog=["I/345/gaia2"],
radius=Angle(np.max(self.tpf.shape[1:]) * pix_scale, "arcsec"))
no_targets_found_message = ValueError(
'Either no sources were found in the query region '
'or Vizier is unavailable')
too_few_found_message = ValueError(
'No sources found brighter than {:0.1f}'.format(
magnitude_limit))
if result is None:
raise no_targets_found_message
elif len(result) == 0:
raise too_few_found_message
result = result["I/345/gaia2"].to_pandas()
result = result[result.Gmag < magnitude_limit]
if len(result) == 0:
raise no_targets_found_message
year = ((self.tpf.time[0].jd - 2457206.375) * u.day).to(u.year)
pmra = ((np.nan_to_num(np.asarray(result.pmRA)) *
u.milliarcsecond / u.year) * year).to(u.deg).value
pmdec = ((np.nan_to_num(np.asarray(result.pmDE)) *
u.milliarcsecond / u.year) * year).to(u.deg).value
result.RA_ICRS += pmra
result.DE_ICRS += pmdec
radecs = np.vstack([result['RA_ICRS'], result['DE_ICRS']]).T
coords = self.tpf.wcs.all_world2pix(radecs, 0)
# Gently size the points by their Gaia magnitude
sizes = 64.0 / 2**(result['Gmag'] / 5.0)
one_over_parallax = 1.0 / (result['Plx'] / 1000.)
source = dict(ra=result['RA_ICRS'],
dec=result['DE_ICRS'],
source=result['Source'].astype(str),
Gmag=result['Gmag'],
plx=result['Plx'],
one_over_plx=one_over_parallax,
x=coords[:, 0],
y=coords[:, 1],
size=sizes)
self.gaiadata = source
class frequency_heatmap:
def __init__(self, tpf, heats, heats_error, frequencies,
gaia_data):
self.heat_stamp = heats
self.gaiadata = gaia_data
self.heatmap_error = heats_error
self.size = tpf.pipeline_mask.shape
self.frequencies = frequencies
def centroid(self):
#Residuals to minimize relative to the error bars
def residual(params, amp, amperr):
x = params['x']
y = params['y']
sigma = params['sigma']
xpix, ypix = np.meshgrid(np.arange(self.size[0]),
np.arange(self.size[1]))
res = []
for i in np.arange(len(self.frequencies)):
height = params['height{0:d}'.format(i)]
model = height * np.exp(-(((x - xpix) / sigma)**2 +
((y - ypix) / sigma)**2) / 2)
res.extend([(amp[i].reshape(self.size) - model) /
amperr[i].reshape(self.size)])
return np.asarray(res)
#Set starting values to converge from
self.heatmap_error[np.where(
self.heatmap_error == None)] = np.nan
composite_heatmap = self.heat_stamp.sum(axis=0).reshape(
self.size) / ((np.nansum(
self.heatmap_error**2, axis=0))**(1 / 2)).reshape(
self.size) #issue with numpy using sqrt?
c = np.where(composite_heatmap == composite_heatmap.max())
params = Parameters()
for i in np.arange(len(frequencies)):
params.add('height{0:d}'.format(i),
value=np.max(self.heat_stamp[i]))
params.add('x', value=c[1][0])
params.add('y', value=c[0][0])
params.add('sigma', value=1)
#Do the fit
minner = Minimizer(residual,
params,
fcn_args=(self.heat_stamp,
self.heatmap_error))
self.result = minner.minimize()
fit = self.result.params.valuesdict()
self.x = fit['x']
self.y = fit['y']
def star_list(self):
gaia_data = self.gaiadata
no_gaia_data_message = ValueError(
'No gaia data initialized in PixelMapPeriodogram class')
if gaia_data == None:
raise no_gaia_data_message
else:
distances = np.square(self.x - gaia_data['x']) + np.square(
self.y - gaia_data['y'])
closest_star_mask = np.where(
np.square(self.x - gaia_data['x']) +
np.square(self.y - gaia_data['y']) == (
np.square(self.x - gaia_data['x']) +
np.square(self.y - gaia_data['y'])).min())
stars = dict(
ra=np.asarray(gaia_data['ra']),
dec=np.asarray(gaia_data['dec']),
source=np.asarray(gaia_data['source']),
x=np.asarray(gaia_data['x']),
y=np.asarray(gaia_data['y']),
distance=distances,
probability=2 * stats.norm.sf(
distances,
scale=np.sqrt(self.result.params['x'].stderr**2 +
self.result.params['y'].stderr**2))
) #I believe mutiply by 2 since we wont have a negative distance
starlist = pd.DataFrame.from_dict(stars)
self.stars = starlist.sort_values(by=[r'distance'])
fh = frequency_heatmap(self.tpf, self.heatmap, self.heatmap_error,
self.frequency_list, self.gaiadata)
fh.centroid()
fh.star_list()
self.centroid = [fh.x, fh.y]
self.heatmap = self.heatmap.sum(axis=0).reshape(
self.aperture.shape[0], self.aperture.shape[1]) / np.sqrt(
(self.heatmap_error**2).sum(axis=0)).reshape(
self.aperture.shape[0], self.aperture.shape[1])
self.starfit = fh.stars.reset_index()
self.result = fh.result
def build(self, object_info, sherlock_dir):
mission_id = object_info.mission_id()
sherlock_id = object_info.sherlock_id()
quarters = None
sectors = None
logging.info("Retrieving star catalog info...")
mission, mission_prefix, id = super().parse_object_id(mission_id)
if mission_prefix not in self.star_catalogs:
raise ValueError("Wrong object id " + mission_id)
star_info = starinfo.StarInfo(
sherlock_id, *self.star_catalogs[mission_prefix].catalog_info(id))
logging.info("Downloading lightcurve files...")
sectors = None if object_info.sectors == 'all' or mission != "TESS" else object_info.sectors
quarters = None if object_info.sectors == 'all' or mission != "K2" else object_info.sectors
campaigns = None if object_info.sectors == 'all' or mission != "Kepler" else object_info.sectors
if object_info.aperture_file is None:
lcf = lk.search_lightcurve(str(mission_id), mission=mission, cadence="short",
sector=sectors, quarter=quarters,
campaign=campaigns, author=self.authors[mission])\
.download_all()
if lcf is None:
raise ObjectProcessingError(
"Light curve not found for object id " + mission_id)
lc = None
matching_objects = []
for i in range(0, len(lcf.PDCSAP_FLUX)):
if lcf.PDCSAP_FLUX[i].label == mission_id:
if lc is None:
lc = lcf.PDCSAP_FLUX[i].normalize()
else:
lc = lc.append(lcf.PDCSAP_FLUX[i].normalize())
else:
matching_objects.append(lcf.PDCSAP_FLUX[i].label)
if len(matching_objects) > 0:
logging.warning(
"================================================")
logging.warning("TICS IN THE SAME PIXEL: " +
str(matching_objects))
logging.warning(
"================================================")
lc = lc.remove_nans()
transits_min_count = self.__calculate_transits_min_count(len(lcf))
if mission_prefix == self.MISSION_ID_KEPLER or mission_id == self.MISSION_ID_KEPLER_2:
quarters = [lcfile.quarter for lcfile in lcf]
elif mission_prefix == self.MISSION_ID_TESS:
sectors = [file.sector for file in lcf]
if mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
quarters = [lcfile.campaign for lcfile in lcf]
lc = lc.to_corrector("sff").correct(windows=20)
return lc, star_info, transits_min_count, np.unique(
sectors), np.unique(quarters)
else:
logging.info("Using user apertures!")
tpfs = lk.search_targetpixelfile(
str(mission_id),
mission=mission,
cadence="short",
sector=sectors,
quarter=quarters,
campaign=campaigns,
author=authors[mission]).download_all()
apertures = {}
if isinstance(object_info.aperture_file, str):
aperture = []
with open(object_info.aperture_file, 'r') as fd:
reader = csv.reader(fd)
for row in reader:
aperture.append(row)
aperture = np.array(aperture)
for tpf in tpfs:
if mission_prefix == self.MISSION_ID_KEPLER:
apertures[tpf.quarter] = aperture
elif mission_prefix == self.MISSION_ID_TESS:
apertures[tpf.sector] = aperture
elif mission_prefix == self.MISSION_ID_KEPLER_2:
apertures[tpf.campaign] = aperture
else:
for sector, aperture_file in object_info.aperture_file.items():
aperture = []
with open(aperture_file, 'r') as fd:
reader = csv.reader(fd)
for row in reader:
aperture.append(row)
aperture = np.array(aperture)
apertures[sector] = aperture
lc = None
for tpf in tpfs:
if mission_prefix == self.MISSION_ID_KEPLER:
sector = tpf.quarter
elif mission_prefix == self.MISSION_ID_TESS:
sector = tpf.sector
elif mission_prefix == self.MISSION_ID_KEPLER_2:
sector = tpf.campaign
aperture = apertures[sector].astype(bool)
tpf.plot(aperture_mask=aperture, mask_color='red')
plt.savefig(sherlock_dir + "/Aperture_[" + str(sector) +
"].png")
plt.close()
if mission_prefix == self.MISSION_ID_KEPLER:
corrector = lk.KeplerCBVCorrector(tpf)
corrector.plot_cbvs([1, 2, 3, 4, 5, 6, 7])
raw_lc = tpf.to_lightcurve(
aperture_mask=aperture).remove_nans()
plt.savefig(sherlock_dir + "/Corrector_components[" +
str(sector) + "].png")
plt.close()
it_lc = corrector.correct([1, 2, 3, 4, 5])
ax = raw_lc.plot(color='C3',
label='SAP Flux',
linestyle='-')
it_lc.plot(ax=ax,
color='C2',
label='CBV Corrected SAP Flux',
linestyle='-')
plt.savefig(sherlock_dir + "/Raw_vs_CBVcorrected_lc[" +
str(sector) + "].png")
plt.close()
elif mission_prefix == self.MISSION_ID_KEPLER_2:
raw_lc = tpf.to_lightcurve(
aperture_mask=aperture).remove_nans()
it_lc = raw_lc.to_corrector("sff").correct(windows=20)
ax = raw_lc.plot(color='C3',
label='SAP Flux',
linestyle='-')
it_lc.plot(ax=ax,
color='C2',
label='CBV Corrected SAP Flux',
linestyle='-')
plt.savefig(sherlock_dir + "/Raw_vs_SFFcorrected_lc[" +
str(sector) + "].png")
plt.close()
elif mission_prefix == self.MISSION_ID_TESS:
temp_lc = tpf.to_lightcurve(aperture_mask=aperture)
where_are_NaNs = np.isnan(temp_lc.flux)
temp_lc = temp_lc[np.where(~where_are_NaNs)]
regressors = tpf.flux[np.argwhere(~where_are_NaNs),
~aperture]
temp_token_lc = [
temp_lc[i:i + 2000]
for i in range(0, len(temp_lc), 2000)
]
regressors_token = [
regressors[i:i + 2000]
for i in range(0, len(regressors), 2000)
]
it_lc = None
raw_it_lc = None
item_index = 0
for temp_token_lc_item in temp_token_lc:
regressors_token_item = regressors_token[item_index]
design_matrix = lk.DesignMatrix(
regressors_token_item,
name='regressors').pca(5).append_constant()
corr_lc = lk.RegressionCorrector(
temp_token_lc_item).correct(design_matrix)
if it_lc is None:
it_lc = corr_lc
raw_it_lc = temp_token_lc_item
else:
it_lc = it_lc.append(corr_lc)
raw_it_lc = raw_it_lc.append(temp_token_lc_item)
item_index = item_index + 1
ax = raw_it_lc.plot(label='Raw light curve')
it_lc.plot(ax=ax, label='Corrected light curve')
plt.savefig(sherlock_dir + "/Raw_vs_DMcorrected_lc[" +
str(sector) + "].png")
plt.close()
if lc is None:
lc = it_lc.normalize()
else:
lc = lc.append(it_lc.normalize())
lc = lc.remove_nans()
lc.plot(label="Normalized light curve")
plt.savefig(sherlock_dir + "/Normalized_lc[" + str(sector) +
"].png")
plt.close()
transits_min_count = self.__calculate_transits_min_count(len(tpfs))
if mission_prefix == self.MISSION_ID_KEPLER or mission_id == self.MISSION_ID_KEPLER_2:
quarters = [lcfile.quarter for lcfile in tpfs]
elif mission_prefix == self.MISSION_ID_TESS:
sectors = [file.sector for file in tpfs]
if mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
quarters = [lcfile.campaign for lcfile in tpfs]
return lc, star_info, transits_min_count, np.unique(
sectors), np.unique(quarters)