def test_sff_knots():
"""Is SFF robust against gaps in time and irregular time sampling?
This test creates a light curve with gaps in time between
days 20-30 and days 78-80. In addition, the time sampling rate changes
in the interval between day 30 and 78. SFF should fail without error.
"""
n_points = 300
fn = get_pkg_data_filename("../../tests/data/ep60021426alldiagnostics.csv")
data = np.genfromtxt(fn, delimiter=",", skip_header=1)
raw_flux = data[:, 1][:n_points]
centroid_col = data[:, 3][:n_points]
centroid_row = data[:, 4][:n_points]
time = np.concatenate((
np.linspace(0, 20, int(n_points / 3)),
np.linspace(30, 78, int(n_points / 3)),
np.linspace(80, 100, int(n_points / 3)),
))
lc = KeplerLightCurve(
time=time,
flux=raw_flux,
flux_err=np.ones(n_points) * 0.0001,
centroid_col=centroid_col,
centroid_row=centroid_row,
)
# These calls should not raise an exception:
SFFCorrector(lc).correct()
lc.to_corrector(method="sff").correct()
def test_sff_priors():
"""SFF Spline flux mean should == lc.flux.mean()
SFF arclength component should have mean 0
"""
n_points = 300
fn = get_pkg_data_filename("../../tests/data/ep60021426alldiagnostics.csv")
data = np.genfromtxt(fn, delimiter=",", skip_header=1)
raw_flux = data[:, 1][:n_points]
centroid_col = data[:, 3][:n_points]
centroid_row = data[:, 4][:n_points]
time = np.concatenate((
np.linspace(0, 20, int(n_points / 3)),
np.linspace(30, 78, int(n_points / 3)),
np.linspace(80, 100, int(n_points / 3)),
))
lc = KeplerLightCurve(
time=time,
flux=raw_flux,
flux_err=np.ones(n_points) * 0.0001,
centroid_col=centroid_col,
centroid_row=centroid_row,
)
sff = SFFCorrector(lc)
sff.correct() # should not raise an exception
assert np.isclose(sff.diagnostic_lightcurves["spline"].flux.mean(),
1,
atol=1e-3)
assert np.isclose(sff.diagnostic_lightcurves["sff"].flux.mean(),
0,
atol=1e-3)
def __init__(self,
time,
flux=None,
flux_err=None,
tic_id=None,
cam=None,
spm=None,
col_cent=None,
row_cent=None,
mission='TESS',
**kwargs):
self.tic_id = tic_id
self.cam = cam
self.spm = spm
self.col_cent = col_cent
self.row_cent = row_cent
self.mission = mission
KeplerLightCurve.__init__(self,
time=time,
flux=flux,
flux_err=flux_err,
**kwargs)
self.time_format = 'jd'
self.time_scale = 'tdb'
def __init__(self,
time=None,
flux=None,
flux_err=None,
time_format=None,
time_scale=None,
time_unit=None,
centroid_col=None,
centroid_row=None,
quality=None,
quality_bitmask=None,
channel=None,
campaign=None,
quarter=None,
sector=None,
mission=None,
cadenceno=None,
targetid=None,
ra=None,
dec=None,
label=None,
meta={},
detrended_flux=None,
detrended_flux_err=None,
flux_trends=None,
gaps=None,
flares=None,
flux_unit=None,
primary_header=None,
data_header=None,
pos_corr1=None,
pos_corr2=None,
origin='FLC',
fake_flares=None,
it_med=None,
pixel_flux=None,
pixel_flux_err=None,
pipeline_mask=None,
camera=None,
ccd=None,
saturation=None):
if mission == 'TESS':
TessLightCurve.__init__(
self,
time=time,
flux=flux,
flux_err=flux_err,
time_format=time_format,
time_scale=time_scale,
centroid_col=centroid_col,
centroid_row=centroid_row,
quality=quality,
quality_bitmask=quality_bitmask,
camera=camera,
cadenceno=cadenceno,
targetid=targetid,
ra=ra,
dec=dec,
label=label,
meta=meta,
sector=sector,
)
self.mission = mission
self.campaign = None
self.quarter = None
else:
KeplerLightCurve.__init__(self,
time=time,
flux=flux,
flux_err=flux_err,
time_format=time_format,
time_scale=time_scale,
centroid_col=centroid_col,
centroid_row=centroid_row,
quality=quality,
quality_bitmask=quality_bitmask,
channel=channel,
campaign=campaign,
quarter=quarter,
mission=mission,
cadenceno=cadenceno,
targetid=targetid,
ra=ra,
dec=dec,
label=label,
meta=meta)
self.flux_unit = flux_unit
self.time_unit = time_unit
self.gaps = gaps
self.flux_trends = flux_trends
self.primary_header = primary_header
self.data_header = data_header
self.pos_corr1 = pos_corr1
self.pos_corr2 = pos_corr2
self.origin = origin
self.detrended_flux = detrended_flux
self.detrended_flux_err = detrended_flux_err
self.pixel_flux = pixel_flux
self.pixel_flux_err = pixel_flux_err
self.pipeline_mask = pipeline_mask
self.it_med = it_med
self.saturation = saturation
columns = [
'istart', 'istop', 'cstart', 'cstop', 'tstart', 'tstop', 'ed_rec',
'ed_rec_err', 'ampl_rec', 'total_n_valid_data_points'
]
if detrended_flux is None:
self.detrended_flux = np.full_like(time, np.nan)
else:
self.detrended_flux = detrended_flux
if detrended_flux_err is None:
self.detrended_flux_err = np.full_like(time, np.nan)
else:
self.detrended_flux_err = detrended_flux_err
if saturation is None:
self.saturation = np.full_like(time, np.nan)
else:
self.saturation = saturation
if flares is None:
self.flares = pd.DataFrame(columns=columns)
else:
self.flares = flares
if fake_flares is None:
other_columns = ['duration_d', 'amplitude', 'ed_inj', 'peak_time']
self.fake_flares = pd.DataFrame(columns=other_columns)
else:
self.fake_flares = fake_flares
def test_sff_corrector():
"""Does our code agree with the example presented in Vanderburg
and Johnson (2014)?"""
# The following csv file, provided by Vanderburg and Johnson
# at https://www.cfa.harvard.edu/~avanderb/k2/ep60021426.html,
# contains the results of applying SFF to EPIC 60021426.
fn = get_pkg_data_filename("../../tests/data/ep60021426alldiagnostics.csv")
data = np.genfromtxt(fn, delimiter=",", skip_header=1)
mask = data[:, -2] == 0 # indicates whether the thrusters were on or off
time = data[:, 0]
raw_flux = data[:, 1]
corrected_flux = data[:, 2]
centroid_col = data[:, 3]
centroid_row = data[:, 4]
# NOTE: we need a small number of windows below because this test data set
# is unusually short, i.e. has an unusually small number of cadences.
lc = LightCurve(time=time,
flux=raw_flux,
flux_err=np.ones(len(raw_flux)) * 0.0001)
sff = SFFCorrector(lc)
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
restore_trend=True,
windows=1,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
assert len(sff.window_points) == 0 # expect 0 break points for 1 window
# masking
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
windows=3,
restore_trend=True,
cadence_mask=mask,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
assert len(sff.window_points) == 2 # expect 2 break points for 3 windows
# masking and breakindex
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
windows=3,
restore_trend=True,
cadence_mask=mask,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
# masking and breakindex and iters
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
windows=3,
restore_trend=True,
cadence_mask=mask,
niters=3,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
# masking and breakindex and bins
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
windows=3,
restore_trend=True,
cadence_mask=mask,
bins=5,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
assert np.all((sff.lc.flux_err / sff.corrected_lc.flux_err) == 1)
# masking and breakindex and bins and propagate_errors
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
windows=3,
restore_trend=True,
cadence_mask=mask,
bins=5,
propagate_errors=True,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
assert np.all((sff.lc.flux_err / sff.corrected_lc.flux_err) < 1)
# test using KeplerLightCurve interface
klc = KeplerLightCurve(
time=time,
flux=raw_flux,
flux_err=np.ones(len(raw_flux)) * 0.0001,
centroid_col=centroid_col,
centroid_row=centroid_row,
)
sff = klc.to_corrector("sff")
klc = sff.correct(windows=3, restore_trend=True)
assert np.isclose(corrected_flux, klc.flux, atol=0.001).all()
# Can plot
sff.diagnose()
def test_remote_data(path):
"""Can we correct a simple K2 light curve?"""
lc = KeplerLightCurve.read(path, quality_bitmask=None)
sff = SFFCorrector(lc.remove_nans())
sff.correct(windows=10, bins=5, timescale=0.5)
sff.correct(windows=10, bins=5, timescale=0.5, sparse=True)
def plot(self, ax=None, xlabel='Time (days)', **kwargs):
return KeplerLightCurve.plot(self, ax=None, xlabel=xlabel, **kwargs)
def execute_triceratops(cpus, indir, object_id, sectors, lc_file,
transit_depth, period, t0, transit_duration,
rp_rstar, a_rstar, bins, scenarios, sigma_mode,
contrast_curve_file):
""" Calculates probabilities of the signal being caused by any of the following astrophysical sources:
TP No unresolved companion. Transiting planet with Porb around target star. (i, Rp)
EB No unresolved companion. Eclipsing binary with Porb around target star. (i, qshort)
EBx2P No unresolved companion. Eclipsing binary with 2 × Porb around target star. (i, qshort)
PTP Unresolved bound companion. Transiting planet with Porb around primary star. (i, Rp, qlong)
PEB Unresolved bound companion. Eclipsing binary with Porb around primary star. (i, qshort, qlong)
PEBx2P Unresolved bound companion. Eclipsing binary with 2 × Porb around primary star. (i, qshort, qlong)
STP Unresolved bound companion. Transiting planet with Porb around secondary star. (i, Rp, qlong)
SEB Unresolved bound companion. Eclipsing binary with Porb around secondary star. (i, qshort, qlong)
SEBx2P Unresolved bound companion. Eclipsing binary with 2 × Porb around secondary star. (i, qshort, qlong)
DTP Unresolved background star. Transiting planet with Porb around target star. (i, Rp, simulated star)
DEB Unresolved background star. Eclipsing binary with Porb around target star. (i, qshort, simulated star)
DEBx2P Unresolved background star. Eclipsing binary with 2 × Porb around target star. (i, qshort, simulated star)
BTP Unresolved background star. Transiting planet with Porb around background star. (i, Rp, simulated star)
BEB Unresolved background star. Eclipsing binary with Porb around background star. (i, qshort, simulated star)
BEBx2P Unresolved background star. Eclipsing binary with 2 × Porb around background star. (i, qshort, simulated star)
NTP No unresolved companion. Transiting planet with Porb around nearby star. (i, Rp)
NEB No unresolved companion. Eclipsing binary with Porb around nearby star. (i, qshort)
NEBx2P No unresolved companion. Eclipsing binary with 2 × Porb around nearby star. (i, qshort)
FPP = 1 - (TP + PTP + DTP)
NFPP = NTP + NEB + NEBx2P
Giacalone & Dressing (2020) define validated planets as TOIs with NFPP < 10−3 and FPP < 0.015 (or FPP ≤ 0.01,
when rounding to the nearest percent)
@param cpus: number of cpus to be used
@param indir: root directory to store the results
@param id_int: the object id for which the analysis will be run
@param sectors: the sectors of the tic
@param lc_file: the light curve source file
@param transit_depth: the depth of the transit signal (ppts)
@param period: the period of the transit signal /days)
@param t0: the t0 of the transit signal (days)
@param transit_duration: the duration of the transit signal (minutes)
@param rp_rstar: radius of planet divided by radius of star
@param a_rstar: semimajor axis divided by radius of star
@param bins: the number of bins to average the folded curve
@param scenarios: the number of scenarios to validate
@param sigma_mode: the way to calculate the sigma for the validation ['flux_err' | 'binning']
@param contrast_curve_file: the auxiliary contrast curve file to give more information to the validation engine.
"""
save_dir = indir + "/triceratops"
if os.path.exists(save_dir):
shutil.rmtree(save_dir, ignore_errors=True)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
duration = transit_duration / 60 / 24
logging.info("----------------------")
logging.info("Validation procedures")
logging.info("----------------------")
logging.info("Pre-processing sectors")
mission, mission_prefix, id_int = LcBuilder().parse_object_info(
object_id)
if mission == "TESS":
sectors = np.array(sectors)
sectors_cut = TesscutClass().get_sectors("TIC " + str(id_int))
sectors_cut = np.array(
[sector_row["sector"] for sector_row in sectors_cut])
if len(sectors) != len(sectors_cut):
logging.warning("WARN: Some sectors were not found in TESSCUT")
logging.warning("WARN: Sherlock sectors were: " + str(sectors))
logging.warning("WARN: TESSCUT sectors were: " +
str(sectors_cut))
sectors = np.intersect1d(sectors, sectors_cut)
if len(sectors) == 0:
logging.warning(
"There are no available sectors to be validated, skipping TRICERATOPS."
)
return save_dir, None, None
logging.info("Will execute validation for sectors: " + str(sectors))
logging.info("Acquiring triceratops target")
target = tr.target(ID=id_int, mission=mission, sectors=sectors)
# TODO allow user input apertures
logging.info("Reading apertures from directory")
apertures = yaml.load(open(object_dir + "/apertures.yaml"),
yaml.SafeLoader)
apertures = apertures["sectors"]
valid_apertures = {}
for sector, aperture in apertures.items():
if sector in sectors:
valid_apertures[sector] = aperture
target.plot_field(save=True,
fname=save_dir + "/field_S" + str(sector),
sector=sector,
ap_pixels=aperture)
apertures = np.array(
[aperture for sector, aperture in apertures.items()])
valid_apertures = np.array(
[aperture for sector, aperture in valid_apertures.items()])
depth = transit_depth / 1000
if contrast_curve_file is not None:
logging.info("Reading contrast curve %s", contrast_curve_file)
plt.clf()
cc = pd.read_csv(contrast_curve_file, header=None)
sep, dmag = cc[0].values, cc[1].values
plt.plot(sep, dmag, 'k-')
plt.ylim(9, 0)
plt.ylabel("$\\Delta K_s$", fontsize=20)
plt.xlabel("separation ('')", fontsize=20)
plt.savefig(save_dir + "/contrast_curve.png")
plt.clf()
logging.info("Calculating validation closest stars depths")
target.calc_depths(depth, valid_apertures)
target.stars.to_csv(save_dir + "/stars.csv", index=False)
lc = pd.read_csv(lc_file, header=0)
time, flux, flux_err = lc["#time"].values, lc["flux"].values, lc[
"flux_err"].values
lc_len = len(time)
zeros_lc = np.zeros(lc_len)
logging.info("Preparing validation light curve for target")
if mission == "TESS":
lc = TessLightCurve(time=time,
flux=flux,
flux_err=flux_err,
quality=zeros_lc)
else:
lc = KeplerLightCurve(time=time,
flux=flux,
flux_err=flux_err,
quality=zeros_lc)
lc.extra_columns = []
fig, axs = plt.subplots(1, 1, figsize=(8, 4), constrained_layout=True)
axs, bin_centers, bin_means, bin_errs = Watson.compute_phased_values_and_fill_plot(
object_id,
axs,
lc,
period,
t0 + period / 2,
depth,
duration,
rp_rstar,
a_rstar,
bins=bins)
plt.savefig(save_dir + "/folded_curve.png")
plt.clf()
bin_centers = (bin_centers - 0.5) * period
logging.info("Sigma mode is %s", sigma_mode)
sigma = np.nanmean(
bin_errs) if sigma_mode == 'binning' else np.nanmean(flux_err)
logging.info("Computed folded curve sigma = %s", sigma)
logging.info("Preparing validation processes inputs")
input_n_times = [
ValidatorInput(save_dir, copy.deepcopy(target), bin_centers,
bin_means, sigma, period, depth, valid_apertures,
value, contrast_curve_file)
for value in range(0, scenarios)
]
logging.info("Start validation processes")
#TODO fix usage of cpus returning same value for all executions
with Pool(processes=1) as pool:
validation_results = pool.map(TriceratopsThreadValidator.validate,
input_n_times)
logging.info("Finished validation processes")
fpp_sum = 0
fpp2_sum = 0
fpp3_sum = 0
nfpp_sum = 0
probs_total_df = None
scenarios_num = len(validation_results[0][4])
star_num = np.zeros((5, scenarios_num))
u1 = np.zeros((5, scenarios_num))
u2 = np.zeros((5, scenarios_num))
fluxratio_EB = np.zeros((5, scenarios_num))
fluxratio_comp = np.zeros((5, scenarios_num))
target = input_n_times[0].target
target.star_num = np.zeros(scenarios_num)
target.u1 = np.zeros(scenarios_num)
target.u2 = np.zeros(scenarios_num)
target.fluxratio_EB = np.zeros(scenarios_num)
target.fluxratio_comp = np.zeros(scenarios_num)
logging.info("Computing final probabilities from the %s scenarios",
scenarios)
i = 0
with open(save_dir + "/validation.csv", 'w') as the_file:
the_file.write("scenario,FPP,NFPP,FPP2,FPP3+\n")
for fpp, nfpp, fpp2, fpp3, probs_df, star_num_arr, u1_arr, u2_arr, fluxratio_EB_arr, fluxratio_comp_arr \
in validation_results:
if probs_total_df is None:
probs_total_df = probs_df
else:
probs_total_df = pd.concat((probs_total_df, probs_df))
fpp_sum = fpp_sum + fpp
fpp2_sum = fpp2_sum + fpp2
fpp3_sum = fpp3_sum + fpp3
nfpp_sum = nfpp_sum + nfpp
star_num[i] = star_num_arr
u1[i] = u1_arr
u2[i] = u2_arr
fluxratio_EB[i] = fluxratio_EB_arr
fluxratio_comp[i] = fluxratio_comp_arr
the_file.write(
str(i) + "," + str(fpp) + "," + str(nfpp) + "," +
str(fpp2) + "," + str(fpp3) + "\n")
i = i + 1
for i in range(0, scenarios_num):
target.star_num[i] = np.mean(star_num[:, i])
target.u1[i] = np.mean(u1[:, i])
target.u2[i] = np.mean(u2[:, i])
target.fluxratio_EB[i] = np.mean(fluxratio_EB[:, i])
target.fluxratio_comp[i] = np.mean(fluxratio_comp[:, i])
fpp_sum = fpp_sum / scenarios
nfpp_sum = nfpp_sum / scenarios
fpp2_sum = fpp2_sum / scenarios
fpp3_sum = fpp3_sum / scenarios
logging.info("---------------------------------")
logging.info("Final probabilities computed")
logging.info("---------------------------------")
logging.info("FPP=%s", fpp_sum)
logging.info("NFPP=%s", nfpp_sum)
logging.info("FPP2(Lissauer et al, 2012)=%s", fpp2_sum)
logging.info("FPP3+(Lissauer et al, 2012)=%s", fpp3_sum)
the_file.write("MEAN" + "," + str(fpp_sum) + "," + str(nfpp_sum) +
"," + str(fpp2_sum) + "," + str(fpp3_sum))
probs_total_df = probs_total_df.groupby("scenario",
as_index=False).mean()
probs_total_df["scenario"] = pd.Categorical(
probs_total_df["scenario"], [
"TP", "EB", "EBx2P", "PTP", "PEB", "PEBx2P", "STP", "SEB",
"SEBx2P", "DTP", "DEB", "DEBx2P", "BTP", "BEB", "BEBx2P",
"NTP", "NEB", "NEBx2P"
])
probs_total_df = probs_total_df.sort_values("scenario")
probs_total_df.to_csv(save_dir + "/validation_scenarios.csv",
index=False)
target.probs = probs_total_df
# target.plot_fits(save=True, fname=save_dir + "/scenario_fits", time=lc.time.value, flux_0=lc.flux.value,
# flux_err_0=sigma)
return save_dir