def test_sff_priors():
"""SFF Spline flux mean should == lc.flux.mean()
SFF arclength component should have mean 0
"""
n_points = 300
fn = get_pkg_data_filename("../../tests/data/ep60021426alldiagnostics.csv")
data = np.genfromtxt(fn, delimiter=",", skip_header=1)
raw_flux = data[:, 1][:n_points]
centroid_col = data[:, 3][:n_points]
centroid_row = data[:, 4][:n_points]
time = np.concatenate((
np.linspace(0, 20, int(n_points / 3)),
np.linspace(30, 78, int(n_points / 3)),
np.linspace(80, 100, int(n_points / 3)),
))
lc = KeplerLightCurve(
time=time,
flux=raw_flux,
flux_err=np.ones(n_points) * 0.0001,
centroid_col=centroid_col,
centroid_row=centroid_row,
)
sff = SFFCorrector(lc)
sff.correct() # should not raise an exception
assert np.isclose(sff.diagnostic_lightcurves["spline"].flux.mean(),
1,
atol=1e-3)
assert np.isclose(sff.diagnostic_lightcurves["sff"].flux.mean(),
0,
atol=1e-3)
def test_sine_sff():
"""Can we recover a synthetic sine curve using SFF and LombScargle?"""
# Retrieve the custom, known signal properties
tpf = KeplerTargetPixelFile(filename_synthetic_sine)
true_period = np.float(tpf.hdu[3].header["PERIOD"])
true_amplitude = np.float(tpf.hdu[3].header["SINE_AMP"])
# Run the SFF algorithm
lc = tpf.to_lightcurve()
corrector = SFFCorrector(lc)
cor_lc = corrector.correct(
tpf.pos_corr2,
tpf.pos_corr1,
niters=4,
windows=1,
bins=7,
restore_trend=True,
timescale=0.5,
)
# Verify that we get the period within ~20%
pg = cor_lc.to_periodogram(
method="lombscargle", minimum_period=1, maximum_period=10, oversample_factor=10
)
ret_period = pg.period_at_max_power.value
threshold = 0.2
assert (ret_period > true_period * (1 - threshold)) & (
ret_period < true_period * (1 + threshold)
)
# Verify that we get the amplitude to within 10%
n_cad = len(tpf.time)
design_matrix = np.vstack(
[
np.ones(n_cad),
np.sin(2.0 * np.pi * cor_lc.time.value / ret_period),
np.cos(2.0 * np.pi * cor_lc.time.value / ret_period),
]
).T
ATA = np.dot(design_matrix.T, design_matrix / cor_lc.flux_err[:, None] ** 2)
least_squares_coeffs = np.linalg.solve(
ATA, np.dot(design_matrix.T, cor_lc.flux / cor_lc.flux_err ** 2)
)
const, sin_weight, cos_weight = least_squares_coeffs
fractional_amplitude = (sin_weight ** 2 + cos_weight ** 2) ** (0.5) / const
assert (fractional_amplitude > true_amplitude / 1.1) & (
fractional_amplitude < true_amplitude * 1.1
)
def test_sff_knots():
"""Is SFF robust against gaps in time and irregular time sampling?
This test creates a light curve with gaps in time between
days 20-30 and days 78-80. In addition, the time sampling rate changes
in the interval between day 30 and 78. SFF should fail without error.
"""
n_points = 300
fn = get_pkg_data_filename("../../tests/data/ep60021426alldiagnostics.csv")
data = np.genfromtxt(fn, delimiter=",", skip_header=1)
raw_flux = data[:, 1][:n_points]
centroid_col = data[:, 3][:n_points]
centroid_row = data[:, 4][:n_points]
time = np.concatenate((
np.linspace(0, 20, int(n_points / 3)),
np.linspace(30, 78, int(n_points / 3)),
np.linspace(80, 100, int(n_points / 3)),
))
lc = KeplerLightCurve(
time=time,
flux=raw_flux,
flux_err=np.ones(n_points) * 0.0001,
centroid_col=centroid_col,
centroid_row=centroid_row,
)
# These calls should not raise an exception:
SFFCorrector(lc).correct()
lc.to_corrector(method="sff").correct()
def test_transit_sff():
"""Can we recover a synthetic exoplanet signal using SFF and BLS?"""
# Retrieve the custom, known signal properties
tpf = KeplerTargetPixelFile(filename_synthetic_transit)
true_period = np.float(tpf.hdu[3].header["PERIOD"])
true_rprs = np.float(tpf.hdu[3].header["RPRS"])
true_transit_lc = tpf.hdu[3].data["NOISELESS_INPUT"]
max_depth = 1 - np.min(true_transit_lc)
# Run the SFF algorithm
lc = tpf.to_lightcurve().normalize()
corrector = SFFCorrector(lc)
cor_lc = corrector.correct(
tpf.pos_corr2,
tpf.pos_corr1,
niters=4,
windows=1,
bins=7,
restore_trend=False,
timescale=0.5,
)
# Verify that we get the transit period within 5%
pg = cor_lc.to_periodogram(
method="bls",
minimum_period=1,
maximum_period=9,
frequency_factor=0.05,
duration=np.arange(0.1, 0.6, 0.1),
)
ret_period = pg.period_at_max_power.value
threshold = 0.05
assert (ret_period > true_period * (1 - threshold)) & (
ret_period < true_period * (1 + threshold)
)
# Verify that we get the transit depth in expected bounds
assert (pg.depth_at_max_power >= true_rprs ** 2) & (
pg.depth_at_max_power < max_depth
)
def test_detrending_residuals():
"""Test the detrending residual distributions"""
# Retrieve the custom, known signal properties
tpf = KeplerTargetPixelFile(filename_synthetic_flat)
# Run the SFF algorithm
lc = tpf.to_lightcurve()
corrector = SFFCorrector(lc)
cor_lc = corrector.correct(
tpf.pos_corr2, tpf.pos_corr1, niters=10, windows=5, bins=7, restore_trend=True
)
# Verify that we get a significant reduction in RMS
cdpp_improvement = lc.estimate_cdpp() / cor_lc.estimate_cdpp()
assert cdpp_improvement > 10.0
# The residuals should be Gaussian-"ish"
# Table 4.1 of Ivezic, Connolly, Vanerplas, Gray 2014
anderson_threshold = 1.57
resid_n_sigmas = (cor_lc.flux - np.mean(cor_lc.flux)) / cor_lc.flux_err
A_value, _, _ = stats.anderson(resid_n_sigmas)
assert A_value ** 2 < anderson_threshold
n_sigma = np.std(resid_n_sigmas)
assert n_sigma < 2.0
corrector = tpf.to_corrector("pld")
cor_lc = corrector.correct(restore_trend=False)
cdpp_improvement = lc.estimate_cdpp() / cor_lc.estimate_cdpp()
assert cdpp_improvement > 10.0
resid_n_sigmas = (cor_lc.flux - np.mean(cor_lc.flux)) / cor_lc.flux_err
A_value, crit, sig = stats.anderson(resid_n_sigmas)
assert A_value ** 2 < anderson_threshold
n_sigma = np.std(resid_n_sigmas)
assert n_sigma < 2.0
def test_sff_breakindex():
"""Regression test for #616."""
lc = LightCurve(flux=np.ones(20))
with warnings.catch_warnings():
# Ignore "LightkurveWarning: The design matrix has low rank".
warnings.simplefilter("ignore", LightkurveWarning)
corr = SFFCorrector(lc)
corr.correct(
breakindex=[5, 10],
centroid_col=np.random.randn(20),
centroid_row=np.random.randn(20),
)
assert 5 in corr.window_points
assert 10 in corr.window_points
corr.correct(
breakindex=[5, 10],
centroid_col=np.random.randn(20),
centroid_row=np.random.randn(20),
windows=1,
)
assert_array_equal(corr.window_points, np.asarray([5, 10]))
def build(self, object_info, sherlock_dir, caches_root_dir):
mission_id = object_info.mission_id()
sherlock_id = object_info.sherlock_id()
logging.info("Retrieving star catalog info...")
mission, mission_prefix, id = super().parse_object_id(mission_id)
cadence = object_info.cadence if object_info.cadence is not None else "long"
author = object_info.author if object_info.author is not None else self.authors[
mission]
transits_min_count = 1
star_info = None
if mission_prefix not in self.star_catalogs:
raise ValueError("Wrong object id " + mission_id)
sectors = None if object_info.sectors == 'all' or mission != constants.MISSION_TESS else object_info.sectors
campaigns = None if object_info.sectors == 'all' or mission != constants.MISSION_K2 else object_info.sectors
quarters = None if object_info.sectors == 'all' or mission != constants.MISSION_KEPLER else object_info.sectors
apertures = {}
tpf_search_results = lk.search_targetpixelfile(str(mission_id))
for tpf_search_result in tpf_search_results:
logging.info(
"There is data for Mission: %s, Year %.0f, Author: %s, ExpTime: %.0f",
tpf_search_result.mission[0], tpf_search_result.year[0],
tpf_search_result.author[0],
tpf_search_result.exptime[0].value)
tpfs_dir = sherlock_dir + "/tpfs/"
if not os.path.exists(tpfs_dir):
os.mkdir(tpfs_dir)
if mission_prefix == self.MISSION_ID_KEPLER or mission_prefix == self.MISSION_ID_KEPLER_2:
source = "tpf"
lcf_search_results = lk.search_lightcurvefile(str(mission_id),
mission=mission,
cadence=cadence,
author=author,
sector=sectors,
quarter=quarters,
campaign=campaigns)
lcf = lcf_search_results.download_all(
download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR)
tpfs = lk.search_targetpixelfile(str(mission_id), mission=mission, cadence=cadence,
sector=sectors, quarter=quarters,
campaign=campaigns, author=author)\
.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR,
cutout_size=(CUTOUT_SIZE, CUTOUT_SIZE))
lc_data = self.extract_lc_data(lcf)
lc = lcf.PDCSAP_FLUX.stitch().remove_nans()
transits_min_count = 1 if len(lcf) == 0 else 2
if mission_prefix == self.MISSION_ID_KEPLER:
sectors = [lcfile.quarter for lcfile in lcf]
elif mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
sectors = [lcfile.campaign for lcfile in lcf]
lc = SFFCorrector(lc).correct(windows=20)
if not os.path.exists(tpfs_dir):
os.mkdir(tpfs_dir)
for tpf in tpfs:
shutil.copy(tpf.path, tpfs_dir + os.path.basename(tpf.path))
if mission_prefix == self.MISSION_ID_KEPLER:
sector = tpf.quarter
elif mission_prefix == self.MISSION_ID_KEPLER_2:
sector = tpf.campaign
apertures[sector] = ApertureExtractor.from_boolean_mask(
tpf.pipeline_mask, tpf.column, tpf.row)
star_info = starinfo.StarInfo(
sherlock_id,
*self.star_catalogs[mission_prefix].catalog_info(id))
else:
source = "eleanor"
if isinstance(object_info, MissionFfiCoordsObjectInfo):
coords = SkyCoord(ra=object_info.ra,
dec=object_info.dec,
unit=(u.deg, u.deg))
star = eleanor.source.multi_sectors(
coords=coords,
sectors=object_info.sectors,
post_dir=caches_root_dir + ELEANOR_CACHE_DIR,
metadata_path=caches_root_dir + ELEANOR_CACHE_DIR)
else:
object_id_parsed = re.search(super().NUMBERS_REGEX,
object_info.id)
object_id_parsed = object_info.id[
object_id_parsed.regs[0][0]:object_id_parsed.regs[0][1]]
star = eleanor.multi_sectors(
tic=object_id_parsed,
sectors=object_info.sectors,
post_dir=caches_root_dir + ELEANOR_CACHE_DIR,
metadata_path=caches_root_dir + ELEANOR_CACHE_DIR)
if star is None:
raise ValueError("No data for this object")
if star[0].tic:
# TODO FIX star info objectid
logging.info("Assotiated TIC is " + str(star[0].tic))
tpfs = lk.search_targetpixelfile("TIC " + str(star[0].tic), mission=constants.MISSION_TESS,
cadence=cadence, sector=sectors, quarter=quarters,
campaign=campaigns, author="TESS-SPOC") \
.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR,
cutout_size=(CUTOUT_SIZE, CUTOUT_SIZE))
star_info = starinfo.StarInfo(
object_info.sherlock_id(),
*self.star_catalog.catalog_info(int(star[0].tic)))
data = []
for s in star:
datum = TargetData(s,
height=CUTOUT_SIZE,
width=CUTOUT_SIZE,
do_pca=True)
data.append(datum)
for tpf in tpfs:
if tpf.sector == s.sector:
shutil.copy(tpf.path,
tpfs_dir + os.path.basename(tpf.path))
apertures[
s.sector] = ApertureExtractor.from_boolean_mask(
datum.aperture.astype(bool), tpf.column,
tpf.row)
quality_bitmask = np.bitwise_and(data[0].quality.astype(int), 175)
lc_data = self.extract_eleanor_lc_data(data)
lc = data[0].to_lightkurve(
data[0].__dict__[object_info.eleanor_corr_flux],
quality_mask=quality_bitmask).remove_nans().flatten()
sectors = [datum.source_info.sector for datum in data]
if len(data) > 1:
for datum in data[1:]:
quality_bitmask = np.bitwise_and(datum.quality, 175)
lc = lc.append(
datum.to_lightkurve(datum.pca_flux,
quality_mask=quality_bitmask).
remove_nans().flatten())
transits_min_count = 2
return LcBuild(lc, lc_data, star_info, transits_min_count, cadence,
None, sectors, source, apertures)
def test_sff_corrector():
"""Does our code agree with the example presented in Vanderburg
and Johnson (2014)?"""
# The following csv file, provided by Vanderburg and Johnson
# at https://www.cfa.harvard.edu/~avanderb/k2/ep60021426.html,
# contains the results of applying SFF to EPIC 60021426.
fn = get_pkg_data_filename("../../tests/data/ep60021426alldiagnostics.csv")
data = np.genfromtxt(fn, delimiter=",", skip_header=1)
mask = data[:, -2] == 0 # indicates whether the thrusters were on or off
time = data[:, 0]
raw_flux = data[:, 1]
corrected_flux = data[:, 2]
centroid_col = data[:, 3]
centroid_row = data[:, 4]
# NOTE: we need a small number of windows below because this test data set
# is unusually short, i.e. has an unusually small number of cadences.
lc = LightCurve(time=time,
flux=raw_flux,
flux_err=np.ones(len(raw_flux)) * 0.0001)
sff = SFFCorrector(lc)
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
restore_trend=True,
windows=1,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
assert len(sff.window_points) == 0 # expect 0 break points for 1 window
# masking
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
windows=3,
restore_trend=True,
cadence_mask=mask,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
assert len(sff.window_points) == 2 # expect 2 break points for 3 windows
# masking and breakindex
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
windows=3,
restore_trend=True,
cadence_mask=mask,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
# masking and breakindex and iters
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
windows=3,
restore_trend=True,
cadence_mask=mask,
niters=3,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
# masking and breakindex and bins
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
windows=3,
restore_trend=True,
cadence_mask=mask,
bins=5,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
assert np.all((sff.lc.flux_err / sff.corrected_lc.flux_err) == 1)
# masking and breakindex and bins and propagate_errors
corrected_lc = sff.correct(
centroid_col=centroid_col,
centroid_row=centroid_row,
windows=3,
restore_trend=True,
cadence_mask=mask,
bins=5,
propagate_errors=True,
)
assert np.isclose(corrected_flux, corrected_lc.flux, atol=0.001).all()
assert np.all((sff.lc.flux_err / sff.corrected_lc.flux_err) < 1)
# test using KeplerLightCurve interface
klc = KeplerLightCurve(
time=time,
flux=raw_flux,
flux_err=np.ones(len(raw_flux)) * 0.0001,
centroid_col=centroid_col,
centroid_row=centroid_row,
)
sff = klc.to_corrector("sff")
klc = sff.correct(windows=3, restore_trend=True)
assert np.isclose(corrected_flux, klc.flux, atol=0.001).all()
# Can plot
sff.diagnose()
def test_remote_data(path):
"""Can we correct a simple K2 light curve?"""
lc = KeplerLightCurve.read(path, quality_bitmask=None)
sff = SFFCorrector(lc.remove_nans())
sff.correct(windows=10, bins=5, timescale=0.5)
sff.correct(windows=10, bins=5, timescale=0.5, sparse=True)
def test_sff_tess_warning():
"""SFF is not designed for TESS, so we raise a warning."""
lc = TessLightCurve(flux=[1, 2, 3], meta={"MISSION": "TESS"})
with pytest.warns(LightkurveWarning, match="not suitable"):
corr = SFFCorrector(lc)