def test_setup_detrend(self):
object_info = MissionObjectInfo("TIC 12345",
sectors="all",
smooth_enabled=False,
high_rms_enabled=False,
high_rms_threshold=3,
high_rms_bin_hours=9,
auto_detrend_enabled=True,
auto_detrend_ratio=1 / 2,
auto_detrend_method="cosine")
sherlock = Sherlock([
SherlockTarget(object_info=object_info,
detrends_number=2,
detrend_method="gp",
cpu_cores=3)
])
self.assertEqual(
False, sherlock.sherlock_targets[0].object_info.smooth_enabled)
self.assertEqual(
False, sherlock.sherlock_targets[0].object_info.high_rms_enabled)
self.assertEqual(
3, sherlock.sherlock_targets[0].object_info.high_rms_threshold)
self.assertEqual(
9, sherlock.sherlock_targets[0].object_info.high_rms_bin_hours)
self.assertEqual(2, sherlock.sherlock_targets[0].detrends_number)
self.assertEqual("gp", sherlock.sherlock_targets[0].detrend_method)
self.assertEqual(3, sherlock.sherlock_targets[0].cpu_cores)
self.assertEqual(
True,
sherlock.sherlock_targets[0].object_info.auto_detrend_enabled)
self.assertEqual(
1 / 2, sherlock.sherlock_targets[0].object_info.auto_detrend_ratio)
self.assertEqual(
"cosine",
sherlock.sherlock_targets[0].object_info.auto_detrend_method)
def test_build(self):
lc_build = LcBuilder().build(MissionObjectInfo("TIC 352315023", [13]), "./")
self.assertEqual(18140, len(lc_build.lc))
self.assertEqual(20479, len(lc_build.lc_data))
lc_build = LcBuilder().build(MissionObjectInfo("KIC 12557548", [13]), "./")
self.assertEqual(126304, len(lc_build.lc))
self.assertEqual(130290, len(lc_build.lc_data))
lc_build = LcBuilder().build(MissionObjectInfo("EPIC 211945201", 'all'), "./")
self.assertEqual(106384, len(lc_build.lc))
self.assertEqual(116820, len(lc_build.lc_data))
lc_build = LcBuilder().build(MissionFfiIdObjectInfo("KIC 12557548", [1]), "./")
self.assertEqual(1543, len(lc_build.lc))
self.assertEqual(1639, len(lc_build.lc_data))
lc_build = LcBuilder().build(MissionFfiIdObjectInfo("EPIC 211945201", [5]), "./")
self.assertEqual(3271, len(lc_build.lc))
self.assertEqual(3663, len(lc_build.lc_data))
lc_build = LcBuilder().build(MissionFfiIdObjectInfo("TIC 352315023", [13]), "./")
self.assertEqual(1223, len(lc_build.lc))
self.assertEqual(1320, len(lc_build.lc_data))
def inject(self, phases, min_period, max_period, step_period, min_radius,
max_radius, step_radius):
#TODO get mission and proper id and author
object_info = MissionObjectInfo(self.id, self.sectors)
lc, lc_data, star_info, transits_min_count, sectors, quarters = \
MissionLightcurveBuilder().build(object_info, None)
ab, mass, massmin, massmax, radius, radiusmin, radiusmax = catalog_info(
TIC_ID=id)
# units for ellc
rstar = star_info.radius * u.R_sun
mstar = star_info.mass * u.M_sun
mstar_min = star_info.mass_min * u.M_sun
mstar_max = star_info.mass_max * u.M_sun
rstar_min = star_info.radius_min * u.R_sun
rstar_max = star_info.radius_max * u.R_sun
print('\n STELLAR PROPERTIES FOR THE SIGNAL SEARCH')
print('================================================\n')
print('limb-darkening estimates using quadratic LD (a,b)=', ab)
print('mass =', format(mstar, '0.5f'))
print('mass_min =', format(mstar_min, '0.5f'))
print('mass_max =', format(mstar_max, '0.5f'))
print('radius =', format(rstar, '0.5f'))
print('radius_min =', format(rstar_min, '0.5f'))
print('radius_max =', format(rstar_max, '0.5f'))
lc_new = lk.LightCurve(time=lc.time,
flux=lc.flux,
flux_err=lc.flux_err)
clean = lc_new.remove_outliers(sigma_lower=float('inf'), sigma_upper=3)
flux0 = clean.flux
time = clean.time
flux_err = clean.flux_err
if not os.path.isdir(self.dir):
os.mkdir(self.dir)
for period in np.arange(min_period, max_period, step_period):
for t0 in np.arange(time[60], time[60] + period - 0.1,
period / phases):
for rplanet in np.arange(min_radius, max_radius, step_radius):
rplanet = np.around(rplanet, decimals=2) * u.R_earth
print('\n')
print('P = ' + str(period) + ' days, Rp = ' +
str(rplanet) + ", T0 = " + str(t0))
time_model, flux_model, flux_err_model = self.__make_model(
time, flux0, flux_err, rstar, mstar, t0, period,
rplanet)
file_name = os.path.join(self.dir + '/P' + str(period) +
'_R' + str(rplanet.value) + '_' +
str(t0) + '.csv')
lc_df = pd.DataFrame(columns=['#time', 'flux', 'flux_err'])
lc_df['#time'] = time_model
lc_df['flux'] = flux_model
lc_df['flux_err'] = flux_err_model
lc_df.to_csv(file_name, index=False)
def test_run_with_autodetrend(self):
run_dir = None
try:
Sherlock([
SherlockTarget(MissionObjectInfo("TIC 259377017", [5],
auto_detrend_enabled=True),
detrends_number=1,
max_runs=1,
oversampling=0.05)
], True).run()
run_dir = "TIC259377017_[5]"
self.assertTrue(os.path.exists(run_dir))
self.assertTrue(
os.path.exists(
run_dir +
"/Phase_detrend_period_TIC259377017_[5]_0.52_days.png"))
finally:
self.__clean(run_dir)
def test_short_cadence_epic(self):
lc_build = LcBuilder().build(MissionObjectInfo("EPIC 211945201", 'all'), "./")
self.assertEqual(lc_build.cadence, 59)
self.assertGreater(len(lc_build.lc), 0)
self.__test_k2_star_params(lc_build.star_info)
def test_short_cadence_kic(self):
lc_build = LcBuilder().build(MissionObjectInfo("KIC 12557548", 'all'), "./")
self.assertEqual(lc_build.cadence, 59)
self.assertGreater(len(lc_build.lc), 0)
self.__test_kepler_star_params(lc_build.star_info)
def test_short_cadence(self):
lc_build = LcBuilder().build(MissionObjectInfo("TIC 352315023", 'all'), "./")
self.assertEqual(lc_build.cadence, 120)
self.assertGreater(len(lc_build.lc), 0)
self.__test_tess_star_params(lc_build.star_info)
# 'auto_detrend_ratio' value, which ensures that we are detrending the light curve at 'auto_detrend_ratio' times
# the stronger period.
# 7 Set the maximum number of runs to be executed.
# 8 Select the period protect value, which restricts the minimum detrending window length.
# 9 Select the min period for a transit to be fit.
# 10 Select the max period for a transit to be fit.
# 11 Select the binning to calculate RMS
# 12 Select the number of CPU cores to be used for the transit search.
# 13 Select the found transits masking method. We use subtract here as example, but it is discouraged.
# 14 Select the best signal algorithm, which provides a different implementation to decide which of the detrend
# signals is the stronger one to be selected.
# 15 Set the strength of the quorum algorithm votes, which makes every vote that is found to increase the SNR by
# a factor of 1.2 for our selection.
arguments = {"smoooth_enabled": True, "high_rms_enabled": True, "high_rms_threshold": 2.5,
"high_rms_bin_hours": 3,
"detrends_number": 12, "detrend_method": "gp", "cpu_cores": 2, "auto_detrend_enabled": True,
"auto_detrend_ratio": 0.33, "auto_detrend_method": "cosine",
"max_runs": 10, "period_protect": 12, "period_min": 1, "period_max": 10, "bin_minutes": 20,
"run_cores": 3, "snr_min": 6, "sde_min": 6, "mask_mode": "subtract",
"best_signal_algorithm": 'quorum', "quorum_strength": 1.2}
sherlock = Sherlock([SherlockTarget(MissionFfiIdObjectInfo("TIC 181804752", 'all'), **arguments),
SherlockTarget(MissionObjectInfo("TIC 259168516", [15]), **arguments),
SherlockTarget(MissionObjectInfo('KIC 10905746', 'all'), **arguments),
SherlockTarget(MissionObjectInfo('EPIC 249631677', 'all'), **arguments),
SherlockTarget(MissionInputObjectInfo("TIC 181804752", 'example_lc.csv',
initial_mask=[[1625, 1626], [1645, 1646]]), **arguments),
SherlockTarget(InputObjectInfo("example_lc.csv", initial_detrend_period=0.8), **arguments)]) \
.run()
print("Analysis took " + elapsed() + "s")
def build(self, object_info: MissionObjectInfo, sherlock_dir, caches_root_dir):
mission_id = object_info.mission_id()
sherlock_id = object_info.sherlock_id()
logging.info("Retrieving star catalog info...")
mission, mission_prefix, id = super().parse_object_id(mission_id)
if mission_prefix not in self.star_catalogs:
raise ValueError("Wrong object id " + mission_id)
cadence = object_info.cadence if object_info.cadence is not None else "short"
author = object_info.author if object_info.author is not None else self.authors[mission]
star_info = starinfo.StarInfo(sherlock_id, *self.star_catalogs[mission_prefix].catalog_info(id))
logging.info("Downloading lightcurve files...")
sectors = None if object_info.sectors == 'all' or mission != constants.MISSION_TESS else object_info.sectors
campaigns = None if object_info.sectors == 'all' or mission != constants.MISSION_K2 else object_info.sectors
quarters = None if object_info.sectors == 'all' or mission != constants.MISSION_KEPLER else object_info.sectors
tokens = sectors if sectors is not None else campaigns if campaigns is not None else quarters
tokens = tokens if tokens is not None else "all"
apertures = {}
tpf_search_results = lk.search_targetpixelfile(str(mission_id))
for tpf_search_result in tpf_search_results:
logging.info("There is data for Mission: %s, Year %.0f, Author: %s, ExpTime: %.0f",
tpf_search_result.mission[0], tpf_search_result.year[0], tpf_search_result.author[0],
tpf_search_result.exptime[0].value)
tpfs_dir = sherlock_dir + "/tpfs/"
if not os.path.exists(tpfs_dir):
os.mkdir(tpfs_dir)
if object_info.apertures is None:
lcf_search_results = lk.search_lightcurve(str(mission_id), mission=mission, cadence=cadence,
sector=sectors, quarter=quarters,
campaign=campaigns, author=author)
lcf = lcf_search_results.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR)
tpfs = lk.search_targetpixelfile(str(mission_id), mission=mission, cadence=cadence,
sector=sectors, quarter=quarters,
campaign=campaigns, author=author)\
.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR,
cutout_size=(CUTOUT_SIZE, CUTOUT_SIZE))
if lcf is None:
raise ObjectProcessingError("The target " + str(mission_id) + " is not available for the author " + author +
", cadence " + str(cadence) + "s and sectors " + str(tokens))
lc_data = self.extract_lc_data(lcf)
lc = None
matching_objects = []
for tpf in tpfs:
shutil.copy(tpf.path, tpfs_dir + os.path.basename(tpf.path))
if mission_prefix == self.MISSION_ID_KEPLER:
sector = tpf.quarter
elif mission_prefix == self.MISSION_ID_TESS:
sector = tpf.sector
if mission_prefix == self.MISSION_ID_KEPLER_2:
sector = tpf.campaign
apertures[sector] = ApertureExtractor.from_boolean_mask(tpf.pipeline_mask, tpf.column, tpf.row)
for i in range(0, len(lcf.data)):
if lcf.data[i].label == mission_id:
if lc is None:
lc = lcf.data[i].normalize()
else:
lc = lc.append(lcf.data[i].normalize())
else:
matching_objects.append(lcf.data[i].label)
matching_objects = set(matching_objects)
if len(matching_objects) > 0:
logging.warning("================================================")
logging.warning("TICS IN THE SAME PIXEL: " + str(matching_objects))
logging.warning("================================================")
if lc is None:
tokens = sectors if sectors is not None else campaigns if campaigns is not None else quarters
tokens = tokens if tokens is not None else "all"
raise ObjectProcessingError("The target " + str(mission_id) + " is not available for the author " + author +
", cadence " + str(cadence) + "s and sectors " + str(tokens))
lc = lc.remove_nans()
transits_min_count = self.__calculate_transits_min_count(len(lcf))
if mission_prefix == self.MISSION_ID_KEPLER:
sectors = [lcfile.quarter for lcfile in lcf]
elif mission_prefix == self.MISSION_ID_TESS:
sectors = [file.sector for file in lcf]
elif mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
sectors = [lcfile.campaign for lcfile in lcf]
lc = lc.to_corrector("sff").correct(windows=20)
source = "tpf"
else:
logging.info("Using user apertures!")
tpf_search_results = lk.search_targetpixelfile(str(mission_id), mission=mission, cadence=cadence,
sector=sectors, quarter=quarters, campaign=campaigns,
author=author)
tpfs = tpf_search_results.download_all(download_dir=caches_root_dir + LIGHTKURVE_CACHE_DIR,
cutout_size=(CUTOUT_SIZE, CUTOUT_SIZE))
source = "tpf"
apertures = object_info.apertures
lc = None
for tpf in tpfs:
shutil.copy(tpf.path, tpfs_dir + os.path.basename(tpf.path))
if mission_prefix == self.MISSION_ID_KEPLER:
sector = tpf.quarter
elif mission_prefix == self.MISSION_ID_TESS:
sector = tpf.sector
elif mission_prefix == self.MISSION_ID_KEPLER_2:
sector = tpf.campaign
boolean_aperture = ApertureExtractor.from_pixels_to_boolean_mask(apertures[sector], tpf.column, tpf.row,
CUTOUT_SIZE, CUTOUT_SIZE)
tpf.plot(aperture_mask=boolean_aperture, mask_color='red')
plt.savefig(sherlock_dir + "/fov/Aperture_[" + str(sector) + "].png")
plt.close()
if mission_prefix == self.MISSION_ID_KEPLER:
corrector = lk.KeplerCBVCorrector(tpf)
corrector.plot_cbvs([1, 2, 3, 4, 5, 6, 7])
raw_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture).remove_nans()
plt.savefig(sherlock_dir + "/Corrector_components[" + str(sector) + "].png")
plt.close()
it_lc = corrector.correct([1, 2, 3, 4, 5])
ax = raw_lc.plot(color='C3', label='SAP Flux', linestyle='-')
it_lc.plot(ax=ax, color='C2', label='CBV Corrected SAP Flux', linestyle='-')
plt.savefig(sherlock_dir + "/Raw_vs_CBVcorrected_lc[" + str(sector) + "].png")
plt.close()
elif mission_prefix == self.MISSION_ID_KEPLER_2:
raw_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture).remove_nans()
it_lc = raw_lc.to_corrector("sff").correct(windows=20)
ax = raw_lc.plot(color='C3', label='SAP Flux', linestyle='-')
it_lc.plot(ax=ax, color='C2', label='CBV Corrected SAP Flux', linestyle='-')
plt.savefig(sherlock_dir + "/Raw_vs_SFFcorrected_lc[" + str(sector) + "].png")
plt.close()
elif mission_prefix == self.MISSION_ID_TESS:
temp_lc = tpf.to_lightcurve(aperture_mask=boolean_aperture)
where_are_NaNs = np.isnan(temp_lc.flux)
temp_lc = temp_lc[np.where(~where_are_NaNs)]
regressors = tpf.flux[np.argwhere(~where_are_NaNs), ~boolean_aperture]
temp_token_lc = [temp_lc[i: i + 2000] for i in range(0, len(temp_lc), 2000)]
regressors_token = [regressors[i: i + 2000] for i in range(0, len(regressors), 2000)]
it_lc = None
raw_it_lc = None
item_index = 0
for temp_token_lc_item in temp_token_lc:
regressors_token_item = regressors_token[item_index]
design_matrix = lk.DesignMatrix(regressors_token_item, name='regressors').pca(5).append_constant()
corr_lc = lk.RegressionCorrector(temp_token_lc_item).correct(design_matrix)
if it_lc is None:
it_lc = corr_lc
raw_it_lc = temp_token_lc_item
else:
it_lc = it_lc.append(corr_lc)
raw_it_lc = raw_it_lc.append(temp_token_lc_item)
item_index = item_index + 1
ax = raw_it_lc.plot(label='Raw light curve')
it_lc.plot(ax=ax, label='Corrected light curve')
plt.savefig(sherlock_dir + "/Raw_vs_DMcorrected_lc[" + str(sector) + "].png")
plt.close()
if lc is None:
lc = it_lc.normalize()
else:
lc = lc.append(it_lc.normalize())
lc = lc.remove_nans()
lc.plot(label="Normalized light curve")
plt.savefig(sherlock_dir + "/Normalized_lc[" + str(sector) + "].png")
plt.close()
transits_min_count = self.__calculate_transits_min_count(len(tpfs))
if mission_prefix == self.MISSION_ID_KEPLER or mission_id == self.MISSION_ID_KEPLER_2:
sectors = [lcfile.quarter for lcfile in tpfs]
elif mission_prefix == self.MISSION_ID_TESS:
sectors = [file.sector for file in tpfs]
if mission_prefix == self.MISSION_ID_KEPLER_2:
logging.info("Correcting K2 motion in light curve...")
sectors = [lcfile.campaign for lcfile in tpfs]
sectors = None if sectors is None else np.unique(sectors)
lc_data = None
return LcBuild(lc, lc_data, star_info, transits_min_count, cadence, None, sectors, source, apertures)
from contextlib import contextmanager
from timeit import default_timer
from sherlockpipe.sherlock import Sherlock
from lcbuilder.objectinfo.MissionObjectInfo import MissionObjectInfo
from sherlockpipe.sherlock_target import SherlockTarget
@contextmanager
def elapsed_timer():
start = default_timer()
elapser = lambda: str(default_timer() - start)
yield lambda: elapser()
end = default_timer()
elapser = lambda: str(end - start)
with elapsed_timer() as elapsed:
# We will use only one mission object and will mask a time range from 1550 to 1551 and from 1560 to 1561
sherlock = Sherlock([SherlockTarget(MissionObjectInfo("TIC 181804752", 'all',
initial_mask=[[1550, 1551], [1560, 1561]]))])\
.run()
print("Analysis took " + elapsed() + "s")
@contextmanager
def elapsed_timer():
start = default_timer()
elapser = lambda: str(default_timer() - start)
yield lambda: elapser()
end = default_timer()
elapser = lambda: str(end - start)
with elapsed_timer() as elapsed:
# We will use only one object id so we can explain better the detrend configs that the coder can select
# We will:
# 1 Enable the initial smooth function, which reduces local noise in the signal.
# 2 Enable the initial High RMS areas masking. This procedure will mask all the lightcurve time binned ranges by
# the 'initial_rms_bin_hours' value with a threshold of the 'initial_rms_threshold' value * RMS_median.
# 3 Set the number of detrends to be done from PDCSAP_FLUX for each run.
# 4 Set the SHERLOCK PDCSAP_FLUX detrends method to Gaussian Processes.
# 5 Set the number of CPU cores to be used by the detrending procedure.
# 6 Enable the Auto-Detrend detection, which will search for strong periodicities in the light curve and do an
# initial detrend for it based on the selected 'auto_detrend_method' method and the value of the
# 'auto_detrend_ratio' value, which ensures that we are detrending the light curve at 'auto_detrend_ratio' times
# the stronger period.
sherlock = Sherlock([SherlockTarget(object_info=MissionObjectInfo("TIC 181804752", 'all'),
smooth_enabled=True, high_rms_enabled=True, high_rms_threshold=2.5, high_rms_bin_hours=3,
detrends_number=12, detrend_method="gp", cpu_cores=2,
auto_detrend_enabled=True, auto_detrend_ratio=0.33,
auto_detrend_method="cosine")]) \
.run()
print("Analysis took " + elapsed() + "s")
from sherlockpipe.sherlock import Sherlock
from lcbuilder.objectinfo.InputObjectInfo import InputObjectInfo
from lcbuilder.objectinfo.MissionFfiCoordsObjectInfo import MissionFfiCoordsObjectInfo
from lcbuilder.objectinfo.MissionFfiIdObjectInfo import MissionFfiIdObjectInfo
from lcbuilder.objectinfo.MissionInputObjectInfo import MissionInputObjectInfo
from lcbuilder.objectinfo.MissionObjectInfo import MissionObjectInfo
from sherlockpipe.sherlock_target import SherlockTarget
@contextmanager
def elapsed_timer():
start = default_timer()
elapser = lambda: str(default_timer() - start)
yield lambda: elapser()
end = default_timer()
elapser = lambda: str(end - start)
with elapsed_timer() as elapsed:
# Adding several kinds of objects to the run: one short cadence TIC, one FFI TIC, one coordinates FFI, one input
# file related to a TIC and one plain input file.
# Ensure that your input light curve CSV files have three columns: #TBJD,flux,flux_err
sherlock = Sherlock([SherlockTarget(MissionObjectInfo("TIC 181804752", 'all')),
SherlockTarget(MissionFfiIdObjectInfo("TIC 259168516", [14, 15])),
SherlockTarget(MissionFfiCoordsObjectInfo(14, 19, 'all')),
SherlockTarget(MissionInputObjectInfo("TIC 470381900", "example_lightcurve.csv")),
SherlockTarget(InputObjectInfo("example_lc.csv"))])\
.run()
print("Analysis took " + elapsed() + "s")
def build_object_info(self,
target_name,
author,
sectors,
file,
cadence,
initial_mask,
initial_transit_mask,
star_info,
aperture,
eleanor_corr_flux='pca_flux',
outliers_sigma=None,
high_rms_enabled=True,
high_rms_threshold=2.5,
high_rms_bin_hours=4,
smooth_enabled=False,
auto_detrend_enabled=False,
auto_detrend_method="cosine",
auto_detrend_ratio=0.25,
auto_detrend_period=None,
prepare_algorithm=None,
reduce_simple_oscillations=False,
oscillation_snr_threshold=4,
oscillation_amplitude_threshold=0.1,
oscillation_ws_scale=60,
oscillation_min_period=0.002,
oscillation_max_period=0.2,
binning=1):
mission, mission_prefix, id = MissionLightcurveBuilder(
).parse_object_id(target_name)
coords = None if mission is not None else self.parse_coords(
target_name)
cadence = cadence if cadence is not None else self.DEFAULT_CADENCES_FOR_MISSION[
mission]
if mission is not None and file is None and cadence <= 300:
return MissionObjectInfo(
target_name, sectors, author, cadence, initial_mask,
initial_transit_mask, star_info, aperture, outliers_sigma,
high_rms_enabled, high_rms_threshold, high_rms_bin_hours,
smooth_enabled, auto_detrend_enabled, auto_detrend_method,
auto_detrend_ratio, auto_detrend_period, prepare_algorithm,
reduce_simple_oscillations, oscillation_snr_threshold,
oscillation_amplitude_threshold, oscillation_ws_scale,
oscillation_min_period, oscillation_max_period, binning)
elif mission is not None and file is None and cadence > 300:
return MissionFfiIdObjectInfo(
target_name, sectors, author, cadence, initial_mask,
initial_transit_mask, star_info, aperture, eleanor_corr_flux,
outliers_sigma, high_rms_enabled, high_rms_threshold,
high_rms_bin_hours, smooth_enabled, auto_detrend_enabled,
auto_detrend_method, auto_detrend_ratio, auto_detrend_period,
prepare_algorithm, reduce_simple_oscillations,
oscillation_snr_threshold, oscillation_amplitude_threshold,
oscillation_ws_scale, oscillation_min_period,
oscillation_max_period, binning)
elif mission is not None and file is not None:
return MissionInputObjectInfo(
target_name, file, initial_mask, initial_transit_mask,
star_info, outliers_sigma, high_rms_enabled,
high_rms_threshold, high_rms_bin_hours, smooth_enabled,
auto_detrend_enabled, auto_detrend_method, auto_detrend_ratio,
auto_detrend_period, prepare_algorithm,
reduce_simple_oscillations, oscillation_snr_threshold,
oscillation_amplitude_threshold, oscillation_ws_scale,
oscillation_min_period, oscillation_max_period, binning)
elif mission is None and coords is not None and cadence > 300:
return MissionFfiCoordsObjectInfo(
coords[0], coords[1], sectors, author, cadence, initial_mask,
initial_transit_mask, star_info, aperture, eleanor_corr_flux,
outliers_sigma, high_rms_enabled, high_rms_threshold,
high_rms_bin_hours, smooth_enabled, auto_detrend_enabled,
auto_detrend_method, auto_detrend_ratio, auto_detrend_period,
prepare_algorithm, reduce_simple_oscillations,
oscillation_snr_threshold, oscillation_amplitude_threshold,
oscillation_ws_scale, oscillation_min_period,
oscillation_max_period, binning)
elif mission is None and file is not None:
return InputObjectInfo(
file, initial_mask, initial_transit_mask, star_info,
outliers_sigma, high_rms_enabled, high_rms_threshold,
high_rms_bin_hours, smooth_enabled, auto_detrend_enabled,
auto_detrend_method, auto_detrend_ratio, auto_detrend_period,
prepare_algorithm, reduce_simple_oscillations,
oscillation_snr_threshold, oscillation_amplitude_threshold,
oscillation_ws_scale, oscillation_min_period,
oscillation_max_period, binning)
else:
raise ValueError(
"Invalid target definition with target_name={}, mission={}, id={}, coords={}, sectors={}, file={}, "
"cadence={}".format(target_name, mission, id, coords, sectors,
file, cadence))
from contextlib import contextmanager
from timeit import default_timer
from sherlockpipe.sherlock import Sherlock
from lcbuilder.objectinfo.MissionObjectInfo import MissionObjectInfo
from sherlockpipe.sherlock_target import SherlockTarget
@contextmanager
def elapsed_timer():
start = default_timer()
elapser = lambda: str(default_timer() - start)
yield lambda: elapser()
end = default_timer()
elapser = lambda: str(end - start)
with elapsed_timer() as elapsed:
# We will use only one mission object and set the period to be initially detrended with a window size of 1/3 times
# the given period.
sherlock = Sherlock([SherlockTarget(MissionObjectInfo("TIC 181804752", 'all'), auto_detrend_ratio=1 / 3)])\
.run()
print("Analysis took " + elapsed() + "s")
def prepare_tic(self, prepare_tic_input):
tic_id = str(prepare_tic_input.tic)
target_dir = prepare_tic_input.dir + tic_id + "/"
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
tpfs_short_dir = target_dir + "tpfs_short/"
tpfs_long_dir = target_dir + "tpfs_long/"
if not os.path.isdir(tpfs_short_dir):
os.mkdir(tpfs_short_dir)
if not os.path.isdir(tpfs_long_dir):
os.mkdir(tpfs_long_dir)
lc_short = None
lc_data_short = None
failed_target = None
try:
mission_lightcurve_builder = MissionLightcurveBuilder()
mission_ffi_lightcurve_builder = MissionFfiLightcurveBuilder()
except Exception as e:
failed_target = tic_id
logging.exception(e)
try:
logging.info("Trying to get short cadence info for " +
str(prepare_tic_input.tic))
lcbuild_short = \
mission_lightcurve_builder.build(MissionObjectInfo(tic_id, 'all'), None, self.cache_dir)
lc_short = lcbuild_short.lc
lc_data_short = self.store_lc_data(
lcbuild_short.lc_data, target_dir + "time_series_short.csv")
tpf_short = lk.search_targetpixelfile(
tic_id, cadence="short",
author="spoc").download_all(download_dir=self.cache_dir +
".lightkurve-cache")
for tpf in tpf_short.data:
shutil.copy(tpf.path,
tpfs_short_dir + os.path.basename(tpf.path))
short_periodogram = lc_short.to_periodogram(oversample_factor=5)
periodogram_df = pd.DataFrame(columns=['period', 'power'])
periodogram_df["period"] = short_periodogram.period.value
periodogram_df["power"] = short_periodogram.power.value
periodogram_df.to_csv(target_dir + "periodogram_short.csv")
except Exception as e:
logging.warning("No Short Cadence data for target " +
str(prepare_tic_input.tic))
logging.exception(e)
logging.info("Trying to get long cadence info for " +
str(prepare_tic_input.tic))
try:
lcbuild_long = \
mission_ffi_lightcurve_builder.build(MissionFfiIdObjectInfo(tic_id, 'all'), None,
self.cache_dir)
star_df = pd.DataFrame(columns=[
'obj_id', 'ra', 'dec', 'R_star', 'R_star_lerr', 'R_star_uerr',
'M_star', 'M_star_lerr', 'M_star_uerr', 'Teff_star',
'Teff_star_lerr', 'Teff_star_uerr', 'ld_a', 'ld_b'
])
ld_a = lcbuild_long.star_info.ld_coefficients[
0] if lcbuild_long.star_info.ld_coefficients is not None else None
ld_b = lcbuild_long.star_info.ld_coefficients[
1] if lcbuild_long.star_info.ld_coefficients is not None else None
star_df = star_df.append(
{
'obj_id':
tic_id,
'ra':
lcbuild_long.star_info.ra,
'dec':
lcbuild_long.star_info.dec,
'R_star':
lcbuild_long.star_info.radius,
'R_star_lerr':
lcbuild_long.star_info.radius -
lcbuild_long.star_info.radius_min,
'R_star_uerr':
lcbuild_long.star_info.radius_max -
lcbuild_long.star_info.radius,
'M_star':
lcbuild_long.star_info.mass,
'M_star_lerr':
lcbuild_long.star_info.mass -
lcbuild_long.star_info.mass_min,
'M_star_uerr':
lcbuild_long.star_info.mass_max -
lcbuild_long.star_info.mass,
'Teff_star':
lcbuild_long.star_info.teff,
'Teff_star_lerr':
200,
'Teff_star_uerr':
200,
'logg':
lcbuild_long.star_info.logg,
'logg_err':
lcbuild_long.star_info.logg_err,
'ld_a':
ld_a,
'ld_b':
ld_b,
'feh':
lcbuild_long.star_info.feh,
'feh_err':
lcbuild_long.star_info.feh_err,
'v':
lcbuild_long.star_info.v,
'v_err':
lcbuild_long.star_info.v_err,
'j':
lcbuild_long.star_info.j,
'j_err':
lcbuild_long.star_info.j_err,
'k':
lcbuild_long.star_info.k,
'k_err':
lcbuild_long.star_info.k_err,
'h':
lcbuild_long.star_info.h,
'h_err':
lcbuild_long.star_info.h_err,
'kp':
lcbuild_long.star_info.kp
},
ignore_index=True)
star_df.to_csv(target_dir + "params_star.csv", index=False)
sectors = lcbuild_long.sectors
lc_long = lcbuild_long.lc
lc_data_long = self.store_lc_data(
lcbuild_long.lc_data, target_dir + "time_series_long.csv")
lcf_long = lc_long.remove_nans()
tpf_long = lk.search_targetpixelfile(tic_id, cadence="long", author="tess-spoc")\
.download_all(download_dir=self.cache_dir + ".lightkurve-cache")
for tpf in tpf_long.data:
shutil.copy(tpf.path,
tpfs_long_dir + os.path.basename(tpf.path))
long_periodogram = lc_long.to_periodogram(oversample_factor=5)
periodogram_df = pd.DataFrame(columns=['period', 'power'])
periodogram_df["period"] = long_periodogram.period.value
periodogram_df["power"] = long_periodogram.power.value
periodogram_df.to_csv(target_dir + "periodogram_long.csv")
logging.info("Downloading neighbour stars for " +
prepare_tic_input.tic)
#TODO get neighbours light curves stars = self.download_neighbours(prepare_tic_input.tic, sectors)
logging.info("Classifying candidate points for " +
prepare_tic_input.tic)
target_ois = prepare_tic_input.target_ois[
(prepare_tic_input.target_ois["Disposition"] == "CP") |
(prepare_tic_input.target_ois["Disposition"] == "KP")]
target_ois_df = pd.DataFrame(columns=[
'id', 'name', 'period', 'period_err', 't0', 'to_err', 'depth',
'depth_err', 'duration', 'duration_err'
])
if lc_data_short is not None:
tags_series_short = np.full(len(lc_data_short.time), "BL")
tags_series_long = np.full(len(lc_data_long.time), "BL")
if prepare_tic_input.label is not None:
for index, row in target_ois.iterrows():
if row["OI"] not in prepare_tic_input.excluded_ois:
logging.info(
"Classifying candidate points with OI %s, period %s, t0 %s and duration %s for "
+ prepare_tic_input.tic, row["OI"],
row["Period (days)"], row["Epoch (BJD)"],
row["Duration (hours)"])
target_ois_df = target_ois_df.append(
{
"id": row["Object Id"],
"name": row["OI"],
"period": row["Period (days)"],
"period_err": row["Period (days) err"],
"t0": row["Epoch (BJD)"] - 2457000.0,
"to_err": row["Epoch (BJD) err"],
"depth": row["Depth (ppm)"],
"depth_err": row["Depth (ppm) err"],
"duration": row["Duration (hours)"],
"duration_err": row["Duration (hours) err"]
},
ignore_index=True)
if lc_short is not None:
mask_short = tls.transit_mask(
lc_data_short["time"].to_numpy(),
row["Period (days)"],
row["Duration (hours)"] / 24,
row["Epoch (BJD)"] - 2457000.0)
tags_series_short[
mask_short] = prepare_tic_input.label
mask_long = tls.transit_mask(
lc_data_long["time"].to_numpy(),
row["Period (days)"], row["Duration (hours)"] / 24,
row["Epoch (BJD)"] - 2457000.0)
tags_series_long[mask_long] = prepare_tic_input.label
for index, row in prepare_tic_input.target_additional_ois_df.iterrows(
):
if row["OI"] not in prepare_tic_input.excluded_ois:
target_ois_df = target_ois_df.append(
{
"id": row["Object Id"],
"name": row["OI"],
"period": row["Period (days)"],
"period_err": row["Period (days) err"],
"t0": row["Epoch (BJD)"] - 2457000.0,
"to_err": row["Epoch (BJD) err"],
"depth": row["Depth (ppm)"],
"depth_err": row["Depth (ppm) err"],
"duration": row["Duration (hours)"],
"duration_err": row["Duration (hours) err"]
},
ignore_index=True)
if lc_short is not None:
mask_short = tls.transit_mask(
lc_data_short["time"].to_numpy(),
row["Period (days)"],
row["Duration (hours)"] / 24,
row["Epoch (BJD)"] - 2457000.0)
tags_series_short[
mask_short] = prepare_tic_input.label
mask_long = tls.transit_mask(
lc_data_long["time"].to_numpy(),
row["Period (days)"], row["Duration (hours)"] / 24,
row["Epoch (BJD)"] - 2457000.0)
tags_series_long[mask_long] = prepare_tic_input.label
target_ois_df.to_csv(target_dir + "/ois.csv")
if lc_data_short is not None:
lc_data_short["tag"] = tags_series_short
lc_data_short.to_csv(target_dir + "time_series_short.csv")
lc_data_long["tag"] = tags_series_long
lc_data_long.to_csv(target_dir + "time_series_long.csv")
# TODO store folded light curves -with local and global views-(masking previous candidates?)
except Exception as e:
failed_target = tic_id
logging.exception(e)
return failed_target
def recovery(self, cores, simplified=True, sherlock_samples=3):
object_info = MissionObjectInfo(self.id, self.sectors)
lc, lc_data, star_info, transits_min_count, sectors, quarters = \
MissionLightcurveBuilder().build(object_info, None)
ab, mass, massmin, massmax, radius, radiusmin, radiusmax = catalog_info(
TIC_ID=id)
# units for ellc
rstar = star_info.radius * u.R_sun
mstar = star_info.mass * u.M_sun
mstar_min = star_info.mass_min * u.M_sun
mstar_max = star_info.mass_max * u.M_sun
rstar_min = star_info.radius_min * u.R_sun
rstar_max = star_info.radius_max * u.R_sun
print('\n STELLAR PROPERTIES FOR THE SIGNAL SEARCH')
print('================================================\n')
print('limb-darkening estimates using quadratic LD (a,b)=', ab)
print('mass =', format(mstar, '0.5f'))
print('mass_min =', format(mstar_min, '0.5f'))
print('mass_max =', format(mstar_max, '0.5f'))
print('radius =', format(rstar, '0.5f'))
print('radius_min =', format(rstar_min, '0.5f'))
print('radius_max =', format(rstar_max, '0.5f'))
lc_new = lk.LightCurve(time=lc.time,
flux=lc.flux,
flux_err=lc.flux_err)
clean = lc_new.remove_outliers(sigma_lower=float('inf'), sigma_upper=3)
flux0 = clean.flux
time = clean.time
flux_err = clean.flux_err
report = {}
reports_df = pd.DataFrame(columns=[
'period', 'radius', 'epoch', 'found', 'snr', 'sde', 'run'
])
known_transits = [{
"T0": 2458684.832891 - 2450000,
"P": 3.119035,
"D": 1.271571 / 24
}, {
"T0": 2458687.539955 - 2450000,
"P": 6.387611,
"D": 1.246105 / 24
}]
inject_dir = self.dir + "/curves/"
for file in os.listdir(inject_dir):
if file.endswith(".csv"):
try:
period = float(re.search("P([0-9]+\\.[0-9]+)", file)[1])
r_planet = float(re.search("R([0-9]+\\.[0-9]+)", file)[1])
epoch = float(
re.search("_([0-9]+\\.[0-9]+)\\.csv", file)[1])
df = pd.read_csv(inject_dir + file,
float_precision='round_trip',
sep=',',
usecols=['#time', 'flux', 'flux_err'])
if len(df) == 0:
found = True
snr = 20
sde = 20
run = 1
else:
lc = lk.LightCurve(time=df['#time'],
flux=df['flux'],
flux_err=df['flux_err'])
clean = lc.remove_nans().remove_outliers(
sigma_lower=float('inf'),
sigma_upper=3) # remove outliers over 3sigma
flux = clean.flux
time = clean.time
intransit = self.__transit_masks(known_transits, time)
found, snr, sde, run = self.__tls_search(
time, flux, radius, rstar_min, rstar_max, mass,
mstar_min, mstar_max, intransit, epoch, period,
0.5, 45, 5, cores, "default")
new_report = {
"period": period,
"radius": r_planet,
"epoch": epoch,
"found": found,
"snr": snr,
"sde": sde,
"run": run
}
reports_df = reports_df.append(new_report,
ignore_index=True)
print("P=" + str(period) + ", R=" + str(r_planet) +
", T0=" + str(epoch) + ", FOUND WAS " + str(found) +
" WITH SNR " + str(snr) + " AND SDE " + str(sde))
reports_df.to_csv(inject_dir + "a_tls_report.csv",
index=False)
except Exception as e:
traceback.print_exc()
print("File not valid: " + file)
if not simplified:
report = {}
reports_df = pd.DataFrame(columns=[
'period', 'radius', 'epoch', 'found', 'snr', 'sde', 'run'
])
a = False
tls_report_df = pd.read_csv(
inject_dir + "a_tls_report.csv",
float_precision='round_trip',
sep=',',
usecols=['period', 'radius', 'epoch', 'found', 'snr', 'run'])
samples_analysed = sherlock_samples
for index, row in tls_report_df[::-1].iterrows():
file = os.path.join('P' + str(row['period']) + '_R' +
str(row['radius']) + '_' +
str(row['epoch']) + '.csv')
first_false = index > 0 and tls_report_df.iloc[index - 1]['found'] and \
not tls_report_df.iloc[index]['found']
if first_false:
samples_analysed = 0
elif tls_report_df.iloc[index]['found']:
samples_analysed = sherlock_samples
if samples_analysed < sherlock_samples:
try:
samples_analysed = samples_analysed + 1
period = float(
re.search("P([0-9]+\\.[0-9]+)", file)[1])
r_planet = float(
re.search("R([0-9]+\\.[0-9]+)", file)[1])
epoch = float(
re.search("_([0-9]+\\.[0-9]+)\\.csv", file)[1])
signal_selection_algorithm = QuorumSnrBorderCorrectedStopWhenMatchSignalSelector(
1, 0, period, epoch)
df = pd.read_csv(inject_dir + file,
float_precision='round_trip',
sep=',',
usecols=['#time', 'flux', 'flux_err'])
if len(df) == 0:
found = True
snr = 20
sde = 20
run = 1
else:
sherlock.Sherlock(sherlock_targets=[MissionInputObjectInfo(self.id, inject_dir + file)]) \
.setup_detrend(True, True, 1.5, 4, 12, "biweight", 0.2, 1.0, 20, False,
0.25, "cosine", None) \
.setup_transit_adjust_params(5, None, None, 10, None, None, 0.4, 14, 10,
20, 5, 5.5, 0.05, "mask", "quorum", 1, 0,
signal_selection_algorithm) \
.run()
df = pd.read_csv(self.id.replace(" ", "") +
"_INP/candidates.csv",
float_precision='round_trip',
sep=',',
usecols=[
'curve', 'period', 't0',
'run', 'snr', 'sde', 'rad_p',
'transits'
])
snr = df["snr"].iloc[len(df) - 1]
run = df["run"].iloc[len(df) - 1]
sde = df["sde"].iloc[len(df) - 1]
per_run = 0
found_period = False
j = 0
for per in df["period"]:
if signal_selection_algorithm.is_harmonic(
per, period / 2.):
found_period = True
t0 = df["t0"].iloc[j]
break
j = j + 1
right_epoch = False
if found_period:
for i in range(-5, 5):
right_epoch = right_epoch or (
np.abs(t0 - epoch + i * period) <
(1. / 24.))
if right_epoch:
snr = df["snr"].iloc[j]
run = df["run"].iloc[j]
sde = df["sde"].iloc[j]
break
found = right_epoch
new_report = {
"period": period,
"radius": r_planet,
"epoch": epoch,
"found": found,
"sde": sde,
"snr": snr,
"run": int(run)
}
reports_df = reports_df.append(new_report,
ignore_index=True)
reports_df.to_csv(inject_dir + "a_sherlock_report.csv",
index=False)
print("P=" + str(period) + ", R=" + str(r_planet) +
", T0=" + str(epoch) + ", FOUND WAS " +
str(found) + " WITH SNR " + str(snr) +
"and SDE " + str(sde))
except Exception as e:
print(e)
print("File not valid: " + file)