def download_exofop_toi_light_curves_to_directory(self, directory: Path):
"""
Downloads the `ExoFOP database <https://exofop.ipac.caltech.edu/tess/view_toi.php>`_ light curve files to the
given directory.
:param directory: The directory to download the light curves to. Defaults to the data interface directory.
"""
print("Downloading ExoFOP TOI disposition CSV...")
tess_data_interface = TessDataInterface()
if isinstance(directory, str):
directory = Path(directory)
tic_ids = self.toi_dispositions[ToiColumns.tic_id.value].unique()
print('Downloading TESS observation list...')
single_sector_observations = tess_data_interface.get_all_two_minute_single_sector_observations(
tic_ids)
print(
"Downloading light curves which are confirmed or suspected planets in TOI dispositions..."
)
suspected_planet_dispositions = self.toi_dispositions[
self.toi_dispositions[ToiColumns.disposition.value] != 'FP']
suspected_planet_observations = pd.merge(
single_sector_observations,
suspected_planet_dispositions,
how='inner',
on=[ToiColumns.tic_id.value, ToiColumns.sector.value])
suspected_planet_data_products = tess_data_interface.get_product_list(
suspected_planet_observations)
suspected_planet_light_curve_data_products = suspected_planet_data_products[
suspected_planet_data_products['productFilename'].str.endswith(
'lc.fits')]
suspected_planet_download_manifest = tess_data_interface.download_products(
suspected_planet_light_curve_data_products,
data_directory=self.data_directory)
print(f'Verifying and moving light curves to {directory}...')
directory.mkdir(parents=True, exist_ok=True)
for row_index, row in suspected_planet_download_manifest.iterrows():
if row['Status'] == 'COMPLETE':
file_path = Path(row['Local Path'])
file_path.rename(directory.joinpath(file_path.name))
def download_exofop_toi_lightcurves_to_synthetic_directory(self):
"""
Downloads the `ExoFOP database <https://exofop.ipac.caltech.edu/tess/view_toi.php>`_ lightcurve files to the
synthetic directory.
"""
print("Downloading ExoFOP TOI disposition CSV...")
self.create_data_directories()
toi_csv_url = 'https://exofop.ipac.caltech.edu/tess/download_toi.php?sort=toi&output=csv'
response = requests.get(toi_csv_url)
with self.toi_dispositions_path.open('wb') as csv_file:
csv_file.write(response.content)
toi_dispositions = self.tess_toi_data_interface.dispositions
tic_ids = toi_dispositions[ToiColumns.tic_id.value].unique()
print('Downloading TESS obdservation list...')
tess_data_interface = TessDataInterface()
tess_observations = tess_data_interface.get_all_tess_time_series_observations(tic_id=tic_ids)
single_sector_observations = tess_data_interface.filter_for_single_sector_observations(tess_observations)
single_sector_observations = tess_data_interface.add_tic_id_column_to_single_sector_observations(
single_sector_observations)
single_sector_observations = tess_data_interface.add_sector_column_to_single_sector_observations(
single_sector_observations)
print("Downloading lightcurves which are confirmed or suspected planets in TOI dispositions...")
suspected_planet_dispositions = toi_dispositions[toi_dispositions[ToiColumns.disposition.value] != 'FP']
suspected_planet_observations = pd.merge(single_sector_observations, suspected_planet_dispositions, how='inner',
on=[ToiColumns.tic_id.value, ToiColumns.sector.value])
observations_not_found = suspected_planet_dispositions.shape[0] - suspected_planet_observations.shape[0]
print(f"{suspected_planet_observations.shape[0]} observations found that match the TOI dispositions.")
print(f"No observations found for {observations_not_found} entries in TOI dispositions.")
suspected_planet_data_products = tess_data_interface.get_product_list(suspected_planet_observations)
suspected_planet_lightcurve_data_products = suspected_planet_data_products[
suspected_planet_data_products['productFilename'].str.endswith('lc.fits')
]
suspected_planet_download_manifest = tess_data_interface.download_products(
suspected_planet_lightcurve_data_products, data_directory=self.data_directory)
print(f'Moving lightcurves to {self.synthetic_signal_directory}...')
for file_path_string in suspected_planet_download_manifest['Local Path']:
file_path = Path(file_path_string)
file_path.rename(self.synthetic_signal_directory.joinpath(file_path.name))
def download_exofop_toi_database(
self, number_of_negative_lightcurves_to_download=10000):
"""
Downloads the `ExoFOP database <https://exofop.ipac.caltech.edu/tess/view_toi.php>`_.
"""
# print('Clearing data directory...')
# self.clear_data_directory()
print("Downloading ExoFOP TOI disposition CSV...")
toi_csv_url = 'https://exofop.ipac.caltech.edu/tess/download_toi.php?sort=toi&output=csv'
response = requests.get(toi_csv_url)
with self.toi_dispositions_path.open('wb') as csv_file:
csv_file.write(response.content)
print('Downloading TESS observation list...')
tess_data_interface = TessDataInterface()
tess_observations = tess_data_interface.get_all_tess_time_series_observations(
)
single_sector_observations = tess_data_interface.filter_for_single_sector_observations(
tess_observations)
single_sector_observations = tess_data_interface.add_tic_id_column_to_single_sector_observations(
single_sector_observations)
single_sector_observations = tess_data_interface.add_sector_column_to_single_sector_observations(
single_sector_observations)
print(
"Downloading lightcurves which are confirmed or suspected planets in TOI dispositions..."
)
tess_toi_data_interface = TessToiDataInterface()
toi_database.tess_data_interface.download_exofop_toi_lightcurves_to_directory(
toi_database.synthetic_signal_directory)
toi_dispositions = tess_toi_data_interface.load_toi_dispositions_in_project_format(
)
print(
"Downloading lightcurves which are not in TOI dispositions and do not have TCEs (not planets)..."
)
print(
f'Download limited to {number_of_negative_lightcurves_to_download} lightcurves...'
)
# noinspection SpellCheckingInspection
toi_tic_ids = toi_dispositions['TIC ID'].values
not_toi_observations = single_sector_observations[
~single_sector_observations['TIC ID'].
isin(toi_tic_ids) # Don't include even false positives.
]
not_toi_observations = not_toi_observations.sample(frac=1,
random_state=0)
# Shorten product list obtaining.
not_toi_observations = not_toi_observations.head(
number_of_negative_lightcurves_to_download * 2)
not_toi_data_products = tess_data_interface.get_product_list(
not_toi_observations)
not_toi_data_products = tess_data_interface.add_tic_id_column_to_single_sector_observations(
not_toi_data_products)
not_toi_lightcurve_data_products = not_toi_data_products[
not_toi_data_products['productFilename'].str.endswith('lc.fits')]
not_toi_data_validation_data_products = not_toi_data_products[
not_toi_data_products['productFilename'].str.endswith('dvr.xml')]
tic_ids_with_dv = not_toi_data_validation_data_products[
'TIC ID'].values
not_planet_lightcurve_data_products = not_toi_lightcurve_data_products[
~not_toi_lightcurve_data_products['TIC ID'].
isin(tic_ids_with_dv) # Remove any lightcurves with TCEs.
]
# Shuffle rows.
not_planet_lightcurve_data_products = not_planet_lightcurve_data_products.sample(
frac=1, random_state=0)
not_planet_download_manifest = tess_data_interface.download_products(
not_planet_lightcurve_data_products.head(
number_of_negative_lightcurves_to_download),
data_directory=self.data_directory)
print(
f'Verifying and moving lightcurves to {self.lightcurve_directory}...'
)
self.lightcurve_directory.mkdir(parents=True, exist_ok=True)
for file_path_string in not_planet_download_manifest['Local Path']:
file_path = Path(file_path_string)
lightcurve_path = self.lightcurve_directory.joinpath(
file_path.name)
try:
file_path.rename(lightcurve_path)
hdu_list = fits.open(str(lightcurve_path))
lightcurve = hdu_list[1].data
_ = lightcurve['TIME'][0]
except (OSError, TypeError):
print(
f'{file_path} seems to be corrupt. Re-downloading and replacing.'
)
sector = tess_data_interface.get_sector_from_single_sector_obs_id(
str(lightcurve_path.stem))
tic_id = tess_data_interface.get_tic_id_from_single_sector_obs_id(
str(lightcurve_path.stem))
tess_data_interface.download_lightcurve(
tic_id, sector, save_directory=lightcurve_path.parent)
print('Database ready.')
def download_exofop_toi_database(
self, number_of_negative_lightcurves_to_download=10000):
"""
Downloads the `ExoFOP database <https://exofop.ipac.caltech.edu/tess/view_toi.php>`_.
"""
print('Clearing data directory...')
self.clear_data_directory()
print("Downloading ExoFOP TOI disposition CSV...")
toi_csv_url = 'https://exofop.ipac.caltech.edu/tess/download_toi.php?sort=toi&output=csv'
response = requests.get(toi_csv_url)
with self.toi_dispositions_path.open('wb') as csv_file:
csv_file.write(response.content)
print('Downloading TESS observation list...')
tess_data_interface = TessDataInterface()
tess_observations = tess_data_interface.get_all_tess_time_series_observations(
)
single_sector_observations = tess_data_interface.filter_for_single_sector_observations(
tess_observations)
single_sector_observations = tess_data_interface.add_tic_id_column_to_single_sector_observations(
single_sector_observations)
single_sector_observations = tess_data_interface.add_sector_column_to_single_sector_observations(
single_sector_observations)
print(
"Downloading lightcurves which are confirmed or suspected planets in TOI dispositions..."
)
toi_dispositions = self.load_toi_dispositions_in_project_format()
suspected_planet_dispositions = toi_dispositions[
toi_dispositions['disposition'] != 'FP']
suspected_planet_observations = pd.merge(single_sector_observations,
suspected_planet_dispositions,
how='inner',
on=['TIC ID', 'Sector'])
observations_not_found = suspected_planet_dispositions.shape[
0] - suspected_planet_observations.shape[0]
print(
f"{suspected_planet_observations.shape[0]} observations found that match the TOI dispositions."
)
print(
f"No observations found for {observations_not_found} entries in TOI dispositions."
)
suspected_planet_data_products = tess_data_interface.get_product_list(
suspected_planet_observations)
suspected_planet_lightcurve_data_products = suspected_planet_data_products[
suspected_planet_data_products['productFilename'].str.endswith(
'lc.fits')]
suspected_planet_download_manifest = tess_data_interface.download_products(
suspected_planet_lightcurve_data_products,
data_directory=self.data_directory)
print(f'Moving lightcurves to {self.lightcurve_directory}...')
for file_path_string in suspected_planet_download_manifest[
'Local Path']:
file_path = Path(file_path_string)
file_path.rename(self.lightcurve_directory.joinpath(
file_path.name))
print(
"Downloading lightcurves which are not in TOI dispositions and do not have TCEs (not planets)..."
)
print(
f'Download limited to {number_of_negative_lightcurves_to_download} lightcurves...'
)
# noinspection SpellCheckingInspection
toi_tic_ids = toi_dispositions['TIC ID'].values
not_toi_observations = single_sector_observations[
~single_sector_observations['TIC ID'].
isin(toi_tic_ids) # Don't include even false positives.
]
not_toi_observations = not_toi_observations.sample(frac=1,
random_state=0)
# Shorten product list obtaining.
not_toi_observations = not_toi_observations.head(
number_of_negative_lightcurves_to_download * 2)
not_toi_data_products = tess_data_interface.get_product_list(
not_toi_observations)
not_toi_data_products = tess_data_interface.add_tic_id_column_to_single_sector_observations(
not_toi_data_products)
not_toi_lightcurve_data_products = not_toi_data_products[
not_toi_data_products['productFilename'].str.endswith('lc.fits')]
not_toi_data_validation_data_products = not_toi_data_products[
not_toi_data_products['productFilename'].str.endswith('dvr.xml')]
tic_ids_with_dv = not_toi_data_validation_data_products[
'TIC ID'].values
not_planet_lightcurve_data_products = not_toi_lightcurve_data_products[
~not_toi_lightcurve_data_products['TIC ID'].
isin(tic_ids_with_dv) # Remove any lightcurves with TCEs.
]
# Shuffle rows.
not_planet_lightcurve_data_products = not_planet_lightcurve_data_products.sample(
frac=1, random_state=0)
not_planet_download_manifest = tess_data_interface.download_products(
not_planet_lightcurve_data_products.head(
number_of_negative_lightcurves_to_download),
data_directory=self.data_directory)
print(f'Moving lightcurves to {self.lightcurve_directory}...')
for file_path_string in not_planet_download_manifest['Local Path']:
file_path = Path(file_path_string)
file_path.rename(self.lightcurve_directory.joinpath(
file_path.name))
print('Database ready.')