def download_exofop_toi_lightcurves_to_synthetic_directory(self): """ Downloads the `ExoFOP database <https://exofop.ipac.caltech.edu/tess/view_toi.php>`_ lightcurve files to the synthetic directory. """ print("Downloading ExoFOP TOI disposition CSV...") self.create_data_directories() toi_csv_url = 'https://exofop.ipac.caltech.edu/tess/download_toi.php?sort=toi&output=csv' response = requests.get(toi_csv_url) with self.toi_dispositions_path.open('wb') as csv_file: csv_file.write(response.content) toi_dispositions = self.tess_toi_data_interface.dispositions tic_ids = toi_dispositions[ToiColumns.tic_id.value].unique() print('Downloading TESS obdservation list...') tess_data_interface = TessDataInterface() tess_observations = tess_data_interface.get_all_tess_time_series_observations(tic_id=tic_ids) single_sector_observations = tess_data_interface.filter_for_single_sector_observations(tess_observations) single_sector_observations = tess_data_interface.add_tic_id_column_to_single_sector_observations( single_sector_observations) single_sector_observations = tess_data_interface.add_sector_column_to_single_sector_observations( single_sector_observations) print("Downloading lightcurves which are confirmed or suspected planets in TOI dispositions...") suspected_planet_dispositions = toi_dispositions[toi_dispositions[ToiColumns.disposition.value] != 'FP'] suspected_planet_observations = pd.merge(single_sector_observations, suspected_planet_dispositions, how='inner', on=[ToiColumns.tic_id.value, ToiColumns.sector.value]) observations_not_found = suspected_planet_dispositions.shape[0] - suspected_planet_observations.shape[0] print(f"{suspected_planet_observations.shape[0]} observations found that match the TOI dispositions.") print(f"No observations found for {observations_not_found} entries in TOI dispositions.") suspected_planet_data_products = tess_data_interface.get_product_list(suspected_planet_observations) suspected_planet_lightcurve_data_products = suspected_planet_data_products[ suspected_planet_data_products['productFilename'].str.endswith('lc.fits') ] suspected_planet_download_manifest = tess_data_interface.download_products( suspected_planet_lightcurve_data_products, data_directory=self.data_directory) print(f'Moving lightcurves to {self.synthetic_signal_directory}...') for file_path_string in suspected_planet_download_manifest['Local Path']: file_path = Path(file_path_string) file_path.rename(self.synthetic_signal_directory.joinpath(file_path.name))
def download_catalog_eclipsing_binaries(self): """ Downloads the eclipsing binaries listed in Brian Powell's catalog to the synthetic signals directory. """ catalog = pd.read_csv(self.catalog_csv_path) catalog = catalog[catalog['2min'] == 1] tess_data_interface = TessDataInterface() tess_observations = tess_data_interface.get_all_tess_time_series_observations(tic_id=catalog['ID']) single_sector_observations = tess_data_interface.filter_for_single_sector_observations(tess_observations) single_sector_observations = tess_data_interface.add_tic_id_column_to_single_sector_observations( single_sector_observations) single_sector_observations = tess_data_interface.add_sector_column_to_single_sector_observations( single_sector_observations) single_sector_data_products = tess_data_interface.get_product_list(single_sector_observations) data_products = single_sector_data_products[ single_sector_data_products['productFilename'].str.endswith('lc.fits') ] download_manifest = self.tess_data_interface.download_products( data_products, data_directory=self.data_directory) print(f'Moving lightcurves to {self.synthetic_signal_directory}...') self.synthetic_signal_directory.mkdir(parents=True, exist_ok=True) for file_path_string in download_manifest['Local Path']: file_path = Path(file_path_string) file_path.rename(self.synthetic_signal_directory.joinpath(file_path.name))
def download_exofop_toi_database( self, number_of_negative_lightcurves_to_download=10000): """ Downloads the `ExoFOP database <https://exofop.ipac.caltech.edu/tess/view_toi.php>`_. """ # print('Clearing data directory...') # self.clear_data_directory() print("Downloading ExoFOP TOI disposition CSV...") toi_csv_url = 'https://exofop.ipac.caltech.edu/tess/download_toi.php?sort=toi&output=csv' response = requests.get(toi_csv_url) with self.toi_dispositions_path.open('wb') as csv_file: csv_file.write(response.content) print('Downloading TESS observation list...') tess_data_interface = TessDataInterface() tess_observations = tess_data_interface.get_all_tess_time_series_observations( ) single_sector_observations = tess_data_interface.filter_for_single_sector_observations( tess_observations) single_sector_observations = tess_data_interface.add_tic_id_column_to_single_sector_observations( single_sector_observations) single_sector_observations = tess_data_interface.add_sector_column_to_single_sector_observations( single_sector_observations) print( "Downloading lightcurves which are confirmed or suspected planets in TOI dispositions..." ) tess_toi_data_interface = TessToiDataInterface() toi_database.tess_data_interface.download_exofop_toi_lightcurves_to_directory( toi_database.synthetic_signal_directory) toi_dispositions = tess_toi_data_interface.load_toi_dispositions_in_project_format( ) print( "Downloading lightcurves which are not in TOI dispositions and do not have TCEs (not planets)..." ) print( f'Download limited to {number_of_negative_lightcurves_to_download} lightcurves...' ) # noinspection SpellCheckingInspection toi_tic_ids = toi_dispositions['TIC ID'].values not_toi_observations = single_sector_observations[ ~single_sector_observations['TIC ID']. isin(toi_tic_ids) # Don't include even false positives. ] not_toi_observations = not_toi_observations.sample(frac=1, random_state=0) # Shorten product list obtaining. not_toi_observations = not_toi_observations.head( number_of_negative_lightcurves_to_download * 2) not_toi_data_products = tess_data_interface.get_product_list( not_toi_observations) not_toi_data_products = tess_data_interface.add_tic_id_column_to_single_sector_observations( not_toi_data_products) not_toi_lightcurve_data_products = not_toi_data_products[ not_toi_data_products['productFilename'].str.endswith('lc.fits')] not_toi_data_validation_data_products = not_toi_data_products[ not_toi_data_products['productFilename'].str.endswith('dvr.xml')] tic_ids_with_dv = not_toi_data_validation_data_products[ 'TIC ID'].values not_planet_lightcurve_data_products = not_toi_lightcurve_data_products[ ~not_toi_lightcurve_data_products['TIC ID']. isin(tic_ids_with_dv) # Remove any lightcurves with TCEs. ] # Shuffle rows. not_planet_lightcurve_data_products = not_planet_lightcurve_data_products.sample( frac=1, random_state=0) not_planet_download_manifest = tess_data_interface.download_products( not_planet_lightcurve_data_products.head( number_of_negative_lightcurves_to_download), data_directory=self.data_directory) print( f'Verifying and moving lightcurves to {self.lightcurve_directory}...' ) self.lightcurve_directory.mkdir(parents=True, exist_ok=True) for file_path_string in not_planet_download_manifest['Local Path']: file_path = Path(file_path_string) lightcurve_path = self.lightcurve_directory.joinpath( file_path.name) try: file_path.rename(lightcurve_path) hdu_list = fits.open(str(lightcurve_path)) lightcurve = hdu_list[1].data _ = lightcurve['TIME'][0] except (OSError, TypeError): print( f'{file_path} seems to be corrupt. Re-downloading and replacing.' ) sector = tess_data_interface.get_sector_from_single_sector_obs_id( str(lightcurve_path.stem)) tic_id = tess_data_interface.get_tic_id_from_single_sector_obs_id( str(lightcurve_path.stem)) tess_data_interface.download_lightcurve( tic_id, sector, save_directory=lightcurve_path.parent) print('Database ready.')
def download_exofop_toi_database( self, number_of_negative_lightcurves_to_download=10000): """ Downloads the `ExoFOP database <https://exofop.ipac.caltech.edu/tess/view_toi.php>`_. """ print('Clearing data directory...') self.clear_data_directory() print("Downloading ExoFOP TOI disposition CSV...") toi_csv_url = 'https://exofop.ipac.caltech.edu/tess/download_toi.php?sort=toi&output=csv' response = requests.get(toi_csv_url) with self.toi_dispositions_path.open('wb') as csv_file: csv_file.write(response.content) print('Downloading TESS observation list...') tess_data_interface = TessDataInterface() tess_observations = tess_data_interface.get_all_tess_time_series_observations( ) single_sector_observations = tess_data_interface.filter_for_single_sector_observations( tess_observations) single_sector_observations = tess_data_interface.add_tic_id_column_to_single_sector_observations( single_sector_observations) single_sector_observations = tess_data_interface.add_sector_column_to_single_sector_observations( single_sector_observations) print( "Downloading lightcurves which are confirmed or suspected planets in TOI dispositions..." ) toi_dispositions = self.load_toi_dispositions_in_project_format() suspected_planet_dispositions = toi_dispositions[ toi_dispositions['disposition'] != 'FP'] suspected_planet_observations = pd.merge(single_sector_observations, suspected_planet_dispositions, how='inner', on=['TIC ID', 'Sector']) observations_not_found = suspected_planet_dispositions.shape[ 0] - suspected_planet_observations.shape[0] print( f"{suspected_planet_observations.shape[0]} observations found that match the TOI dispositions." ) print( f"No observations found for {observations_not_found} entries in TOI dispositions." ) suspected_planet_data_products = tess_data_interface.get_product_list( suspected_planet_observations) suspected_planet_lightcurve_data_products = suspected_planet_data_products[ suspected_planet_data_products['productFilename'].str.endswith( 'lc.fits')] suspected_planet_download_manifest = tess_data_interface.download_products( suspected_planet_lightcurve_data_products, data_directory=self.data_directory) print(f'Moving lightcurves to {self.lightcurve_directory}...') for file_path_string in suspected_planet_download_manifest[ 'Local Path']: file_path = Path(file_path_string) file_path.rename(self.lightcurve_directory.joinpath( file_path.name)) print( "Downloading lightcurves which are not in TOI dispositions and do not have TCEs (not planets)..." ) print( f'Download limited to {number_of_negative_lightcurves_to_download} lightcurves...' ) # noinspection SpellCheckingInspection toi_tic_ids = toi_dispositions['TIC ID'].values not_toi_observations = single_sector_observations[ ~single_sector_observations['TIC ID']. isin(toi_tic_ids) # Don't include even false positives. ] not_toi_observations = not_toi_observations.sample(frac=1, random_state=0) # Shorten product list obtaining. not_toi_observations = not_toi_observations.head( number_of_negative_lightcurves_to_download * 2) not_toi_data_products = tess_data_interface.get_product_list( not_toi_observations) not_toi_data_products = tess_data_interface.add_tic_id_column_to_single_sector_observations( not_toi_data_products) not_toi_lightcurve_data_products = not_toi_data_products[ not_toi_data_products['productFilename'].str.endswith('lc.fits')] not_toi_data_validation_data_products = not_toi_data_products[ not_toi_data_products['productFilename'].str.endswith('dvr.xml')] tic_ids_with_dv = not_toi_data_validation_data_products[ 'TIC ID'].values not_planet_lightcurve_data_products = not_toi_lightcurve_data_products[ ~not_toi_lightcurve_data_products['TIC ID']. isin(tic_ids_with_dv) # Remove any lightcurves with TCEs. ] # Shuffle rows. not_planet_lightcurve_data_products = not_planet_lightcurve_data_products.sample( frac=1, random_state=0) not_planet_download_manifest = tess_data_interface.download_products( not_planet_lightcurve_data_products.head( number_of_negative_lightcurves_to_download), data_directory=self.data_directory) print(f'Moving lightcurves to {self.lightcurve_directory}...') for file_path_string in not_planet_download_manifest['Local Path']: file_path = Path(file_path_string) file_path.rename(self.lightcurve_directory.joinpath( file_path.name)) print('Database ready.')