def test_db_reader( make_test_database: Callable[[str], DatabaseConnector], write_to_split: Callable[[str], DatabaseConnector], ) -> None: """ Test we can create read from the database. Specifically this test that: 1. We can make a query 2. That an empty db has not been populated 3. That a populated db has table entries 4. Check that we can extract the id for a given set of values which exist 5. Check that no id is returned if a given set of values is not found in the database 6. That we can get the latest row id which has been written to Parameters ---------- make_test_database : function Function which returns the database connection write_to_split : function Function returning the database connection where `split` has been populated """ empty_db_connector = make_test_database("empty_read_test") empty_db_reader = DatabaseReader(empty_db_connector) # Check that we can make a query table = empty_db_reader.query("SELECT 1+1 AS col") assert table.loc[0, "col"] == 2 # pylint: disable=no-member # Check that the tables has not been created in an empty db assert not empty_db_reader.check_tables_created() db_connector = write_to_split("read_test") db_reader = DatabaseReader(db_connector) # Check that tables exist assert db_reader.check_tables_created() # As write_to_split writes to the split table, we can get the # written values with the following query table = db_reader.query("SELECT * FROM split") entries_dict = table.to_dict(orient="records")[0] # Remove the 'id' entries_dict.pop("id") row_id = db_reader.get_entry_id("split", entries_dict) assert row_id == 1 # Modify entries_dict so that row_id returns None entries_dict[list(entries_dict.keys())[0]] += 1 new_row_id = db_reader.get_entry_id("split", entries_dict) assert new_row_id is None # Assert that get_latest_row_id is working assert db_reader.get_latest_row_id() == 1
def test_db_creator( make_test_database: Callable[[str], DatabaseConnector], make_test_schema: Callable[ [str], Tuple[DatabaseConnector, Dict[str, Dict[str, str]]] ], ) -> None: """ Test we can create the database schemas. Specifically this test that: 1. The database is empty on creation 2. The tables are created 3. It is not possible to create the schema more than once 4. Check that all expected tables have been created Parameters ---------- make_test_database : function Function returning the database connection make_test_schema : function Function returning the database connection and the final parameters as sql types """ db_connector_no_schema = make_test_database("test_creation_without_schema") db_reader_no_schema = DatabaseReader(db_connector_no_schema) # There should be no tables before creating them assert not db_reader_no_schema.check_tables_created() db_connector_schema, final_parameters_as_sql_types = make_test_schema( "test_creation_with_schema" ) db_reader_schema = DatabaseReader(db_connector_schema) db_creator = DatabaseCreator(db_connector_schema) # The tables should now have been created assert db_reader_schema.check_tables_created() with pytest.raises(sqlite3.OperationalError): db_creator.create_all_schema_tables(final_parameters_as_sql_types) # Check that all tables has been created non_parameter_tables = { "system_info", "split", "file_modification", "parameters", "run", } parameter_tables = set( el.replace(":", "_") for el in final_parameters_as_sql_types.keys() ) query_str = 'SELECT name FROM sqlite_master WHERE type="table"' table = db_reader_schema.query(query_str) actual = table.loc[:, "name"].values # pylint: disable=no-member assert non_parameter_tables.union(parameter_tables) == set(actual)
def assert_first_run(bout_paths: BoutPaths, db_connector: DatabaseConnector) -> DatabaseReader: """ Assert that the first run went well. Parameters ---------- bout_paths : BoutPaths The object containing the paths db_connector : DatabaseConnector The database connection Returns ------- db_reader : DatabaseReader The database reader object """ db_reader = DatabaseReader(db_connector) assert_dump_files_exist(bout_paths.bout_inp_dst_dir) assert db_reader.check_tables_created() return db_reader
class StatusChecker: r""" Class to check and update the status of runs. Attributes ---------- __db_connector : DatabaseConnector Connection to the database under consideration __db_reader : DatabaseReader Object to read the database with project_path : Path Path to the project Methods ------- check_and_update_status() Check and update the status for the schema check_and_update_status_until_complete() Check and update the status until all runs are stopped __check_submitted(metadata_updater, submitted_to_check) Check the status of all runs which has status `submitted` __check_running(metadata_updater, running_to_check) Check the status of all runs which has status `running` __check_if_stopped(log_reader, metadata_updater) Check if a run has stopped check_if_running_or_errored(log_reader) Check if a run is still running or has errored Examples -------- >>> from pathlib import Path >>> from bout_runners.database.database_connector import \ ... DatabaseConnector >>> db_connector = DatabaseConnector('name_of_db') >>> project_path = Path('path').joinpath('to', 'project') >>> status_checker = StatusChecker(db_connector, project_path) >>> status_checker.check_and_update_status() Any updates to the runs will be written to the database. Alternatively, one can run the program until all jobs have stopped by calling >>> status_checker.check_and_update_until_complete() """ def __init__( self, db_connector: Optional[DatabaseConnector] = None, project_path: Optional[Union[Path, str]] = None, ) -> None: """ Set connector, reader and a project path. Notes ----- The StatusChecker instance only checks the project belonging to the same database schema grouped together by the `db_connector` Parameters ---------- db_connector : DatabaseConnector Connection to the database project_path : Path Path to the project (the root directory with which usually contains the makefile and the executable) """ self.__db_connector = (db_connector if db_connector is not None else DatabaseConnector()) self.__db_reader = DatabaseReader(self.__db_connector) self.__project_path = Path( project_path) if project_path is not None else Path() def check_and_update_status(self) -> None: """ Check and update the status for the schema. Raises ------ RuntimeError If the schema does not exist """ # Check that run table exist if not self.__db_reader.check_tables_created(): logging.error( "No tables found in %s", self.__db_reader.db_connector.db_path, ) message = "Can not check the status of schemas that does not exist" raise RuntimeError(message) # Create place holder metadata_updater metadata_updater = MetadataUpdater(self.__db_connector, run_id=-1) # Check runs with status 'submitted' query = ("SELECT name, id AS run_id FROM run WHERE\n" "latest_status = 'submitted' OR\n" "latest_status = 'created'") submitted_to_check = self.__db_reader.query(query) self.__check_submitted(metadata_updater, submitted_to_check) # Check runs with status 'running' query = 'SELECT name, id FROM run WHERE latest_status = "running"' running_to_check = self.__db_reader.query(query) self.__check_running(metadata_updater, running_to_check) @staticmethod def get_query_string_for_non_errored_runs() -> str: """ Return the query string for non errored results. Returns ------- str Query string for non errored results """ return ("SELECT name, id AS run_id FROM run WHERE\n" "latest_status = 'submitted' OR\n" "latest_status = 'created' OR\n" "latest_status = 'running'") def check_and_update_until_complete(self, seconds_between_update: int = 5 ) -> None: """ Check and update the status until all runs are stopped. Parameters ---------- seconds_between_update : int Number of seconds before a new status check is performed """ query = self.get_query_string_for_non_errored_runs() while len(self.__db_reader.query(query).index) != 0: self.check_and_update_status() time.sleep(seconds_between_update) def __check_submitted(self, metadata_updater: MetadataUpdater, submitted_to_check: DataFrame) -> None: """ Check the status of all runs which has status `submitted`. Parameters ---------- metadata_updater : MetadataUpdater Object which updates the database submitted_to_check : DataFrame DataFrame containing the `id` and `name` of the runs with status `submitted` Raises ------ RuntimeError In case log_reader.started() is True and log_reader.start_time is None """ for name, run_id in submitted_to_check.itertuples(index=False): metadata_updater.run_id = run_id log_path = self.__project_path.joinpath(name, "BOUT.log.0") if log_path.is_file(): log_reader = LogReader(log_path) if log_reader.started(): start_time = log_reader.start_time # Assert to prevent "Incompatible types in assignment" with Optional if start_time is None: raise RuntimeError( "log_reader.start_time is None although " "log_reader.started is True") metadata_updater.update_start_time(start_time) latest_status = self.__check_if_stopped( log_reader, metadata_updater) else: # No started time is found in the log latest_status = self.check_if_running_or_errored( log_reader) else: # No log file exists # NOTE: This means that the execution is either in a # queue or has failed the submission. # For now, we still consider this as submitted # This can maybe be decided by checking either the # pid or the status from the submitter latest_status = "submitted" metadata_updater.update_latest_status(latest_status) def __check_running(self, metadata_updater: MetadataUpdater, running_to_check: DataFrame) -> None: """ Check the status of all runs which has status `running`. Parameters ---------- metadata_updater : MetadataUpdater Object which updates the database running_to_check : DataFrame DataFrame containing the `id` and `name` of the runs with status `running` """ for name, run_id in running_to_check.itertuples(index=False): metadata_updater.run_id = run_id log_path = self.__project_path.joinpath(name, "BOUT.log.0") log_reader = LogReader(log_path) latest_status = self.check_if_running_or_errored(log_reader) metadata_updater.update_latest_status(latest_status) def __check_if_stopped(self, log_reader: LogReader, metadata_updater: MetadataUpdater) -> str: """ Check if a run has stopped. Parameters ---------- log_reader : LogReader The object which reads log files metadata_updater : MetadataUpdater Object which updates the database Returns ------- latest_status : str The latest status Raises ------ RuntimeError In case log_reader.ended() is True and log_reader.end_time is None """ if log_reader.ended(): end_time = log_reader.end_time # Assert to prevent "Incompatible types in assignment" with Optional if end_time is None: raise RuntimeError("log_reader.end_time is None although " "log_reader.ended() is True") metadata_updater.update_stop_time(end_time) latest_status = "complete" else: latest_status = self.check_if_running_or_errored(log_reader) return latest_status @staticmethod def check_if_running_or_errored(log_reader: LogReader) -> str: """ Check if a run is still running or has errored. Parameters ---------- log_reader : LogReader The object which reads log files Returns ------- latest_status : str The latest status """ pid = log_reader.pid if pid is None: latest_status = "created" elif psutil.pid_exists(pid): latest_status = "running" else: latest_status = "error" return latest_status