def __init__(self, sqlite_path: Path):
"""
param global_settings_id: if None the first global setting entry is taken (default)
"""
self.sqlite_path = sqlite_path
self.database = ThreediDatabase(
connection_settings={"db_path": self.sqlite_path.as_posix()},
db_type="spatialite",
)
def south_latest_sqlite(tmp_path):
"""An empty SQLite that is in its latest South migration state"""
tmp_sqlite = tmp_path / "south_latest.sqlite"
shutil.copyfile(os.path.join(data_dir, "south_latest.sqlite"), tmp_sqlite)
return ThreediDatabase({"db_path": tmp_sqlite},
db_type="spatialite",
echo=False)
def oldest_sqlite(tmp_path):
"""A real SQLite that is in its oldest possible south migration state (160)"""
tmp_sqlite = tmp_path / "noordpolder.sqlite"
shutil.copyfile(os.path.join(data_dir, "noordpolder.sqlite"), tmp_sqlite)
return ThreediDatabase({"db_path": tmp_sqlite},
db_type="spatialite",
echo=False)
def run_modelchecker(**kwargs):
print('Starting modelchecker')
sqlite_path = kwargs.get('sqlite_path')
log_path = sqlite_path.split(".")[0] + "_modelchecker.csv"
if os.path.exists(log_path):
os.remove(log_path)
database = ThreediDatabase(connection_settings={"db_path": sqlite_path},
db_type="spatialite")
model_checker = ThreediModelChecker(database)
session = model_checker.db.get_session()
with open(log_path, "w", newline="") as output_file:
writer = csv.writer(output_file)
writer.writerow(
["id", "table", "column", "value", "description", "type of check"])
for i, check in enumerate(model_checker.checks()):
model_errors = check.get_invalid(session)
for error_row in model_errors:
writer.writerow([
error_row.id,
check.table.name,
check.column.name,
getattr(error_row, check.column.name),
check.description(),
check,
])
def threedi_modelchecker(ctx, sqlite, database, host, port, username,
password):
"""Checks the threedi-model for errors / warnings / info messages"""
ctx.ensure_object(dict)
if sqlite:
sqlite_settings = {"db_path": sqlite, "db_file": sqlite}
db = ThreediDatabase(connection_settings=sqlite_settings,
db_type="spatialite",
echo=False)
else:
postgis_settings = {
"host": host,
"port": port,
"database": database,
"username": username,
"password": password,
}
db = ThreediDatabase(connection_settings=postgis_settings,
db_type="postgres",
echo=False)
ctx.obj["db"] = db
def on_run(self):
"""Handler when user presses the 'run' button.
Starts the threedi-modelchecker script on the selected database and writes the
results to the selected outputfile."""
selected_database_key = self.database_combobox.currentText()
selected_db = self.available_databases.get(selected_database_key)
db_type = selected_db.get("db_type")
connection_settings = selected_db.get("db_settings")
output_file_path = self.save_file_location_display.text()
threedi_db = ThreediDatabase(connection_settings, db_type=db_type)
success = self.command.run_it(threedi_db, output_file_path)
if success:
self.open_result_button.setEnabled(True)
def threedi_db(request):
"""Fixture which yields a empty 3di database
Fixture is parameterized to yield two types of databases: a postgis and a
spatialite database.
A global Session object is configured based on database type. This allows
the factories to operate on the same session object. See:
https://factoryboy.readthedocs.io/en/latest/orms.html#managing-sessions
"""
if request.param[0] == "postgres" and psycopg2 is None:
pytest.skip("Skipping postgres test as psycopg2 is not available.")
db = ThreediDatabase(request.param[1],
db_type=request.param[0],
echo=False)
engine = db.get_engine()
Session.configure(bind=engine)
# monkey-patch get_session
db.get_session = lambda: Session()
yield db
Session.remove()
def in_memory_sqlite():
"""An in-memory database without a schema (to test schema migrations)"""
return ThreediDatabase({"db_path": ""}, db_type="spatialite", echo=False)
class SimulationTemplateExtractor(object):
def __init__(self, sqlite_path: Path):
"""
param global_settings_id: if None the first global setting entry is taken (default)
"""
self.sqlite_path = sqlite_path
self.database = ThreediDatabase(
connection_settings={"db_path": self.sqlite_path.as_posix()},
db_type="spatialite",
)
def _extract_simulation_template(
self, session: Session, global_settings_id: Optional[int] = None
) -> SimulationTemplate:
"""
Extract a SimulationTemplate instance using the given database session
"""
qr = Query(GlobalSetting).with_session(session)
if global_settings_id is not None:
qr = qr.filter(GlobalSetting.id == global_settings_id)
global_settings: GlobalSetting = qr.first()
if global_settings is None:
raise SchematisationError(
f"Global settings with id: {global_settings_id} not found."
)
dwf_laterals = DWFCalculator(session, global_settings.use_0d_inflow).laterals
initial_waterlevels = InitialWaterlevelExtractor(session, global_settings_id)
settings = SettingsExtractor(session, global_settings.id)
return SimulationTemplate(
events=Events(
structure_controls=StructureControlExtractor(
session, control_group_id=global_settings.control_group_id
).all_controls(),
laterals=LateralsExtractor(session).as_list(),
dwf_laterals=dwf_laterals,
boundaries=BoundariesExtractor(session).as_list(),
),
settings=settings.all_settings(),
initial_waterlevels=initial_waterlevels.all_initial_waterlevels(),
)
def _get_global_settings_options(
self, session: Session
) -> List[GlobalSettingOption]:
return [
GlobalSettingOption(x.id, x.name)
for x in Query(GlobalSetting).with_session(session)
]
def global_settings_options(self) -> List[GlobalSettingOption]:
try:
session = self.database.get_session()
return self._get_global_settings_options(session)
finally:
session.close()
def extract(self, global_settings_id: Optional[int] = None) -> SimulationTemplate:
"""
Return simulation template for
"""
try:
session = self.database.get_session()
return self._extract_simulation_template(session, global_settings_id)
finally:
session.close()
def check_rasters(self):
"""Run rasters checker."""
try:
from ThreeDiToolbox.tool_commands.raster_checker.raster_checker_main import RasterChecker
from ThreeDiToolbox.utils.threedi_database import ThreediDatabase
except ImportError:
raise
class PatchedRasterChecker(RasterChecker):
def run_all_checks(self, progress_bar):
"""
Overriding an existing method to use QProgressBar instance instead of feedback
"""
progress_per_phase = 100 / self.nr_phases
nr_items = len(self.entries.items())
progress_per_item = progress_per_phase / nr_items
phase = 1
progress_bar.setValue(0)
current_progress = 0
for setting_id, rasters in self.entries.items():
self.run_phase_checks(setting_id, rasters, phase)
self.results.update_result_per_phase(
setting_id, rasters, phase)
current_progress += progress_per_item
progress_bar.setValue(current_progress)
phase = 2
for setting_id, rasters in self.entries.items():
# we only check rasters that passed blocking checks previous phase
rasters_ready = self.results.get_rasters_ready(
setting_id, phase)
if rasters_ready:
self.run_phase_checks(setting_id, rasters_ready, phase)
self.results.update_result_per_phase(
setting_id, rasters, phase)
current_progress += progress_per_item
progress_bar.setValue(current_progress)
phase = 3
for setting_id, rasters in self.entries.items():
rasters_ready = self.results.get_rasters_ready(
setting_id, phase)
self.run_phase_checks(setting_id, rasters_ready, phase)
self.results.update_result_per_phase(
setting_id, rasters, phase)
current_progress += progress_per_item
progress_bar.setValue(current_progress)
phase = 4
for setting_id, rasters in self.entries.items():
rasters_ready = self.results.get_rasters_ready(
setting_id, 3)
if len(rasters_ready) >= 2 and rasters[0] in rasters_ready:
rasters_sorted = self.dem_to_first_index(
rasters, rasters_ready)
self.run_phase_checks(setting_id, rasters_sorted,
phase)
self.results.update_result_per_phase(
setting_id, rasters, phase)
current_progress += progress_per_item
progress_bar.setValue(current_progress)
phase = 5
self.input_data_shp = []
for setting_id, rasters in self.entries.items():
rasters_ready = self.results.get_rasters_ready(
setting_id, phase)
if len(rasters_ready) >= 1:
rasters_ready.insert(0, rasters[0])
self.run_phase_checks(setting_id, rasters_ready, phase)
self.results.update_result_per_phase(
setting_id, rasters, phase)
current_progress += progress_per_item
progress_bar.setValue(current_progress)
# Run raster checks
db_type = "spatialite"
db_settings = {"db_path": self.schematisation_sqlite}
try:
db = ThreediDatabase(db_settings, db_type)
checker = PatchedRasterChecker(db)
checker.run_all_checks(self.pbar_check_rasters)
checker.close_session()
except Exception as e:
error_msg = f"Raster checker errors:\n{e}"
self.communication.show_error(error_msg, self)
return
for result_row_dict in checker.results.result_per_check:
result_row_str = checker.results.result_per_check_to_msg(
result_row_dict)
result_row = result_row_str.split(",")
level = result_row[0].upper()
if level == LogLevels.INFO.value:
continue
self.raster_checker_logger.log_result_row(result_row, level)
self.pbar_check_rasters.setMaximum(100)
self.pbar_check_rasters.setValue(100)
self.communication.bar_info("Finished raster checks.")
self.pbar_check_rasters.hide()
self.lbl_check_rasters.show()
def check_schematisation(self):
"""Run schematisation checker."""
try:
from sqlalchemy.exc import OperationalError
from threedi_modelchecker.threedi_database import ThreediDatabase
from threedi_modelchecker.model_checks import ThreediModelChecker
from threedi_modelchecker.schema import ModelSchema
from threedi_modelchecker import errors
except ImportError:
raise
db_settings = {"db_path": self.schematisation_sqlite}
threedi_db = ThreediDatabase(db_settings)
schema = ModelSchema(threedi_db)
try:
schema.validate_schema()
except errors.MigrationMissingError:
warn_and_ask_msg = (
"The selected spatialite cannot be used because its database schema version is out of date. "
"Would you like to migrate your spatialite to the current schema version?"
)
do_migration = self.communication.ask(None, "Missing migration",
warn_and_ask_msg)
if not do_migration:
self.communication.bar_warn("Schematisation checks skipped!")
return
wip_revision = self.current_local_schematisation.wip_revision
backup_filepath = wip_revision.backup_sqlite()
schema.upgrade(backup=False, upgrade_spatialite_version=True)
shutil.rmtree(os.path.dirname(backup_filepath))
except errors.UpgradeFailedError:
error_msg = (
"There are errors in the spatialite. Please re-open this file in QGIS 3.16, run the model checker and "
"fix error messages. Then attempt to upgrade again. For questions please contact the servicedesk."
)
self.communication.show_error(error_msg, self)
return
except Exception as e:
error_msg = f"{e}"
self.communication.show_error(error_msg, self)
return
model_checker = None
try:
model_checker = ThreediModelChecker(threedi_db)
model_checker.db.check_connection()
except OperationalError as exc:
error_msg = (
f"Failed to start a connection with the database.\n"
f"Something went wrong trying to connect to the database, "
f"please check the connection settings: {exc.args[0]}")
self.communication.show_error(error_msg, self)
return
except errors.MigrationMissingError:
error_msg = (
"The selected 3Di model does not have the latest migration.\n"
"The selected 3Di model does not have the latest migration,"
"please migrate your model to the latest version.")
self.communication.show_error(error_msg, self)
return
except errors.MigrationTooHighError:
error_msg = (
"The selected 3Di model has a higher migration than expected.\n"
"The 3Di model has a higher migration than expected, "
"do you have the latest version of ThreediToolbox?")
self.communication.show_error(error_msg, self)
return
except errors.MigrationNameError:
warn_msg = (
"Unexpected migration name, but migration id is matching.\n"
"We are gonna continue for now and hope for the best.")
self.communication.bar_warn(warn_msg)
session = model_checker.db.get_session()
total_checks = len(model_checker.config.checks)
self.pbar_check_spatialite.setMaximum(total_checks)
self.pbar_check_spatialite.setValue(0)
results_rows = []
for i, check in enumerate(
model_checker.checks(level=LogLevels.INFO.value), start=1):
for result_row in check.get_invalid(session):
results_rows.append([
check.level.name,
check.error_code,
result_row.id,
check.table.name,
check.column.name,
getattr(result_row, check.column.name),
check.description(),
])
self.pbar_check_spatialite.setValue(i)
if results_rows:
for result_row in results_rows:
level = result_row[0].upper()
self.schematisation_checker_logger.log_result_row(
result_row, level)
self.communication.bar_info("Finished schematisation checks.")
self.pbar_check_spatialite.setValue(total_checks)
self.pbar_check_spatialite.hide()
self.lbl_check_spatialite.show()