def run(self):
"""Shows dialog with version information."""
# TODO: add link to sites
version_file = PLUGIN_DIR / "version.rst"
version = version_file.read_text().rstrip()
pop_up_info(
"3Di Toolbox version %s" % version, "About", self.iface.mainWindow()
)
def pop_up_unkown_datasource_type():
msg = (
"QGIS3 works with ThreeDiToolbox >v1.6 and can only handle \n"
"results created after March 2018 (groundwater release). \n\n"
"You can do two things: \n"
"1. simulate this model again and load the result in QGIS3 \n"
"2. load this result into QGIS2.18 ThreeDiToolbox v1.6 "
)
logger.error(msg)
pop_up_info(msg, title="Error")
def pop_up_finished(self):
header = "Raster checker is finished"
if self.need_to_create_shp:
msg = (
"The check results have been written to: \n %s \n "
"The coordinates of wrong pixels are written to: \n"
"%s" % (self.results.log_path, self.shape_path)
)
else:
msg = "The check results have been written to:\n%s" % self.results.log_path
pop_up_info(msg, header)
def run(self):
"""Show dialog with a simple clickable link to the logfile.
Later on, we could also show the entire logfile inside the dialog. Or
suggest an email. The clickable link is OK for now.
Note: such a link does not work within the development docker.
"""
title = "Show logfile"
location = qlogging.logfile_path()
message = "Logfile location: <a href='file:///%s'>%s</a>" % (location, location)
pop_up_info(message, title, self.iface.mainWindow())
def new_file_event(self, file_path):
"""New file has been selected by the user
Try to read in the timestamps from the file
"""
if file_path == '':
self.reset()
return
try:
with h5py.File(file_path, 'r') as results:
timestamps = results['time'].value
self.set_timestamps(timestamps)
except Exception as e:
logger.exception(e)
pop_up_info(
msg=
"Unable to read the file, see the logging for more information."
)
self.reset()
def run(self):
"""Find cached spatialite and csv layer files for *ALL* items in the
TimeseriesDatasourceModel (i.e., *ALL* rows) object and delete them.
"""
# TODO: can ts_datasources tell us its cached files? Or can we order it
# to clean up its cache? (Instead of us poking around in its internals).
spatialite_filepaths = [
item.sqlite_gridadmin_filepath()
for item in self.ts_datasources.rows
if os.path.exists(item.sqlite_gridadmin_filepath())
]
# Note: convert to set because duplicates are possible if the same
# datasource is loaded multiple times
cached = set(spatialite_filepaths)
if not cached:
pop_up_info("No cached files found.")
return
# Files linked to the layers in the map registry are held open by
# Windows. You need to delete them manually from the registry to be
# able to remove the underlying data. Note that deleting the layer
# from the legend doesn't necessarily delete the layer from the map
# registry, even though it may appear that no more layers are loaded
# visually.
# The specific error message (for googling):
# "error 32 the process cannot access the file because it is being used
# by another process"
all_layers = list(QgsProject.instance().mapLayers().values())
loaded_layers = [
layer
for layer in all_layers
if any(identifier in layer.name() for identifier in IDENTIFIER_LIKE)
]
loaded_layer_ids = [layer.id() for layer in loaded_layers]
yes = pop_up_question(
"The following files will be deleted:\n"
+ ",\n".join(cached)
+ "\n\nContinue?"
)
if yes:
try:
QgsProject.instance().removeMapLayers(loaded_layer_ids)
except RuntimeError:
logger.exception("Failed to delete map layers")
for cached_spatialite_file in cached:
try:
os.remove(cached_spatialite_file)
except OSError:
msg = "Failed to delete %s." % cached_spatialite_file
logger.exception(msg)
pop_up_info(msg)
pop_up_info(
"Cache cleared. You may need to restart QGIS and reload your data."
)
def _install_h5py(hdf5_version: str):
if hdf5_version not in SUPPORTED_HDF5_VERSIONS:
# raise a error because we cannot continue
message = f"Unsupported HDF5 version: {hdf5_version}. " \
f"The following HDF5 versions are supported: {SUPPORTED_HDF5_VERSIONS}"
raise RuntimeError(message)
use_pypi = hdf5_version == "1.10.5"
# In case the (old) h5py library is already imported, we cannot uninstall
# h5py because the windows acquires a lock on the *.dll-files. Therefore
# we need to restart Qgis.
# _uninstall_dependency(H5PY_DEPENDENCY)
try:
_install_dependencies([H5PY_DEPENDENCY],
target_dir=_dependencies_target_dir(),
use_pypi=use_pypi)
except RuntimeError:
from ThreeDiToolbox.utils.user_messages import pop_up_info
pop_up_info(
"Please restart QGIS to complete the installation process of "
"ThreediToolbox.",
title="Restart required")
return
H5pyMarker.create(hdf5_version)
def handle_log_in(self):
"""Handle logging in and populating DownloadableResultModel."""
# Get the username and password
username = self.login_dialog.user_name_input.text()
password = self.login_dialog.user_password_input.text()
if username == "" or password == "":
pop_up_info("Username or password cannot be empty.")
return
try:
scenarios_endpoint = Endpoint(username=username,
password=password,
endpoint="scenarios")
endpoint = scenarios_endpoint.get_paginated(page_size=10)
except HTTPError as e:
logger.exception("Error trying to log in")
if e.code == 401:
pop_up_info("Incorrect username and/or password.")
else:
pop_up_info(str(e))
return
self.set_logged_in_status(username, password)
self.toggle_login_interface()
# don't persist info in the dialog: useful when logged out
self.login_dialog.user_name_input.clear()
self.login_dialog.user_password_input.clear()
# start thread
self.thread = DownloadWorker(endpoint=endpoint,
username=username,
password=password)
self.thread.connection_failure.connect(self.handle_connection_failure)
self.thread.output.connect(self.update_download_result_model)
self.thread.start()
# return to widget
self.login_dialog.close()
def handle_download(self):
result_type_codes_download = [
"logfiles",
# non-groundwater codes
"subgrid_map",
"flow-aggregate",
"id-mapping",
# groundwater codes
"results-3di",
"aggregate-results-3di",
"grid-admin",
]
selection_model = self.dialog.downloadResultTableView.selectionModel()
proxy_indexes = selection_model.selectedIndexes()
if len(proxy_indexes) != 1:
pop_up_info("Please select one result.")
return
proxy_selection_index = proxy_indexes[0]
selection_index = self.dialog.download_proxy_model.mapToSource(
proxy_selection_index)
item = self.downloadable_results.rows[selection_index.row()]
to_download = [
r for r in item.results.value
if r["result_type"]["code"] in result_type_codes_download
]
to_download_urls = [dl["attachment_url"] for dl in to_download]
logger.debug(item.name.value)
# ask user where to store download
directory = QFileDialog.getExistingDirectory(None,
"Choose a directory",
os.path.expanduser("~"))
if not directory:
return
dir_name = get_valid_filename(item.name.value)
self.download_directory = os.path.join(directory, dir_name)
# For now, only work with empty directories that we create ourselves.
# Because the files are downloaded and processed in chunks, we cannot
# guarantee data integrity with existing files.
if os.path.exists(self.download_directory):
pop_up_info("The directory %s already exists." %
self.download_directory)
return
logger.info("Creating download directory.")
os.mkdir(self.download_directory)
logger.debug(self.download_directory)
CHUNK_SIZE = 16 * 1024
# Important note: QNetworkAccessManager is asynchronous, which means
# the downloads are processed asynchronous using callbacks.
for url in to_download_urls:
request = QNetworkRequest(QUrl(url))
request.setRawHeader(b"username", bytes(self.username, "utf-8"))
request.setRawHeader(b"password", bytes(self.password, "utf-8"))
request.setAttribute(USER_DOWNLOAD_DIRECTORY,
self.download_directory)
reply = self.network_manager.get(request)
# Get replies in chunks, and process them
reply.setReadBufferSize(CHUNK_SIZE)
reply.readyRead.connect(
self.on_single_download_ready_to_read_chunk)
reply.finished.connect(self.on_single_download_finished)
pop_up_info("Download started.")
def pop_up_finished_or_question(self):
"""3 things (columns below) can be true or false. Dependent on that we
return a pop_up_info (user clicks okay),
pop_up_question (user clicks yes/no), Assertionerror
self.results.nr_error_logrows self.need_to_create_shp self.too_many_wrong_pixels (more rows than shp can handle)
count_error > 0 shp contains pixels too many pixels for shp
1. True False False --> pop_up_info
2. True False True --> pop_up_info + warning
3. True True False --> pop_up_question
4. True True True --> pop_up_question + warning
5. False False False --> pop_up_info
6. False False True --> raise AssertionError
7. False True False --> raise AssertionError
8. False True True --> raise AssertionError
"""
nr_errors = self.results.nr_error_logrows
create_shp = self.need_to_create_shp
too_many_wrong_pixels = self.too_many_wrong_pixels
nr_warnings = self.results.nr_warning_logrows
header = "Raster checker is finished"
question = "Do you want to add .shp to current view?"
# case 1
if nr_errors > 0 and not create_shp and not too_many_wrong_pixels:
# pop_up_info
msg = (
"Found %d errors, %d warnings (see .log) and no wrong pixels. \n\n"
"The check results have been written to: \n "
"%s" % (nr_errors, nr_warnings, self.results.log_path)
)
pop_up_info(msg, header)
# case 2
elif nr_errors > 0 and not create_shp and too_many_wrong_pixels:
# pop_up_info + warning
msg = (
"Found %d errors, %d warnings (see .log). \n"
"Found too many wrong pixels to write to .shp file "
"(see .log). \n\n "
"The check results have been written to: \n "
"%s" % (nr_errors, nr_warnings, self.results.log_path)
)
pop_up_info(msg, header)
# case 3
elif nr_errors > 0 and create_shp and not too_many_wrong_pixels:
# pop_up_question
msg = (
"Found %d errors, %d warnings and some wrong pixels. \n\n "
"The check results have been written to: \n %s \n\n "
"The coordinates of wrong pixels are written to: \n %s"
% (nr_errors, nr_warnings, self.results.log_path, self.shape_path)
)
pop_up_info(msg, header)
if pop_up_question(question, "Add shapefile?"):
self.add_shp_to_iface()
# case 4
elif nr_errors > 0 and create_shp and too_many_wrong_pixels:
# pop_up_question + warning
msg = (
"Found %d errors, %d warnings and some wrong pixels. \n "
"Also found for 1 or more rasters too many wrong pixels "
"to write to .shp file. \n\n"
"The check results have been written to: \n %s \n\n "
"The coordinates of wrong pixels are written to: \n %s"
% (nr_errors, nr_warnings, self.results.log_path, self.shape_path)
)
pop_up_info(msg, header)
if pop_up_question(question, "Add shapefile?"):
self.add_shp_to_iface()
# case 5
elif nr_errors == 0 and not create_shp and not too_many_wrong_pixels:
# pop_up_info()
msg = (
"Found %d errors, %d warnings (see .log) and no wrong pixels. \n\n "
"The check results have been written to: \n "
"%s" % (nr_errors, nr_warnings, self.results.log_path)
)
pop_up_info(msg, header)
# scenario 6, 7, or 8
elif nr_errors == 0 and create_shp ^ too_many_wrong_pixels:
raise AssertionError("this result combination is impossible")
def handle_connection_failure(self, status, reason):
pop_up_info("Something went wrong trying to connect to "
"lizard: {0} {1}".format(status, reason))
self.handle_log_out()
def select_ts_datasource(self):
"""
Open File dialog for selecting netCDF result files, triggered by button
:return: boolean, if file is selected
"""
settings = QSettings("3di", "qgisplugin")
try:
init_path = settings.value("last_used_datasource_path", type=str)
except TypeError:
logger.debug(
"Last used datasource path is no string, setting it to our home dir."
)
init_path = os.path.expanduser("~")
filename, __ = QFileDialog.getOpenFileName(
self,
"Open resultaten file",
init_path,
"NetCDF (subgrid_map.nc results_3di.nc)",
)
if filename:
# Little test for checking if there is an id mapping file available
# If not we check if an .h5 file is available
# If not we're not going to proceed
datasource_type = detect_netcdf_version(filename)
logger.info("Netcdf result file selected: %s, type is %s",
filename, datasource_type)
if datasource_type == "netcdf-groundwater":
try:
find_h5_file(filename)
except FileNotFoundError:
logger.warning(
"Groundwater h5 not found (%s), warning the user.",
filename)
pop_up_info(
"You selected a netcdf that was created "
"(after May 2018) with a 3Di calculation"
"core that is able to include groundwater"
" calculations. The ThreeDiToolbox reads "
"this netcdf together with an .h5 file, we "
"could however not find this .h5 file. Please "
"add this file next to the netcdf and try "
"again",
title="Error",
)
return False
elif datasource_type == "netcdf":
logger.warning(
"Result file (%s) version is too old. Warning the user.",
filename)
pop_up_info(
"The selected result data is too old and no longer "
"supported in this version of ThreediToolbox. Please "
"recalculate the results with a newer version of the "
"threedicore or use the ThreediToolbox plugin for QGIS 2",
title="Error",
)
# TODO: the below "return False" was previously missing. Check
# if Reinout was right in adding it :-)
return False
items = [{
"type": datasource_type,
"name": os.path.basename(filename).lower().rstrip(".nc"),
"file_path": filename,
}]
self.ts_datasources.insertRows(items)
settings.setValue("last_used_datasource_path",
os.path.dirname(filename))
return True
return False
def run_it(self, threedi_db, output_file_path):
"""Apply the threedi-modelchecker to `threedi_db`
The connection to the `threedi_db` and its south_migration_history are first
validated. Next, any model errors are written to `output_file_path` as csv file.
"""
logger.info("Starting threedi-modelchecker")
try:
model_checker = ThreediModelChecker(threedi_db)
model_checker.db.check_connection()
except OperationalError as exc:
logger.exception("Failed to start a connection with the database.")
pop_up_info(
"Something went wrong trying to connect to the database, please check"
" the connection settings: %s" % exc.args[0])
return
except errors.MigrationMissingError:
logger.exception(
"The selected 3Di model does not have the latest migration")
pop_up_info(
"The selected 3Di model does not have the latest migration, please "
"migrate your model to the latest version. Download the latest "
"version of the model here: <a href='https://3di.lizard.net/models/'>https://3di.lizard.net/models/</a>" # noqa
)
return
except errors.MigrationTooHighError:
logger.exception(
"The selected 3Di model has a higher migration than expected.")
pop_up_info(
"The 3Di model has a higher migration than expected, do you have "
"the latest version of ThreediToolbox?")
return
except errors.MigrationNameError:
logger.exception(
"Unexpected migration name, but migration id is matching. "
"We are gonna continue for now and hope for the best.")
_, output_filename = os.path.split(output_file_path)
session = model_checker.db.get_session()
total_checks = len(model_checker.config.checks)
try:
with progress_bar(self.iface, max_value=total_checks) as pb, open(
output_file_path, "w", newline="") as output_file:
writer = csv.writer(output_file)
writer.writerow([
"id", "table", "column", "value", "description",
"type of check"
])
for i, check in enumerate(model_checker.checks()):
model_errors = check.get_invalid(session)
for error_row in model_errors:
writer.writerow([
error_row.id,
check.table.name,
check.column.name,
getattr(error_row, check.column.name),
check.description(),
check,
])
pb.setValue(i)
except PermissionError:
# PermissionError happens for example when a user has the file already open
# with Excel on Windows, which locks the file.
logger.error("Unable to write to file %s", output_file_path)
pop_up_info(
"Not enough permissions to write the file '%s'.\n\n"
"The file might be used by another program. Please close all "
"other programs using the file or select another output "
"file." % output_file_path,
title="Warning",
)
return
logger.info("Successfully finished running threedi-modelchecker")
messagebar_message(
"Info",
"Finished running schematisation-checker",
level=Qgis.Success,
duration=5,
)
return True