def supervising_user_input(self, msg):
messagebar_message("Ready", msg, level=0, duration=4)
# load the levee layer so we can check for intersections with it
iter = self.connected_pnt_lyr.getFeatures()
try:
_feat = max(iter, key=lambda f: f["id"])
self._feat_id = _feat["id"] + 1
except ValueError:
self._feat_id = 1
self.connected_pnt_lyr.startEditing()
self._added_features = []
self._moved_features = []
self.connected_pnt_lyr.featureAdded.connect(self.add_feature)
self.connected_pnt_lyr.geometryChanged.connect(self.move_feature)
self.connected_pnt_lyr.editCommandEnded.connect(
self.on_edit_command_ended)
self.fnames_connected_pnt = [
field.name() for field in self.connected_pnt_lyr.fields()
]
self.fnames_calc_pnt = [
field.name() for field in self.calc_pnt_lyr.fields()
]
def run_it(self, action_list, only_empty_fields, db_set, db_type):
db = ThreediDatabase(db_set, db_type)
guesser = guess_indicators_utils.Guesser(db)
msg = guesser.run(action_list, only_empty_fields)
messagebar_message("Guess indicators ready", msg, duration=20)
logger.info("Guess indicators ready.\n" + msg)
def run_import(self):
"""
main function for performing all import tasks
"""
# self.db.create_and_check_fields()
if self.file_type == "sufhyd":
data = self.load_sufhyd_data()
self.check_import_data(data)
self.transform_import_data(data)
commit_counts = self.write_data_to_db(data)
logger.warning("Summary of import:\n" + self.log.get_summary())
# write logging to file
log_file = open(self.import_file + ".log", "w")
log_file.write("Import on {0} of file: {1}.\n".format(
datetime.datetime.now().strftime("%d-%m-%Y %H:%M:%S"),
self.import_file,
))
db_set = copy(self.db.settings)
if "password" in db_set:
del db_set["password"]
if "username" in db_set:
del db_set["username"]
log_file.write(
"Added to the {0} database with connection settings {1} :\n".
format(self.db.db_type, str(db_set)))
log_file.write("{profiles} profiles\n"
"{manholes} manholes\n"
"{pipes} pipes\n"
"{structures} structures"
"{outlets} outlets\n"
"{impervious_surfaces} impervious surfaces\n"
"".format(**commit_counts))
log_file.write(self.log.get_full_log())
log_file.close()
msg = ("{errors} errors and {warnings} warnings, see qgis log for "
"the summary and {log_file} for the full log".format(
errors=self.log.level_count.get(logging.ERROR, 0),
warnings=self.log.level_count.get(logging.WARNING, 0),
log_file=log_file,
))
messagebar_message("sufhyd import ready", msg, duration=20)
logger.info("sufhyd import ready = " + msg)
def add_objects(self, layer, features):
"""
:param layer: layer of features
:param features: Qgis layer features to be added
:return: boolean: new objects are added
"""
# Get the active database as URI, conn_info is something like:
# u"dbname='/home/jackieleng/git/threedi-turtle/var/models/
# DS_152_1D_totaal_bergingsbak/results/
# DS_152_1D_totaal_bergingsbak_result.sqlite'"
if layer.name() not in ("flowlines", "nodes", "pumplines"):
msg = """Please select results from either the 'flowlines', 'nodes' or
'pumplines' layer."""
messagebar_message("Info", msg, level=0, duration=5)
return
conn_info = QgsDataSourceUri(
layer.dataProvider().dataSourceUri()).connectionInfo()
try:
filename = conn_info.split("'")[1]
except IndexError:
raise RuntimeError(
"Active database (%s) doesn't look like an sqlite filename" %
conn_info)
# get attribute information from selected layers
existing_items = [
"%s_%s" % (item.object_type.value, str(item.object_id.value))
for item in self.model.rows
]
items = self.get_new_items(layer, features, filename, existing_items)
if len(items) > 20:
msg = ("%i new objects selected. Adding those to the plot can "
"take a while. Do you want to continue?" % len(items))
reply = QMessageBox.question(self, "Add objects", msg,
QMessageBox.Yes, QMessageBox.No)
if reply == QMessageBox.No:
return False
self.model.insertRows(items)
msg = "%i new objects added to plot " % len(items)
skipped_items = len(features) - len(items)
if skipped_items > 0:
msg += "(skipped %s already present objects)" % skipped_items
statusbar_message(msg)
return True
def _handle_added(self, feature_id):
"""
Actually add the feature to the layer.
"""
try:
self.connected_pnt_lyr.beginEditCommand(
u"Add to connected_pnt_lyr")
connected_pnt, feat = self._get_connected_pnt_feature(feature_id)
calculation_pnt_id = connected_pnt["calculation_pnt_id"]
calc_pnt, calc_pnt_feat = self._get_calculation_pnt_feature(
calculation_pnt_id)
if calc_pnt is None:
self.connected_pnt_lyr.deleteFeature(feature_id)
current_calc_type = calc_pnt["calc_type"]
request = QgsFeatureRequest().setFilterExpression(
u'"calculation_pnt_id" = {}'.format(calculation_pnt_id))
selected_features = self.connected_pnt_lyr.getFeatures(request)
# QgsFeatureRequest has no count so we have to loop through the
# feature set to get a count
unique_ids = set()
for item in selected_features:
_item = dict(
list(zip(self.fnames_connected_pnt, item.attributes())))
unique_ids.add(_item["id"])
thresh = constants.CONNECTED_PNTS_THRESHOLD[current_calc_type]
if len(unique_ids) > thresh:
msg = ("Calculation type {} allows only for {} "
"connected points! "
"Deleting point...".format(current_calc_type, thresh))
messagebar_message("Error", msg, level=2, duration=3)
self.connected_pnt_lyr.deleteFeature(feature_id)
if feature_id < 0:
feat.setAttribute("id", self._feat_id)
exchange_level = connected_pnt["exchange_level"]
if exchange_level is None:
exchange_level = -9999
feat.setAttribute("exchange_level", exchange_level)
levee_id = self.find_levee_intersection(calc_pnt_feat, feat)
if levee_id:
feat.setAttribute("levee_id", levee_id)
intersect_msg = "Created a new crevasse location at levee {}.".format(
levee_id)
messagebar_message("Info", intersect_msg, level=0, duration=5)
self.connected_pnt_lyr.updateFeature(feat)
self.connected_pnt_lyr.endEditCommand()
except Exception:
self.connected_pnt_lyr.destroyEditCommand()
raise
def run_it(self, breach_loc, auto_commit):
"""
execute the tool
:param breach_loc: threedi_schema_edits.breach_location.BresLocation
instance
:param auto_commit: save the potential breach location directly to
the database (only in case the dry-run option has not been
selected)
"""
breach_location = breach_loc
if not breach_location.has_valid_selection:
msg = "You need to select at least two connection points"
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
return
calc_points_dict = breach_location.get_calc_points_by_content()
cnt_iterations = len(calc_points_dict)
cnt = 1
with progress_bar(self.iface) as pb:
for key, values in calc_points_dict.items():
calc_type = key[1]
connected_points_selection = breach_location.get_connected_points(
values, calc_type)
breach_location.move_points_behind_levee(
connected_points_selection, calc_type)
current = (cnt / float(cnt_iterations)) * 100
pb.setValue(current)
cnt += 1
if breach_location.is_dry_run:
breach_location.pnt_layer.commitChanges()
breach_location.pnt_layer.updateExtents()
breach_location.line_layer.updateExtents()
QgsProject.instance().addMapLayers(
[breach_location.pnt_layer, breach_location.line_layer])
if auto_commit:
breach_location.connected_pnt_lyr.commitChanges()
breach_location.connected_pnt_lyr.updateExtents()
self.iface.mapCanvas().refresh()
breach_location.connected_pnt_lyr.triggerRepaint()
if not breach_location.is_dry_run:
msg = "Created {} potential breach locations".format(
breach_location.cnt_moved_pnts)
messagebar_message("Finished", msg, level=Qgis.Success, duration=8)
def _get_connected_pnt_feature(self, feature_id):
"""
:param feature_id: id of the connected point feature
:returns if the feature does not exist a tuple of None,
otherwise a tuple of an dict of {<column_name>: <attribute>, ...}
and an pyqgis feature instance
"""
try:
feat = next(
self.connected_pnt_lyr.getFeatures(
QgsFeatureRequest(feature_id)))
except StopIteration:
msg = "The connected point... does not exist."
messagebar_message("Error", msg, level=2, duration=4)
return None, None
connected_pnt = dict(
list(zip(self.fnames_connected_pnt, feat.attributes())))
return connected_pnt, feat
def _get_active_parameter_config(self):
active_ts_datasource = self.root_tool.timeslider_widget.active_ts_datasource
if active_ts_datasource is not None:
# TODO: just taking the first datasource, not sure if correct:
threedi_result = active_ts_datasource.threedi_result()
available_subgrid_vars = threedi_result.available_subgrid_map_vars
available_agg_vars = threedi_result.available_aggregation_vars
if not available_agg_vars:
messagebar_message("Warning",
"No aggregation netCDF was found.",
level=1,
duration=5)
parameter_config = generate_parameter_config(
available_subgrid_vars, available_agg_vars)
else:
parameter_config = {"q": {}, "h": {}}
return parameter_config
def _get_calculation_pnt_feature(self, calculation_pnt_id):
"""
:param calculation_pnt_id: id of the calculation point
:returns if the feature does not exist a tuple of None,
otherwise a tuple of an dict of {<column_name>: <attribute>, ...}
and an pyqgis feature instance
"""
calc_pnt_request = QgsFeatureRequest().setFilterExpression(
u'"id" = {}'.format(calculation_pnt_id))
try:
calc_pnt_feat = next(
self.calc_pnt_lyr.getFeatures(calc_pnt_request))
except StopIteration:
msg = "The calculation point ID you provided does not exist."
messagebar_message("Error", msg, level=2, duration=4)
return None, None
calc_pnt = dict(
list(zip(self.fnames_calc_pnt, calc_pnt_feat.attributes())))
return calc_pnt, calc_pnt_feat
def delete_from_database(self, table_name, where=""):
"""
Function to delete data from a table (table_name).
Args
(str) table_name: The table name of a spatialite or postgres
database
(str) where: A where clause for the delete statement.
"""
try:
with self.engine.connect() as con:
con.execute("""DELETE FROM {table}{where};""".format(
table=table_name, where=where))
except OperationalError as e:
logger.exception("Error deleting from table %s", table_name)
msg = str(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
except ProgrammingError as e:
logger.exception("Error deleting from table %s", table_name)
msg = str(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
except Exception as e:
logger.exception("Error deleting from table %s", table_name)
msg = "An unknown exception occured: {}".format(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
def insert_into_table(self, table_name, attributes):
"""
Function to insert data in a table (table_name).
The list of values is inserted in the list of attributes.
Args
(str) table_name: The table name of a spatialite or postgres
database
(dict) attributes: A dictionary of attributes to insert into
the table.
"""
attribute_names = ""
attribute_values = ""
for key, value in attributes.items():
if attribute_names != "":
attribute_names += ", '{}'".format(str(key))
else:
attribute_names += "'{}'".format(str(key))
if attribute_values != "":
attribute_values += ", '{}'".format(str(value))
else:
attribute_values += "'{}'".format(str(value))
try:
with self.engine.connect() as con:
con.execute(
"""INSERT INTO {table} ({attributes}) VALUES ({values});"""
.format(
table=table_name,
attributes=attribute_names,
values=attribute_values,
))
except OperationalError as e:
logger.exception("Error inserting into table %s", table_name)
msg = str(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
except ProgrammingError as e:
logger.exception("Error inserting into table %s", table_name)
msg = str(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
except Exception as e:
logger.exception("Error inserting into table %s", table_name)
msg = "An unknown exception occured: {}".format(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
def get_attributes(self, table_name, attribute_name, all_features=False):
"""
Get all values of an attribute from a table.
Args:
(str) table_name: The table name of a spatialite or postgres
database.
(str) attribute_name: The name of the attribute of a spatialite or
postgres database.
Returns:
(list) list_of_attributes: A list of all the attribute values
of the table.
The attribute values are strings.
"""
list_of_attributes = []
try:
with self.engine.connect() as con:
rs = con.execute("""SELECT {attribute} FROM {table};""".format(
attribute=attribute_name, table=table_name))
attributes = rs.fetchall()
if all_features is False:
list_of_attributes = [
str(attribute_value[0])
for attribute_value in attributes
]
else:
list_of_attributes += attributes
except OperationalError as e:
logger.exception("Error grabbing list of attributes")
msg = str(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
except ProgrammingError as e:
logger.exception("Error grabbing list of attributes")
msg = str(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
except Exception as e:
logger.exception("Error grabbing list of attributes")
msg = "An unknown exception occured: {}".format(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
return list_of_attributes
def get_features_with_where_clause(self, table_name, attribute_name,
where):
"""
Get all values of an attribute from a table.
Args:
(str) table_name: The table name of a spatialite or postgres
database.
(str) attribute_name: The name of the attribute of a spatialite or
postgres database.
(str) where: The where clause for the sql statement.
Returns:
(list) list_of_features: A list of all the features
of the table.
The features are tuples.
"""
list_of_features = []
try:
with self.engine.connect() as con:
rs = con.execute(
"""SELECT {attribute} FROM {table} WHERE {where};""".
format(attribute=attribute_name,
table=table_name,
where=where))
features = rs.fetchall()
list_of_features = [feature for feature in features]
except OperationalError as e:
logger.exception("Error grabbing list of features")
msg = str(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
except ProgrammingError as e:
logger.exception("Error grabbing list of features")
msg = str(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
except Exception as e:
logger.exception("Error grabbing list of features")
msg = "An unknown exception occured: {}".format(e)
messagebar_message("Error", msg, level=Qgis.Critical, duration=5)
return list_of_features
def run_it(self, db_set, db_type):
"""
:param db_set: dict of database settings. Expected keywords:
'host': '',
'port': '',
'name': '',
'username': '',
'password': '',
'schema': '',
'database': '',
'db_path': ,
:param db_type: 'spatialite' or 'postgres'
"""
predictor = Predictor(db_type)
uri = predictor.get_uri(**db_set)
calc_pnts_lyr = predictor.get_layer_from_uri(
uri, constants.TABLE_NAME_CALC_PNT, "the_geom"
)
self.connected_pnts_lyr = predictor.get_layer_from_uri(
uri, constants.TABLE_NAME_CONN_PNT, "the_geom"
)
predictor.start_sqalchemy_engine(db_set)
if not self.fresh_start(predictor):
return
default_epsg_code = 28992
epsg_code = predictor.get_epsg_code() or default_epsg_code
logger.info(
"[*] Using epsg code {} to build the calc_type_dict".format(epsg_code)
)
predictor.build_calc_type_dict(epsg_code=epsg_code)
transform = None
# spatialites are in WGS84 so we need a transformation
if db_type == "spatialite":
transform = "{epsg_code}:4326".format(epsg_code=epsg_code)
succces, features = predictor.predict_points(
output_layer=calc_pnts_lyr, transform=transform
)
if succces:
msg = "Predicted {} calculation points".format(len(features))
level = 3
QgsProject.instance().addMapLayer(calc_pnts_lyr)
else:
msg = (
"Predicted calculation points failed! "
'Are you sure the table "v2_calculation_point" '
"is empty?".format()
)
level = 1
messagebar_message("Finished", msg, level=level, duration=12)
cp_succces, cp_features = predictor.fill_connected_pnts_table(
calc_pnts_lyr=calc_pnts_lyr, connected_pnts_lyr=self.connected_pnts_lyr
)
if cp_succces:
cp_msg = "Created {} connected points template".format(len(cp_features))
cp_level = 3
QgsProject.instance().addMapLayer(self.connected_pnts_lyr)
else:
cp_msg = "Creating connected points failed!"
cp_level = 1
messagebar_message("Finished", cp_msg, level=cp_level, duration=12)
logger.info("Done predicting calcualtion points.\n" + msg)
def on_single_download_finished(self):
"""Usage: mostly for notifying the user the download has finished."""
reply = self.sender()
filename = reply.url().toString().split("/")[-1]
reply.close()
messagebar_message("Done", "Finished downloading %s" % filename)
def run_it(self, action_list, db_set, db_type):
db = ThreediDatabase(db_set, db_type)
checker = RasterChecker(db)
msg = checker.run(action_list)
messagebar_message("Raster checker ready", msg, duration=3)
logger.info("Raster checker ready")
def run_it(self, threedi_db, output_file_path):
"""Apply the threedi-modelchecker to `threedi_db`
The connection to the `threedi_db` and its south_migration_history are first
validated. Next, any model errors are written to `output_file_path` as csv file.
"""
logger.info("Starting threedi-modelchecker")
try:
model_checker = ThreediModelChecker(threedi_db)
model_checker.db.check_connection()
except OperationalError as exc:
logger.exception("Failed to start a connection with the database.")
pop_up_info(
"Something went wrong trying to connect to the database, please check"
" the connection settings: %s" % exc.args[0])
return
except errors.MigrationMissingError:
logger.exception(
"The selected 3Di model does not have the latest migration")
pop_up_info(
"The selected 3Di model does not have the latest migration, please "
"migrate your model to the latest version. Download the latest "
"version of the model here: <a href='https://3di.lizard.net/models/'>https://3di.lizard.net/models/</a>" # noqa
)
return
except errors.MigrationTooHighError:
logger.exception(
"The selected 3Di model has a higher migration than expected.")
pop_up_info(
"The 3Di model has a higher migration than expected, do you have "
"the latest version of ThreediToolbox?")
return
except errors.MigrationNameError:
logger.exception(
"Unexpected migration name, but migration id is matching. "
"We are gonna continue for now and hope for the best.")
_, output_filename = os.path.split(output_file_path)
session = model_checker.db.get_session()
total_checks = len(model_checker.config.checks)
try:
with progress_bar(self.iface, max_value=total_checks) as pb, open(
output_file_path, "w", newline="") as output_file:
writer = csv.writer(output_file)
writer.writerow([
"id", "table", "column", "value", "description",
"type of check"
])
for i, check in enumerate(model_checker.checks()):
model_errors = check.get_invalid(session)
for error_row in model_errors:
writer.writerow([
error_row.id,
check.table.name,
check.column.name,
getattr(error_row, check.column.name),
check.description(),
check,
])
pb.setValue(i)
except PermissionError:
# PermissionError happens for example when a user has the file already open
# with Excel on Windows, which locks the file.
logger.error("Unable to write to file %s", output_file_path)
pop_up_info(
"Not enough permissions to write the file '%s'.\n\n"
"The file might be used by another program. Please close all "
"other programs using the file or select another output "
"file." % output_file_path,
title="Warning",
)
return
logger.info("Successfully finished running threedi-modelchecker")
messagebar_message(
"Info",
"Finished running schematisation-checker",
level=Qgis.Success,
duration=5,
)
return True
