def execute(self, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a GeoAlgorithmExecutionException in case anything goes
wrong.
"""
if feedback is None:
feedback = QgsProcessingFeedback()
self.model = model
try:
self.setOutputCRS()
self.resolveOutputs()
self.evaluateParameterValues()
self.runPreExecutionScript(feedback)
self.processAlgorithm(feedback)
feedback.setProgress(100)
self.convertUnsupportedFormats(feedback)
self.runPostExecutionScript(feedback)
except GeoAlgorithmExecutionException as gaee:
lines = [self.tr("Error while executing algorithm")]
lines.append(traceback.format_exc())
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, gaee.msg)
raise GeoAlgorithmExecutionException(gaee.msg, lines, gaee)
except Exception as e:
# If something goes wrong and is not caught in the
# algorithm, we catch it here and wrap it
lines = [self.tr("Uncaught error while executing algorithm")]
lines.append(traceback.format_exc())
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, lines)
raise GeoAlgorithmExecutionException(str(e) + self.tr("\nSee log for more details"), lines, e)
def executeAlgorithm(alg, parameters, context=None, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a QgsProcessingException in case anything goes
wrong.
:param parameters:
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
#self.model = model
#self.setOutputCRS()
#self.resolveOutputs()
#self.evaluateParameterValues()
#self.runPreExecutionScript(feedback)
result, ok = alg.run(parameters, context, feedback)
#self.processAlgorithm(parameters, context, feedback)
feedback.setProgress(100)
return result, ok
def handleAlgorithmResults(alg, context, feedback=None, showResults=True):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context, typeHint=details.layerTypeHint)
if layer is not None:
set_layer_name(layer, details)
style = None
if details.outputName:
style = RenderingStyles.getStyle(alg.id(), details.outputName)
if style is None:
if layer.type() == QgsMapLayer.RasterLayer:
style = ProcessingConfig.getSetting(ProcessingConfig.RASTER_STYLE)
else:
if layer.geometryType() == QgsWkbTypes.PointGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POINT_STYLE)
elif layer.geometryType() == QgsWkbTypes.LineGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_LINE_STYLE)
else:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POLYGON_STYLE)
if style:
layer.loadNamedStyle(style)
details.project.addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
if details.postProcessor():
details.postProcessor().postProcessLayer(layer, context, feedback)
else:
wrongLayers.append(str(l))
except Exception:
QgsMessageLog.logMessage(QCoreApplication.translate('Postprocessing', "Error loading result layer:") + "\n" + traceback.format_exc(), 'Processing', Qgis.Critical)
wrongLayers.append(str(l))
i += 1
feedback.setProgress(100)
if wrongLayers:
msg = QCoreApplication.translate('Postprocessing', "The following layers were not correctly generated.")
msg += "<ul>" + "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += QCoreApplication.translate('Postprocessing', "You can check the 'Log Messages Panel' in QGIS main window to find more information about the execution of the algorithm.")
feedback.reportError(msg)
return len(wrongLayers) == 0
def handleAlgorithmResults(alg, context, feedback=None, showResults=True):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context)
if layer is not None:
if not ProcessingConfig.getSetting(ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
layer.setName(details.name)
style = None
if details.outputName:
style = RenderingStyles.getStyle(alg.id(), details.outputName)
if style is None:
if layer.type() == QgsMapLayer.RasterLayer:
style = ProcessingConfig.getSetting(ProcessingConfig.RASTER_STYLE)
else:
if layer.geometryType() == QgsWkbTypes.PointGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POINT_STYLE)
elif layer.geometryType() == QgsWkbTypes.LineGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_LINE_STYLE)
else:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POLYGON_STYLE)
if style:
layer.loadNamedStyle(style)
details.project.addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
else:
wrongLayers.append(str(l))
except Exception:
QgsMessageLog.logMessage(QCoreApplication.translate('Postprocessing', "Error loading result layer:") + "\n" + traceback.format_exc(), 'Processing', Qgis.Critical)
wrongLayers.append(str(l))
i += 1
feedback.setProgress(100)
if wrongLayers:
msg = QCoreApplication.translate('Postprocessing', "The following layers were not correctly generated.")
msg += "<ul>" + "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += QCoreApplication.translate('Postprocessing', "You can check the 'Log Messages Panel' in QGIS main window to find more information about the execution of the algorithm.")
feedback.reportError(msg)
return len(wrongLayers) == 0
def handleAlgorithmResults(alg, feedback=None, showResults=True):
wrongLayers = []
htmlResults = False
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(
QCoreApplication.translate('Postprocessing',
'Loading resulting layers'))
i = 0
for out in alg.outputs:
feedback.setProgress(100 * i / float(len(alg.outputs)))
if out.hidden or not out.open:
continue
if isinstance(out, (OutputRaster, OutputVector, OutputTable)):
try:
if hasattr(out, "layer") and out.layer is not None:
out.layer.setName(out.description)
QgsProject.instance().addMapLayers([out.layer])
else:
if ProcessingConfig.getSetting(
ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
name = os.path.basename(out.value)
else:
name = out.description
dataobjects.load(
out.value, name, alg.crs,
RenderingStyles.getStyle(alg.commandLineName(),
out.name))
except Exception:
ProcessingLog.addToLog(
ProcessingLog.LOG_ERROR,
"Error loading result layer:\n" + traceback.format_exc())
wrongLayers.append(out.description)
elif isinstance(out, OutputHTML):
ProcessingResults.addResult(out.description, out.value)
htmlResults = True
i += 1
QApplication.restoreOverrideCursor()
if wrongLayers:
msg = "The following layers were not correctly generated.<ul>"
msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += "You can check the log messages to find more information about the execution of the algorithm"
feedback.reportError(msg)
if showResults and htmlResults and not wrongLayers:
dlg = ResultsDialog()
dlg.exec_()
return len(wrongLayers) == 0
def executeAlgorithm(alg, parameters, context=None, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a GeoAlgorithmExecutionException in case anything goes
wrong.
:param parameters:
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext()
#self.model = model
try:
#self.setOutputCRS()
#self.resolveOutputs()
#self.evaluateParameterValues()
#self.runPreExecutionScript(feedback)
result = alg.run(parameters, context, feedback)
#self.processAlgorithm(parameters, context, feedback)
feedback.setProgress(100)
return result
#self.convertUnsupportedFormats(context, feedback)
#self.runPostExecutionScript(feedback)
except GeoAlgorithmExecutionException as gaee:
lines = [self.tr('Error while executing algorithm')]
lines = []
lines.append(traceback.format_exc())
feedback.reportError(gaee.msg)
QgsMessageLog.logMessage(gaee.msg, self.tr('Processing'),
QgsMessageLog.CRITICAL)
raise GeoAlgorithmExecutionException(gaee.msg, lines, gaee)
#except Exception as e:
# If something goes wrong and is not caught in the
# algorithm, we catch it here and wrap it
#lines = [self.tr('Uncaught error while executing algorithm')]
# lines = []
# lines.append(traceback.format_exc())
#QgsMessageLog.logMessage('\n'.join(lines), self.tr('Processing'), QgsMessageLog.CRITICAL)
#raise GeoAlgorithmExecutionException(str(e) + self.tr('\nSee log for more details'), lines, e)
def helpUrl(self):
return QgsHelp.helpUrl("processing_algs/{}/{}".format(
self.provider().id(), self.id())).toString()
def executeAlgorithm(alg, parameters, context=None, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a GeoAlgorithmExecutionException in case anything goes
wrong.
:param parameters:
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext()
#self.model = model
try:
#self.setOutputCRS()
#self.resolveOutputs()
#self.evaluateParameterValues()
#self.runPreExecutionScript(feedback)
result = alg.run(parameters, context, feedback)
#self.processAlgorithm(parameters, context, feedback)
feedback.setProgress(100)
return result
#self.convertUnsupportedFormats(context, feedback)
#self.runPostExecutionScript(feedback)
except GeoAlgorithmExecutionException as gaee:
lines = [self.tr('Error while executing algorithm')]
lines = []
lines.append(traceback.format_exc())
feedback.reportError(gaee.msg)
QgsMessageLog.logMessage(gaee.msg, self.tr('Processing'), QgsMessageLog.CRITICAL)
raise GeoAlgorithmExecutionException(gaee.msg, lines, gaee)
#except Exception as e:
# If something goes wrong and is not caught in the
# algorithm, we catch it here and wrap it
#lines = [self.tr('Uncaught error while executing algorithm')]
# lines = []
# lines.append(traceback.format_exc())
#QgsMessageLog.logMessage('\n'.join(lines), self.tr('Processing'), QgsMessageLog.CRITICAL)
#raise GeoAlgorithmExecutionException(str(e) + self.tr('\nSee log for more details'), lines, e)
def helpUrl(self):
return QgsHelp.helpUrl("processing_algs/{}/{}".format(
self.provider().id(), self.id())).toString()
def handleAlgorithmResults(alg, context, feedback=None, showResults=True):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(
QCoreApplication.translate('Postprocessing',
'Loading resulting layers'))
i = 0
for out in alg.outputs:
feedback.setProgress(100 * i / float(len(alg.outputs)))
if out.hidden or not out.open:
continue
if isinstance(out, (OutputRaster, OutputVector, OutputTable)):
try:
layer = QgsProcessingUtils.mapLayerFromString(
out.value, context)
if layer:
layer.setName(out.description)
QgsProject.instance().addMapLayer(
context.temporaryLayerStore().takeMapLayer(layer))
else:
if ProcessingConfig.getSetting(
ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
name = os.path.basename(out.value)
else:
name = out.description
isRaster = True if isinstance(out, OutputRaster) else False
dataobjects.load(
out.value, name, alg.crs,
RenderingStyles.getStyle(alg.id(), out.name), isRaster)
except Exception:
QgsMessageLog.logMessage(
"Error loading result layer:\n" + traceback.format_exc(),
'Processing', QgsMessageLog.CRITICAL)
wrongLayers.append(out.description)
elif isinstance(out, OutputHTML):
resultsList.addResult(alg.icon(), out.description, out.value)
i += 1
QApplication.restoreOverrideCursor()
if wrongLayers:
msg = "The following layers were not correctly generated.<ul>"
msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += "You can check the log messages to find more information about the execution of the algorithm"
feedback.reportError(msg)
return len(wrongLayers) == 0
def handleAlgorithmResults(alg, feedback=None, showResults=True):
wrongLayers = []
htmlResults = False
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers'))
i = 0
for out in alg.outputs:
feedback.setProgress(100 * i / float(len(alg.outputs)))
if out.hidden or not out.open:
continue
if isinstance(out, (OutputRaster, OutputVector, OutputTable)):
try:
if hasattr(out, "layer") and out.layer is not None:
out.layer.setName(out.description)
QgsProject.instance().addMapLayers([out.layer])
else:
if ProcessingConfig.getSetting(
ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
name = os.path.basename(out.value)
else:
name = out.description
dataobjects.load(out.value, name, alg.crs,
RenderingStyles.getStyle(alg.commandLineName(),
out.name))
except Exception:
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
"Error loading result layer:\n" + traceback.format_exc())
wrongLayers.append(out.description)
elif isinstance(out, OutputHTML):
ProcessingResults.addResult(out.description, out.value)
htmlResults = True
i += 1
QApplication.restoreOverrideCursor()
if wrongLayers:
msg = "The following layers were not correctly generated.<ul>"
msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += "You can check the log messages to find more information about the execution of the algorithm"
feedback.reportError(msg)
if showResults and htmlResults and not wrongLayers:
dlg = ResultsDialog()
dlg.exec_()
return len(wrongLayers) == 0
def execute(self, parameters, context=None, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a QgsProcessingException in case anything goes
wrong.
:param parameters:
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
self.model = model
try:
self.setOutputCRS()
self.resolveOutputs()
self.runPreExecutionScript(feedback)
self.processAlgorithm(parameters, context, feedback)
feedback.setProgress(100)
self.convertUnsupportedFormats(context, feedback)
self.runPostExecutionScript(feedback)
except QgsProcessingException as gaee:
lines = [self.tr('Error while executing algorithm')]
lines.append(traceback.format_exc())
QgsMessageLog.logMessage(gaee.msg, self.tr('Processing'),
QgsMessageLog.CRITICAL)
raise QgsProcessingException(gaee.msg, lines, gaee)
except Exception as e:
# If something goes wrong and is not caught in the
# algorithm, we catch it here and wrap it
lines = [self.tr('Uncaught error while executing algorithm')]
lines.append(traceback.format_exc())
QgsMessageLog.logMessage('\n'.join(lines), self.tr('Processing'),
QgsMessageLog.CRITICAL)
raise QgsProcessingException(
str(e) + self.tr('\nSee log for more details'), lines, e)
def execute(self, parameters, context=None, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a QgsProcessingException in case anything goes
wrong.
:param parameters:
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
self.model = model
try:
self.setOutputCRS()
self.resolveOutputs()
self.runPreExecutionScript(feedback)
self.processAlgorithm(parameters, context, feedback)
feedback.setProgress(100)
self.convertUnsupportedFormats(context, feedback)
self.runPostExecutionScript(feedback)
except QgsProcessingException as gaee:
lines = [self.tr('Error while executing algorithm')]
lines.append(traceback.format_exc())
QgsMessageLog.logMessage(gaee.msg, self.tr('Processing'), QgsMessageLog.CRITICAL)
raise QgsProcessingException(gaee.msg, lines, gaee)
except Exception as e:
# If something goes wrong and is not caught in the
# algorithm, we catch it here and wrap it
lines = [self.tr('Uncaught error while executing algorithm')]
lines.append(traceback.format_exc())
QgsMessageLog.logMessage('\n'.join(lines), self.tr('Processing'), QgsMessageLog.CRITICAL)
raise QgsProcessingException(str(e) + self.tr('\nSee log for more details'), lines, e)
def execute(self, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a GeoAlgorithmExecutionException in case anything goes
wrong.
"""
if feedback is None:
feedback = QgsProcessingFeedback()
self.model = model
try:
self.setOutputCRS()
self.resolveOutputs()
self.evaluateParameterValues()
self.runPreExecutionScript(feedback)
self.processAlgorithm(feedback)
feedback.setProgress(100)
self.convertUnsupportedFormats(feedback)
self.runPostExecutionScript(feedback)
except GeoAlgorithmExecutionException as gaee:
lines = [self.tr('Error while executing algorithm')]
lines.append(traceback.format_exc())
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, gaee.msg)
raise GeoAlgorithmExecutionException(gaee.msg, lines, gaee)
except Exception as e:
# If something goes wrong and is not caught in the
# algorithm, we catch it here and wrap it
lines = [self.tr('Uncaught error while executing algorithm')]
lines.append(traceback.format_exc())
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, lines)
raise GeoAlgorithmExecutionException(
str(e) + self.tr('\nSee log for more details'), lines, e)
def processAlgorithm(self, parameters, context: QgsProcessingContext,
feedback: QgsProcessingFeedback):
"""Here is where the processing itself takes place."""
feedback.setProgress(0)
# init params
database = self.parameterAsString(parameters, self.DATABASE, context)
template_inp_file = self.parameterAsFile(parameters,
self.TEMPLATE_INP_FILE,
context)
inp_file = self.parameterAsFileOutput(parameters, self.INP_FILE,
context)
# Connect to QGEP database and perform translation
qs = QgepSwmm(datetime.datetime.today().isoformat(), database,
inp_file, template_inp_file, None, None, None, None)
qs.write_input()
if qs.feedbacks is not None:
for i in range(len(qs.feedbacks)):
feedback.reportError(qs.feedbacks[i])
return {self.INP_FILE: inp_file}
def processAlgorithm(self, parameters, context: QgsProcessingContext,
feedback: QgsProcessingFeedback):
"""Here is where the processing itself takes place."""
# init params
log_file = self.parameterAsFile(parameters, self.LOG_FILE, context)
output_file = self.parameterAsFile(parameters, self.OUT_FILE, context)
inp_file = self.parameterAsFileOutput(parameters, self.INP_FILE,
context)
swmm_cli = os.path.abspath(ProcessingConfig.getSetting('SWMM_PATH'))
if not swmm_cli:
# raise GeoAlgorithmExecutionException(
# 'Swmm command line toom is not configured.\n\
# Please configure it before running Swmm algorithms.')
raise QgsProcessingException(
self.tr('Swmm command line tool is not configured.\n\
Please configure it before running Swmm algorithms.'))
qs = QgepSwmm(None, None, inp_file, None, output_file, log_file,
swmm_cli, None)
prompt = qs.execute_swmm()
if qs.feedbacks is not None:
for i in range(len(qs.feedbacks)):
feedback.reportError(qs.feedbacks[i])
feedback.pushInfo(prompt)
if re.search('There are errors', prompt):
feedback.reportError(prompt)
feedback.reportError(
'There were errors, look into logs for details: {log_file}'.
format(log_file=log_file))
feedback.setProgress(100)
return {}
def processAlgorithm(self, parameters, context,
feedback: QgsProcessingFeedback):
# Input / Output
source = self.parameterAsSource(parameters, self.INPUT, context)
(sink, dest_id) = self.create_output_sink(parameters, context,
source.sourceCrs())
# Filter parameters
observation_type_indices = self.parameterAsEnums(
parameters, self.OBSERVATION_TYPE, context)
observation_types = list(
map(lambda i: self.OBSERVATION_TYPES[i][0],
observation_type_indices))
from_date = None
from_date_string = self.parameterAsString(parameters, self.FROM_DATE,
context)
if from_date_string:
from_date = datetime.fromisoformat(from_date_string)
to_date = None
to_date_string = self.parameterAsString(parameters, self.TO_DATE,
context)
if to_date_string:
to_date = datetime.fromisoformat(to_date_string)
fire_connection_string = self.settings.value("fire_connection_string")
fireDb = FireDb(fire_connection_string, debug=True)
features = list(source.getFeatures())
total_num_features = len(features)
total_num_features_processed = 0
# for current, feature in enumerate(features):
for feature in features:
if feedback.isCanceled():
return {}
wkt = feature.geometry().asWkt().upper()
geometry = Geometry(wkt)
observations = fireDb.hent_observationer_naer_geometri(
geometri=geometry, afstand=0, tidfra=from_date, tidtil=to_date)
pid_list = self.get_pids_from_observations(observations)
geometriobjekter = self.get_geometriobjekter_from_pids(
fireDb, pid_list)
idents = self.get_idents_from_pids(fireDb, pid_list)
feedback.setProgressText(
"Fandt {antal} observationer".format(antal=len(observations)))
feedback.setProgressText("Fandt {antal} geometriobjekter".format(
antal=len(geometriobjekter)))
feedback.setProgressText(
"Fandt {antal} idents".format(antal=len(idents)))
for current, observation in enumerate(observations):
observation_type_id = observation.observationstypeid
if observation_type_id in observation_types:
feature = self.create_feature_from_observation(
observation, geometriobjekter, idents, feedback)
if feature:
sink.addFeature(feature, QgsFeatureSink.FastInsert)
total_num_features_processed = total_num_features_processed + 1
feedback.setProgress(total_num_features_processed /
total_num_features)
if feedback.isCanceled():
return {}
apply_theme = self.parameterAsBool(parameters, self.APPLY_THEME,
context)
if apply_theme:
style_file = os.path.join(os.path.dirname(__file__), "..",
"styles", "observation.qml")
alg_params = {"INPUT": dest_id, "STYLE": style_file}
processing.run(
"qgis:setstyleforvectorlayer",
alg_params,
context=context,
feedback=feedback,
is_child_algorithm=True,
)
return {self.OUTPUT: dest_id}
def execute_in_place_run(alg,
parameters,
context=None,
feedback=None,
raise_exceptions=False):
"""Executes an algorithm modifying features in-place in the input layer.
:param alg: algorithm to run
:type alg: QgsProcessingAlgorithm
:param parameters: parameters of the algorithm
:type parameters: dict
:param context: context, defaults to None
:type context: QgsProcessingContext, optional
:param feedback: feedback, defaults to None
:type feedback: QgsProcessingFeedback, optional
:param raise_exceptions: useful for testing, if True exceptions are raised, normally exceptions will be forwarded to the feedback
:type raise_exceptions: boo, default to False
:raises QgsProcessingException: raised when there is no active layer, or it cannot be made editable
:return: a tuple with true if success and results
:rtype: tuple
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
# Only feature based algs have sourceFlags
try:
if alg.sourceFlags(
) & QgsProcessingFeatureSource.FlagSkipGeometryValidityChecks:
context.setInvalidGeometryCheck(QgsFeatureRequest.GeometryNoCheck)
except AttributeError:
pass
active_layer = parameters['INPUT']
# Run some checks and prepare the layer for in-place execution by:
# - getting the active layer and checking that it is a vector
# - making the layer editable if it was not already
# - selecting all features if none was selected
# - checking in-place support for the active layer/alg/parameters
# If one of the check fails and raise_exceptions is True an exception
# is raised, else the execution is aborted and the error reported in
# the feedback
try:
if active_layer is None:
raise QgsProcessingException(tr("There is not active layer."))
if not isinstance(active_layer, QgsVectorLayer):
raise QgsProcessingException(
tr("Active layer is not a vector layer."))
if not active_layer.isEditable():
if not active_layer.startEditing():
raise QgsProcessingException(
tr("Active layer is not editable (and editing could not be turned on)."
))
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(
tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."
))
except QgsProcessingException as e:
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing',
Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)), fatalError=True)
return False, {}
if not active_layer.selectedFeatureIds():
active_layer.selectAll()
# Make sure we are working on selected features only
parameters['INPUT'] = QgsProcessingFeatureSourceDefinition(
active_layer.id(), True)
parameters['OUTPUT'] = 'memory:'
req = QgsFeatureRequest(QgsExpression(r"$id < 0"))
req.setFlags(QgsFeatureRequest.NoGeometry)
req.setSubsetOfAttributes([])
# Start the execution
# If anything goes wrong and raise_exceptions is True an exception
# is raised, else the execution is aborted and the error reported in
# the feedback
try:
new_feature_ids = []
active_layer.beginEditCommand(alg.displayName())
# Checks whether the algorithm has a processFeature method
if hasattr(alg, 'processFeature'): # in-place feature editing
# Make a clone or it will crash the second time the dialog
# is opened and run
alg = alg.create()
if not alg.prepare(parameters, context, feedback):
raise QgsProcessingException(
tr("Could not prepare selected algorithm."))
# Check again for compatibility after prepare
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(
tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."
))
field_idxs = range(len(active_layer.fields()))
iterator_req = QgsFeatureRequest(active_layer.selectedFeatureIds())
iterator_req.setInvalidGeometryCheck(
context.invalidGeometryCheck())
feature_iterator = active_layer.getFeatures(iterator_req)
step = 100 / len(active_layer.selectedFeatureIds()
) if active_layer.selectedFeatureIds() else 1
for current, f in enumerate(feature_iterator):
feedback.setProgress(current * step)
if feedback.isCanceled():
break
# need a deep copy, because python processFeature implementations may return
# a shallow copy from processFeature
input_feature = QgsFeature(f)
new_features = alg.processFeature(input_feature, context,
feedback)
new_features = QgsVectorLayerUtils.makeFeaturesCompatible(
new_features, active_layer)
if len(new_features) == 0:
active_layer.deleteFeature(f.id())
elif len(new_features) == 1:
new_f = new_features[0]
if not f.geometry().equals(new_f.geometry()):
active_layer.changeGeometry(f.id(), new_f.geometry())
if f.attributes() != new_f.attributes():
active_layer.changeAttributeValues(
f.id(), dict(zip(field_idxs, new_f.attributes())),
dict(zip(field_idxs, f.attributes())))
new_feature_ids.append(f.id())
else:
active_layer.deleteFeature(f.id())
# Get the new ids
old_ids = set(
[f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(
tr("Error adding processed features back into the layer."
))
new_ids = set(
[f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
results, ok = {}, True
else: # Traditional 'run' with delete and add features cycle
# There is no way to know if some features have been skipped
# due to invalid geometries
if context.invalidGeometryCheck(
) == QgsFeatureRequest.GeometrySkipInvalid:
selected_ids = active_layer.selectedFeatureIds()
else:
selected_ids = []
results, ok = alg.run(parameters, context, feedback)
if ok:
result_layer = QgsProcessingUtils.mapLayerFromString(
results['OUTPUT'], context)
# TODO: check if features have changed before delete/add cycle
new_features = []
# Check if there are any skipped features
if context.invalidGeometryCheck(
) == QgsFeatureRequest.GeometrySkipInvalid:
missing_ids = list(
set(selected_ids) - set(result_layer.allFeatureIds()))
if missing_ids:
for f in active_layer.getFeatures(
QgsFeatureRequest(missing_ids)):
if not f.geometry().isGeosValid():
new_features.append(f)
active_layer.deleteFeatures(active_layer.selectedFeatureIds())
for f in result_layer.getFeatures():
new_features.extend(
QgsVectorLayerUtils.makeFeaturesCompatible(
[f], active_layer))
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(
tr("Error adding processed features back into the layer."
))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
active_layer.endEditCommand()
if ok and new_feature_ids:
active_layer.selectByIds(new_feature_ids)
elif not ok:
active_layer.rollBack()
return ok, results
except QgsProcessingException as e:
active_layer.endEditCommand()
active_layer.rollBack()
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing',
Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)), fatalError=True)
return False, {}
def processAlgorithm(self, parameters, context: QgsProcessingContext,
feedback: QgsProcessingFeedback):
"""Here is where the processing itself takes place."""
feedback.setProgress(0)
na = qgis_utils.plugins["qgepplugin"].network_analyzer
# init params
reach_layer = self.parameterAsVectorLayer(parameters, self.REACH_LAYER,
context)
flow_layer = self.parameterAsVectorLayer(parameters,
self.FLOWTIMES_LAYER, context)
fk_reach_field = self.parameterAsFields(parameters,
self.FK_REACH_FIELD,
context)[0]
flow_time_field = self.parameterAsFields(parameters,
self.FLOWTIMES_FIELD,
context)[0]
# create feature sink
fields = QgsFields()
fields.append(QgsField('flow_time', QVariant.Double))
(sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT,
context, fields,
QgsWkbTypes.LineString,
reach_layer.sourceCrs())
if sink is None:
raise QgsProcessingException(
self.invalidSinkError(parameters, self.OUTPUT))
# get selected reach
iterator = reach_layer.getSelectedFeatures()
feature_count = reach_layer.selectedFeatureCount()
if feature_count != 1:
raise QgsProcessingException(
self.invalidSourceError(parameters, self.REACH_LAYER))
reach_feature = QgsFeature()
iterator.nextFeature(reach_feature)
assert reach_feature.isValid()
qgep_reach_obj_id = reach_feature.attribute('obj_id')
# get top node
reach_features = na.getFeaturesByAttr(na.getEdgeLayer(), 'obj_id',
[qgep_reach_obj_id]).asDict()
assert len(reach_features) > 0
from_pos = 1
top_node = None
for fid, reach_feature in reach_features.items():
if from_pos > reach_feature.attribute('from_pos'):
top_node = reach_feature.attribute('from_obj_id_interpolate')
from_pos = reach_feature.attribute('from_pos')
assert top_node is not None
nodes = na.getFeaturesByAttr(na.getNodeLayer(), 'obj_id',
[top_node]).asDict()
assert len(nodes) == 1
top_node_id = next(iter(nodes.values())).id()
# create graph
_, edges = na.getTree(top_node_id)
feedback.setProgress(50)
cache_edge_features = na.getFeaturesById(
na.getEdgeLayer(),
[edge[2]['feature'] for edge in edges]).asDict()
# join and accumulate flow times
i = -1
flow_time = 0.0
while True:
i += 1
feedback.setProgress(50 + i / len(edges) * 50)
if i >= len(edges):
break
edge = edges[i]
edge_feature = cache_edge_features[edge[2]['feature']]
# TODO: if top_pos != 1 => merge
if edge_feature.attribute('type') != 'reach':
continue
rate = edge_feature.attribute('to_pos') - edge_feature.attribute(
'from_pos')
assert 0 < rate <= 1
expression = QgsExpression("{fk_reach} = '{obj_id}'".format(
fk_reach=fk_reach_field, obj_id=edge_feature['obj_id']))
print(expression.expression())
request = QgsFeatureRequest(expression)
flow_time_feature = next(flow_layer.getFeatures(request))
if not flow_time_feature.isValid():
break
flow_time += rate * flow_time_feature.attribute(flow_time_field)
sf = QgsFeature()
sf.setFields(fields)
sf.setAttribute('flow_time', flow_time)
sf.setGeometry(edge_feature.geometry())
sink.addFeature(sf, QgsFeatureSink.FastInsert)
#f.setAttributes(attrs)
#sink.addFeature(f, QgsFeatureSink.FastInsert)
#feedback.setProgress(int(current * total))
return {self.OUTPUT: dest_id}
def generate_od_routes(
network_layer: QgsVectorLayer,
origin_layer: QgsVectorLayer,
poi_layer: QgsVectorLayer,
size_field: str,
class_field: str,
work_layer: QgsVectorLayer = None,
work_size_field: str = None,
school_layer: QgsVectorLayer = None,
school_size_field: str = None,
origin_weight_field: str = None,
socio_data=None,
health_data=None,
diversity_data=None,
join_on: str = None,
max_distance: int = 25000,
return_layer: bool = True,
return_raw: bool = False,
feedback: QgsProcessingFeedback = None,
) -> QgsVectorLayer:
"""
Shortest path algorithm based on Dijkstra's algorithm
:param network_layer: road network
:param points_layer: combined from to points
:param relations_data: tabular from to id data
:param origin_field: name of from field
:param destination_field: name of to field
:param max_distance: maximum distance/cost
:param crs: output layer crs
"""
if not network_layer.wkbType() & QgsWkbTypes.LineString:
raise Exception('Network layer must be of type LineString')
crs = network_layer.crs()
## prepare graph
director = QgsVectorLayerDirector(
source=network_layer,
directionFieldId=-1,
directDirectionValue='',
reverseDirectionValue='',
bothDirectionValue='',
defaultDirection=QgsVectorLayerDirector.DirectionBoth,
)
# First strategy is for actual shortest distance calculation
director.addStrategy(QgsNetworkDistanceStrategy()) # 0
# Second strategy is a hack to be able to recover the edge id
director.addStrategy(SaveFidStrategy()) # 1
builder = QgsGraphBuilder(crs)
## spatial index
poi_sidx = QgsSpatialIndex(poi_layer)
work_sidx = QgsSpatialIndex(work_layer)
school_sidx = QgsSpatialIndex(school_layer)
## prepare points
orig_n = len(origin_layer)
poi_n = len(poi_layer)
work_n = len(work_layer) if work_layer else 0
school_n = len(school_layer) if school_layer else 0
dest_n = poi_n + work_n + school_n
orig_points = [None] * orig_n
orig_sizes = np.zeros(orig_n, dtype=float)
if socio_data:
orig_socio = np.zeros(orig_n, dtype=float)
dest_points = [None] * dest_n
dest_sizes = [None] * dest_n
dest_fids = [None] * dest_n
dest_cats = [None] * dest_n
orig_id_field = 'deso'
for i, feat in enumerate(origin_layer.getFeatures()):
orig_points[i] = feat.geometry().asPoint()
orig_sizes[i] = feat[size_field] * (
feat[origin_weight_field] if origin_weight_field else 1
)
if socio_data:
orig_socio[i] = socio_data[
feat[orig_id_field]
] # FIXME: check if all origins have data
if socio_data:
orig_socio = orig_socio / np.mean(orig_socio)
orig_sizes *= orig_socio
for i, feat in enumerate(poi_layer.getFeatures()):
dest_points[i] = feat.geometry().asPoint()
dest_fids[i] = feat.id()
dest_sizes[i] = 1 # TODO: dest size
dest_cats[i] = poi_class_map.get(feat[class_field])
if work_layer:
for i, feat in enumerate(work_layer.getFeatures(), start=poi_n):
dest_points[i] = feat.geometry().asPoint()
dest_fids[i] = feat.id()
dest_sizes[i] = feat[work_size_field] # TODO: dest size
dest_cats[i] = 'work'
if school_layer:
for i, feat in enumerate(school_layer.getFeatures(), start=(poi_n + work_n)):
dest_points[i] = feat.geometry().asPoint()
dest_fids[i] = feat.id()
dest_sizes[i] = feat[school_size_field] # TODO: dest size
dest_cats[i] = 'school'
# points = [origin.point for origin in origins_data] + [
# dest.point for dest in dests_data
# ]
if feedback is None:
feedback = QgsProcessingFeedback()
def progress(p):
if int(10 * p % 100) == 0:
print(f'{int(p):#3d}%')
feedback.progressChanged.connect(progress)
with timing('build network graph'):
tied_points = director.makeGraph(
builder, orig_points + dest_points, feedback=feedback
)
graph = builder.graph()
orig_tied_points = tied_points[:orig_n]
dest_tied_points = tied_points[orig_n:]
poi_tied_points = dest_tied_points[:poi_n]
work_tied_points = dest_tied_points[poi_n : poi_n + work_n]
school_tied_points = dest_tied_points[poi_n + work_n :]
dest_fid_to_tied_points = dict(zip(dest_fids, enumerate(dest_tied_points)))
poi_fid_to_tied_points = dict(zip(dest_fids[:poi_n], enumerate(poi_tied_points)))
work_fid_to_tied_points = dict(
zip(dest_fids[poi_n : poi_n + work_n], enumerate(work_tied_points, start=poi_n))
)
school_fid_to_tied_points = dict(
zip(
dest_fids[poi_n + work_n :],
enumerate(school_tied_points, start=poi_n + work_n),
)
)
orig_dests = [None] * orig_n
for i, point in enumerate(orig_points):
orig_dests[i] = (
[
poi_fid_to_tied_points[fid]
for fid in poi_sidx.nearestNeighbor(
point, neighbors=MAX_NEIGHBORS, maxDistance=max_distance
)
]
+ [
work_fid_to_tied_points[fid]
for fid in work_sidx.nearestNeighbor(
point, neighbors=MAX_NEIGHBORS, maxDistance=max_distance
)
]
+ [
school_fid_to_tied_points[fid]
for fid in school_sidx.nearestNeighbor(
point, neighbors=MAX_NEIGHBORS, maxDistance=max_distance
)
]
)
step = 100.0 / orig_n
time_dijkstra = 0.0
time_find = 0.0
time_route = 0.0
with timing('calculate connecting routes'):
routes = []
# for i, (origin_fid, dest_fids) in enumerate(od_data):
for i, (orig_point, dests) in enumerate(zip(orig_tied_points, orig_dests)):
origin_vertex_id = graph.findVertex(orig_point)
# Calculate the tree and cost using the distance strategy (#0)
ts = time()
(tree, cost) = QgsGraphAnalyzer.dijkstra(graph, origin_vertex_id, 0)
time_dijkstra += time() - ts
for j, dest_point in dests:
if feedback.isCanceled():
return
if dest_sizes[j] <= 0:
continue
category = dest_cats[j]
if category is None:
continue
ts = time()
dest_vertex_id = graph.findVertex(dest_point)
time_find += time() - ts
if tree[dest_vertex_id] != -1 and (
cost[dest_vertex_id] <= MAX_DISTANCE_M
or MAX_DISTANCE_M <= 0 # TODO: enable skipping max distance
):
route_distance = cost[dest_vertex_id]
# route_points = [graph.vertex(dest_vertex_id).point()]
cur_vertex_id = dest_vertex_id
route_fids = []
# Iterate the graph from dest to origin saving the edges
ts = time()
while cur_vertex_id != origin_vertex_id:
cur_edge = graph.edge(tree[cur_vertex_id])
# Here we recover the edge id through strategy #1
route_fids.append(cur_edge.cost(1))
cur_vertex_id = cur_edge.fromVertex()
# route_points.append(graph.vertex(cur_vertex_id).point())
time_route += time() - ts
# route_points.reverse()
# route_geom = QgsGeometry.fromPolylineXY(route_points))
# Hack to remove duplicate fids
route_fids = list(
dict.fromkeys(route_fids)
) # NOTE: requires python >= 3.7 for ordered dict FIXME: add python version check
route_fids.reverse()
# Calc
# TODO: Move to matrix and vectorize calculation using numpy
gravity_value = poi_gravity_values[category]
bike_params = mode_params_bike[category]
ebike_params = mode_params_ebike[category]
# NOTE: we include dest size in decay here
decay = dest_sizes[j] * math.exp(
gravity_value * route_distance / 1000.0
)
p_bike = sigmoid(*bike_params, route_distance)
p_ebike = sigmoid(*ebike_params, route_distance)
# TODO: use namedtuple or dataclass
routes.append(
Route(
i,
j,
category,
route_distance,
decay,
p_bike,
p_ebike,
route_fids,
)
)
feedback.setProgress(i * step)
print(f'dijkstra took: {time_dijkstra:#1.2f} sec')
print(f'find vertex took: {time_find:#1.2f} sec')
print(f'route took: {time_route:#1.2f} sec')
with timing('post process routes'):
alpha_bike = 0.8
alpha_ebke = 0.2
decay_sums = {cat: defaultdict(float) for cat in poi_categories}
bike_values = {cat: defaultdict(float) for cat in poi_categories}
ebike_values = {cat: defaultdict(float) for cat in poi_categories}
for route in routes:
# NOTE: dest size is included in decay
decay_sums[route.cat][route.i] += route.decay
for route in routes:
decay_sum = decay_sums[route.cat][route.i]
# TODO: add T_p and alpha_m
T_p = trip_generation[route.cat]
bike_value = (
T_p
* alpha_bike
* orig_sizes[route.i]
* route.p_bike
* route.decay
/ decay_sum
)
ebike_value = (
T_p
* alpha_ebke
* orig_sizes[route.i]
* route.p_ebike
* route.decay
/ decay_sum
)
for fid in route.net_fids:
bike_values[route.cat][fid] += float(bike_value)
ebike_values[route.cat][fid] += float(ebike_value)
# FIXME: Un-kludge this
with timing('create result features'):
fields = get_fields()
segments = []
for feature in network_layer.getFeatures():
fid = feature.id()
segment = QgsFeature(fields)
segment.setGeometry(QgsGeometry(feature.geometry()))
segment['network_fid'] = fid
flow = 0.0
for cat in poi_categories:
bike_field = f'{cat}_bike_value'
ebike_field = f'{cat}_ebike_value'
flow_bike = segment[bike_field] = bike_values[cat].get(fid)
flow_ebke = segment[ebike_field] = ebike_values[cat].get(fid)
if flow_bike is not None:
flow += flow_bike
if flow_ebke is not None:
flow += flow_ebke
segment['flow'] = flow
segment['lts'] = feature['lts']
segment['vgu'] = feature['vgu']
segment['R'] = flow * feature['ratio']
segments.append(segment)
if not return_layer:
if return_raw:
return segments, bike_values, ebike_values
return segments
with timing('create result layer'):
output_layer = QgsVectorLayer(
f'LineString?crs={crs.toWkt()}', 'segments', 'memory'
)
with edit(output_layer):
for field in fields:
output_layer.addAttribute(field)
output_layer.addFeatures(segments, flags=QgsFeatureSink.FastInsert)
return output_layer
def execute_in_place_run(alg, active_layer, parameters, context=None, feedback=None, raise_exceptions=False):
"""Executes an algorithm modifying features in-place in the input layer.
The input layer must be editable or an exception is raised.
:param alg: algorithm to run
:type alg: QgsProcessingAlgorithm
:param active_layer: the editable layer
:type active_layer: QgsVectoLayer
:param parameters: parameters of the algorithm
:type parameters: dict
:param context: context, defaults to None
:param context: QgsProcessingContext, optional
:param feedback: feedback, defaults to None
:param feedback: QgsProcessingFeedback, optional
:raises QgsProcessingException: raised when the layer is not editable or the layer cannot be found in the current project
:return: a tuple with true if success and results
:rtype: tuple
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
if active_layer is None or not active_layer.isEditable():
raise QgsProcessingException(tr("Layer is not editable or layer is None."))
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."))
parameters['OUTPUT'] = 'memory:'
try:
new_feature_ids = []
active_layer.beginEditCommand(alg.displayName())
req = QgsFeatureRequest(QgsExpression(r"$id < 0"))
req.setFlags(QgsFeatureRequest.NoGeometry)
req.setSubsetOfAttributes([])
# Checks whether the algorithm has a processFeature method
if hasattr(alg, 'processFeature'): # in-place feature editing
# Make a clone or it will crash the second time the dialog
# is opened and run
alg = alg.create()
if not alg.prepare(parameters, context, feedback):
raise QgsProcessingException(tr("Could not prepare selected algorithm."))
# Check again for compatibility after prepare
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."))
field_idxs = range(len(active_layer.fields()))
feature_iterator = active_layer.getFeatures(QgsFeatureRequest(active_layer.selectedFeatureIds())) if parameters['INPUT'].selectedFeaturesOnly else active_layer.getFeatures()
step = 100 / len(active_layer.selectedFeatureIds()) if active_layer.selectedFeatureIds() else 1
for current, f in enumerate(feature_iterator):
feedback.setProgress(current * step)
if feedback.isCanceled():
break
# need a deep copy, because python processFeature implementations may return
# a shallow copy from processFeature
input_feature = QgsFeature(f)
new_features = alg.processFeature(input_feature, context, feedback)
new_features = QgsVectorLayerUtils.makeFeaturesCompatible(new_features, active_layer)
if len(new_features) == 0:
active_layer.deleteFeature(f.id())
elif len(new_features) == 1:
new_f = new_features[0]
if not f.geometry().equals(new_f.geometry()):
active_layer.changeGeometry(f.id(), new_f.geometry())
if f.attributes() != new_f.attributes():
active_layer.changeAttributeValues(f.id(), dict(zip(field_idxs, new_f.attributes())), dict(zip(field_idxs, f.attributes())))
new_feature_ids.append(f.id())
else:
active_layer.deleteFeature(f.id())
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(tr("Error adding processed features back into the layer."))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
results, ok = {}, True
else: # Traditional 'run' with delete and add features cycle
results, ok = alg.run(parameters, context, feedback)
if ok:
result_layer = QgsProcessingUtils.mapLayerFromString(results['OUTPUT'], context)
# TODO: check if features have changed before delete/add cycle
active_layer.deleteFeatures(active_layer.selectedFeatureIds())
new_features = []
for f in result_layer.getFeatures():
new_features.extend(QgsVectorLayerUtils.
makeFeaturesCompatible([f], active_layer))
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(tr("Error adding processed features back into the layer."))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
active_layer.endEditCommand()
if ok and new_feature_ids:
active_layer.selectByIds(new_feature_ids)
elif not ok:
active_layer.rollBack()
return ok, results
except QgsProcessingException as e:
active_layer.endEditCommand()
active_layer.rollBack()
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing', Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)))
return False, {}
def handleAlgorithmResults(alg, context, feedback=None, showResults=True, parameters={}):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context, typeHint=details.layerTypeHint)
if layer is not None:
set_layer_name(layer, details)
'''If running a model, the execution will arrive here when an algorithm that is part of
that model is executed. We check if its output is a final otuput of the model, and
adapt the output name accordingly'''
outputName = details.outputName
expcontext = QgsExpressionContext()
scope = QgsExpressionContextScope()
expcontext.appendScope(scope)
for out in alg.outputDefinitions():
if out.name() not in parameters:
continue
outValue = parameters[out.name()]
if hasattr(outValue, "sink"):
outValue = outValue.sink.valueAsString(expcontext)[0]
else:
outValue = str(outValue)
if outValue == l:
outputName = out.name()
break
style = None
if outputName:
style = RenderingStyles.getStyle(alg.id(), outputName)
if style is None:
if layer.type() == QgsMapLayer.RasterLayer:
style = ProcessingConfig.getSetting(ProcessingConfig.RASTER_STYLE)
else:
if layer.geometryType() == QgsWkbTypes.PointGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POINT_STYLE)
elif layer.geometryType() == QgsWkbTypes.LineGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_LINE_STYLE)
else:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POLYGON_STYLE)
if style:
layer.loadNamedStyle(style)
details.project.addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
if details.postProcessor():
details.postProcessor().postProcessLayer(layer, context, feedback)
else:
wrongLayers.append(str(l))
except Exception:
QgsMessageLog.logMessage(QCoreApplication.translate('Postprocessing', "Error loading result layer:") + "\n" + traceback.format_exc(), 'Processing', Qgis.Critical)
wrongLayers.append(str(l))
i += 1
feedback.setProgress(100)
if wrongLayers:
msg = QCoreApplication.translate('Postprocessing', "The following layers were not correctly generated.")
msg += "<ul>" + "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += QCoreApplication.translate('Postprocessing', "You can check the 'Log Messages Panel' in QGIS main window to find more information about the execution of the algorithm.")
feedback.reportError(msg)
return len(wrongLayers) == 0
def processAlgorithm(self, parameters, context: QgsProcessingContext,
feedback: QgsProcessingFeedback):
"""Here is where the processing itself takes place."""
feedback.setProgress(0)
# init params
out_file = self.parameterAsFileOutput(parameters, self.OUT_FILE,
context)
# create feature sink for node summary
fields = QgsFields()
fields.append(QgsField('id', QVariant.String))
fields.append(QgsField('type', QVariant.String))
fields.append(QgsField('average_depth', QVariant.Double))
fields.append(QgsField('maximum_depth', QVariant.Double))
fields.append(QgsField('maximum_hgl', QVariant.Double))
fields.append(QgsField('time_max_day', QVariant.Int))
fields.append(QgsField('time_max_time', QVariant.Double))
fields.append(QgsField('reported_max_depth', QVariant.Double))
(sink_node, dest_id) = self.parameterAsSink(parameters,
self.NODE_SUMMARY, context,
fields)
if sink_node is None:
raise QgsProcessingException(
self.invalidSinkError(parameters, self.NODE_SUMMARY))
# Get node summary from output file
qs = QgepSwmm(None, None, None, None, out_file, None, None, None)
if qs.feedbacks is not None:
for i in range(len(qs.feedbacks)):
feedback.reportError(qs.feedbacks[i])
node_summary = qs.extract_node_depth_summary()
# Fill node summary with data
for ns in node_summary:
sf = QgsFeature()
sf.setFields(fields)
for k in ns.keys():
index = fields.indexOf(k)
if index != -1:
sf.setAttribute(k, ns[k])
sink_node.addFeature(sf, QgsFeatureSink.FastInsert)
feedback.setProgress(50)
# create feature sink for link summary
fields = QgsFields()
fields.append(QgsField('id', QVariant.String))
fields.append(QgsField('type', QVariant.String))
fields.append(QgsField('maximum_flow', QVariant.Double))
fields.append(QgsField('time_max_day', QVariant.Int))
fields.append(QgsField('time_max_time', QVariant.String))
fields.append(QgsField('maximum_velocity', QVariant.Double))
fields.append(QgsField('max_over_full_flow', QVariant.Double))
fields.append(QgsField('max_over_full_depth', QVariant.Double))
(sink_link, dest_id) = self.parameterAsSink(parameters,
self.LINK_SUMMARY, context,
fields)
if sink_link is None:
raise QgsProcessingException(
self.invalidSinkError(parameters, self.LINK_SUMMARY))
# Get link summary from output file
link_summary = qs.extract_link_flow_summary()
# Fill node summary with data
for ns in link_summary:
sf = QgsFeature()
sf.setFields(fields)
for k in ns.keys():
index = fields.indexOf(k)
if index != -1:
sf.setAttribute(k, ns[k])
sink_link.addFeature(sf, QgsFeatureSink.FastInsert)
feedback.setProgress(100)
return {self.NODE_SUMMARY: sink_node, self.LINK_SUMMARY: sink_link}
def handleAlgorithmResults(alg, context, feedback=None, showResults=True):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context)
if layer is not None:
layer.setName(details.name)
details.project.addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
else:
name = details.name
if ProcessingConfig.getSetting(
ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
name = os.path.basename(l)
dataobjects.load(l, name, alg.crs,
RenderingStyles.getStyle(alg.id(), l))
except Exception:
QgsMessageLog.logMessage("Error loading result layer:\n" + traceback.format_exc(), 'Processing', QgsMessageLog.CRITICAL)
#wrongLayers.append(out.description())
wrongLayers.append(l)
# for out in alg.outputs:
# feedback.setProgress(100 * i / float(len(alg.outputs)))
# if out.flags() & QgsProcessingParameterDefinition.FlagHidden or not out.open:
# continue
# if isinstance(out, (OutputRaster, OutputVector, OutputTable)):
# try:
# layer = QgsProcessingUtils.mapLayerFromString(out.value, context)
# if layer:
# layer.setName(out.description)
# QgsProject.instance().addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
# else:
# if ProcessingConfig.getSetting(
# ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
# name = os.path.basename(out.value)
# else:
# name = out.description
#
# isRaster = True if isinstance(out, OutputRaster) else False
# dataobjects.load(out.value, name, alg.crs,
# RenderingStyles.getStyle(alg.id(), out.name),
# isRaster)
# except Exception:
# QgsMessageLog.logMessage("Error loading result layer:\n" + traceback.format_exc(), 'Processing', QgsMessageLog.CRITICAL)
# wrongLayers.append(out.description)
# elif isinstance(out, OutputHTML):
# resultsList.addResult(alg.icon(), out.description, out.value)
i += 1
QApplication.restoreOverrideCursor()
if wrongLayers:
msg = "The following layers were not correctly generated.<ul>"
msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += "You can check the log messages to find more information about the execution of the algorithm"
feedback.reportError(msg)
return len(wrongLayers) == 0
def handleAlgorithmResults(alg, context, feedback=None, showResults=True):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(
QCoreApplication.translate('Postprocessing',
'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(
100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context)
if layer is not None:
layer.setName(details.name)
details.project.addMapLayer(
context.temporaryLayerStore().takeMapLayer(layer))
else:
name = details.name
if ProcessingConfig.getSetting(
ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
name = os.path.basename(l)
dataobjects.load(l, name, alg.crs,
RenderingStyles.getStyle(alg.id(), l))
except Exception:
QgsMessageLog.logMessage(
"Error loading result layer:\n" + traceback.format_exc(),
'Processing', QgsMessageLog.CRITICAL)
#wrongLayers.append(out.description())
wrongLayers.append(l)
# for out in alg.outputs:
# feedback.setProgress(100 * i / float(len(alg.outputs)))
# if out.flags() & QgsProcessingParameterDefinition.FlagHidden or not out.open:
# continue
# if isinstance(out, (OutputRaster, OutputVector, OutputTable)):
# try:
# layer = QgsProcessingUtils.mapLayerFromString(out.value, context)
# if layer:
# layer.setName(out.description)
# QgsProject.instance().addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
# else:
# if ProcessingConfig.getSetting(
# ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
# name = os.path.basename(out.value)
# else:
# name = out.description
#
# isRaster = True if isinstance(out, OutputRaster) else False
# dataobjects.load(out.value, name, alg.crs,
# RenderingStyles.getStyle(alg.id(), out.name),
# isRaster)
# except Exception:
# QgsMessageLog.logMessage("Error loading result layer:\n" + traceback.format_exc(), 'Processing', QgsMessageLog.CRITICAL)
# wrongLayers.append(out.description)
# elif isinstance(out, OutputHTML):
# resultsList.addResult(alg.icon(), out.description, out.value)
i += 1
QApplication.restoreOverrideCursor()
if wrongLayers:
msg = "The following layers were not correctly generated.<ul>"
msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += "You can check the log messages to find more information about the execution of the algorithm"
feedback.reportError(msg)
return len(wrongLayers) == 0
def mergeDataSources2Vrt(self, dataSources, outFile, union=False, relative=False,
schema=False, feedback=None):
'''Function to do the work of merging datasources in a single vrt format
@param data_sources: Array of path strings
@param outfile: the output vrt file to generate
@param relative: Write relative flag. DOES NOT relativise paths. They have to be already relative
@param schema: Schema flag
@return: vrt in string format
'''
if feedback is None:
feedback = QgsProcessingFeedback()
vrt = '<OGRVRTDataSource>'
if union:
vrt += '<OGRVRTUnionLayer name="UnionedLayer">'
total = 100.0 / len(dataSources) if dataSources else 1
for current, layer in enumerate(dataSources):
if feedback.isCanceled():
break
feedback.setProgress(int(current * total))
inFile = layer.source()
srcDS = ogr.Open(inFile, 0)
if srcDS is None:
raise QgsProcessingException(
self.tr('Invalid datasource: {}'.format(inFile)))
if schema:
inFile = '@dummy@'
for layer in srcDS:
layerDef = layer.GetLayerDefn()
layerName = layerDef.GetName()
vrt += '<OGRVRTLayer name="{}">'.format(self.XmlEsc(layerName))
vrt += '<SrcDataSource relativeToVRT="{}" shared="{}">{}</SrcDataSource>'.format(1 if relative else 0, not schema, self.XmlEsc(inFile))
if schema:
vrt += '<SrcLayer>@dummy@</SrcLayer>'
else:
vrt += '<SrcLayer>{}</SrcLayer>'.format(self.XmlEsc(layerName))
vrt += '<GeometryType>{}</GeometryType>'.format(self.GeomType2Name(layerDef.GetGeomType()))
crs = layer.GetSpatialRef()
if crs is not None:
vrt += '<LayerSRS>{}</LayerSRS>'.format(self.XmlEsc(crs.ExportToWkt()))
# Process all the fields.
for fieldIdx in range(layerDef.GetFieldCount()):
fieldDef = layerDef.GetFieldDefn(fieldIdx)
vrt += '<Field name="{}" type="{}"'.format(self.XmlEsc(fieldDef.GetName()), self.fieldType2Name(fieldDef.GetType()))
if not schema:
vrt += ' src="{}"'.format(self.XmlEsc(fieldDef.GetName()))
if fieldDef.GetWidth() > 0:
vrt += ' width="{}"'.format(fieldDef.GetWidth())
if fieldDef.GetPrecision() > 0:
vrt += ' precision="{}"'.format(fieldDef.GetPrecision())
vrt += '/>'
vrt += '</OGRVRTLayer>'
srcDS.Destroy()
if union:
vrt += '</OGRVRTUnionLayer>'
vrt += '</OGRVRTDataSource>'
#TODO: pretty-print XML
if outFile is not None:
with codecs.open(outFile, 'w') as f:
f.write(vrt)
return vrt
def handleAlgorithmResults(alg,
context,
feedback=None,
showResults=True,
parameters={}):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(
QCoreApplication.translate('Postprocessing',
'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(
100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(
l, context, typeHint=details.layerTypeHint)
if layer is not None:
set_layer_name(layer, details)
'''If running a model, the execution will arrive here when an algorithm that is part of
that model is executed. We check if its output is a final otuput of the model, and
adapt the output name accordingly'''
outputName = details.outputName
expcontext = QgsExpressionContext()
scope = QgsExpressionContextScope()
expcontext.appendScope(scope)
for out in alg.outputDefinitions():
if out.name() not in parameters:
continue
outValue = parameters[out.name()]
if hasattr(outValue, "sink"):
outValue = outValue.sink.valueAsString(expcontext)[0]
else:
outValue = str(outValue)
if outValue == l:
outputName = out.name()
break
style = None
if outputName:
style = RenderingStyles.getStyle(alg.id(), outputName)
if style is None:
if layer.type() == QgsMapLayerType.RasterLayer:
style = ProcessingConfig.getSetting(
ProcessingConfig.RASTER_STYLE)
else:
if layer.geometryType() == QgsWkbTypes.PointGeometry:
style = ProcessingConfig.getSetting(
ProcessingConfig.VECTOR_POINT_STYLE)
elif layer.geometryType() == QgsWkbTypes.LineGeometry:
style = ProcessingConfig.getSetting(
ProcessingConfig.VECTOR_LINE_STYLE)
else:
style = ProcessingConfig.getSetting(
ProcessingConfig.VECTOR_POLYGON_STYLE)
if style:
layer.loadNamedStyle(style)
details.project.addMapLayer(
context.temporaryLayerStore().takeMapLayer(layer))
if details.postProcessor():
details.postProcessor().postProcessLayer(
layer, context, feedback)
else:
wrongLayers.append(str(l))
except Exception:
QgsMessageLog.logMessage(
QCoreApplication.translate(
'Postprocessing', "Error loading result layer:") + "\n" +
traceback.format_exc(), 'Processing', Qgis.Critical)
wrongLayers.append(str(l))
i += 1
feedback.setProgress(100)
if wrongLayers:
msg = QCoreApplication.translate(
'Postprocessing',
"The following layers were not correctly generated.")
msg += "<ul>" + "".join(["<li>%s</li>" % lay
for lay in wrongLayers]) + "</ul>"
msg += QCoreApplication.translate(
'Postprocessing',
"You can check the 'Log Messages Panel' in QGIS main window to find more information about the execution of the algorithm."
)
feedback.reportError(msg)
return len(wrongLayers) == 0
def execute_in_place_run(alg, parameters, context=None, feedback=None, raise_exceptions=False):
"""Executes an algorithm modifying features in-place in the input layer.
:param alg: algorithm to run
:type alg: QgsProcessingAlgorithm
:param parameters: parameters of the algorithm
:type parameters: dict
:param context: context, defaults to None
:type context: QgsProcessingContext, optional
:param feedback: feedback, defaults to None
:type feedback: QgsProcessingFeedback, optional
:param raise_exceptions: useful for testing, if True exceptions are raised, normally exceptions will be forwarded to the feedback
:type raise_exceptions: boo, default to False
:raises QgsProcessingException: raised when there is no active layer, or it cannot be made editable
:return: a tuple with true if success and results
:rtype: tuple
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
# Only feature based algs have sourceFlags
try:
if alg.sourceFlags() & QgsProcessingFeatureSource.FlagSkipGeometryValidityChecks:
context.setInvalidGeometryCheck(QgsFeatureRequest.GeometryNoCheck)
except AttributeError:
pass
active_layer = parameters['INPUT']
# Run some checks and prepare the layer for in-place execution by:
# - getting the active layer and checking that it is a vector
# - making the layer editable if it was not already
# - selecting all features if none was selected
# - checking in-place support for the active layer/alg/parameters
# If one of the check fails and raise_exceptions is True an exception
# is raised, else the execution is aborted and the error reported in
# the feedback
try:
if active_layer is None:
raise QgsProcessingException(tr("There is not active layer."))
if not isinstance(active_layer, QgsVectorLayer):
raise QgsProcessingException(tr("Active layer is not a vector layer."))
if not active_layer.isEditable():
if not active_layer.startEditing():
raise QgsProcessingException(tr("Active layer is not editable (and editing could not be turned on)."))
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."))
except QgsProcessingException as e:
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing', Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)), fatalError=True)
return False, {}
if not active_layer.selectedFeatureIds():
active_layer.selectAll()
# Make sure we are working on selected features only
parameters['INPUT'] = QgsProcessingFeatureSourceDefinition(active_layer.id(), True)
parameters['OUTPUT'] = 'memory:'
req = QgsFeatureRequest(QgsExpression(r"$id < 0"))
req.setFlags(QgsFeatureRequest.NoGeometry)
req.setSubsetOfAttributes([])
# Start the execution
# If anything goes wrong and raise_exceptions is True an exception
# is raised, else the execution is aborted and the error reported in
# the feedback
try:
new_feature_ids = []
active_layer.beginEditCommand(alg.displayName())
# Checks whether the algorithm has a processFeature method
if hasattr(alg, 'processFeature'): # in-place feature editing
# Make a clone or it will crash the second time the dialog
# is opened and run
alg = alg.create()
if not alg.prepare(parameters, context, feedback):
raise QgsProcessingException(tr("Could not prepare selected algorithm."))
# Check again for compatibility after prepare
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."))
field_idxs = range(len(active_layer.fields()))
iterator_req = QgsFeatureRequest(active_layer.selectedFeatureIds())
iterator_req.setInvalidGeometryCheck(context.invalidGeometryCheck())
feature_iterator = active_layer.getFeatures(iterator_req)
step = 100 / len(active_layer.selectedFeatureIds()) if active_layer.selectedFeatureIds() else 1
for current, f in enumerate(feature_iterator):
feedback.setProgress(current * step)
if feedback.isCanceled():
break
# need a deep copy, because python processFeature implementations may return
# a shallow copy from processFeature
input_feature = QgsFeature(f)
new_features = alg.processFeature(input_feature, context, feedback)
new_features = QgsVectorLayerUtils.makeFeaturesCompatible(new_features, active_layer)
if len(new_features) == 0:
active_layer.deleteFeature(f.id())
elif len(new_features) == 1:
new_f = new_features[0]
if not f.geometry().equals(new_f.geometry()):
active_layer.changeGeometry(f.id(), new_f.geometry())
if f.attributes() != new_f.attributes():
active_layer.changeAttributeValues(f.id(), dict(zip(field_idxs, new_f.attributes())), dict(zip(field_idxs, f.attributes())))
new_feature_ids.append(f.id())
else:
active_layer.deleteFeature(f.id())
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(tr("Error adding processed features back into the layer."))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
results, ok = {}, True
else: # Traditional 'run' with delete and add features cycle
# There is no way to know if some features have been skipped
# due to invalid geometries
if context.invalidGeometryCheck() == QgsFeatureRequest.GeometrySkipInvalid:
selected_ids = active_layer.selectedFeatureIds()
else:
selected_ids = []
results, ok = alg.run(parameters, context, feedback)
if ok:
result_layer = QgsProcessingUtils.mapLayerFromString(results['OUTPUT'], context)
# TODO: check if features have changed before delete/add cycle
new_features = []
# Check if there are any skipped features
if context.invalidGeometryCheck() == QgsFeatureRequest.GeometrySkipInvalid:
missing_ids = list(set(selected_ids) - set(result_layer.allFeatureIds()))
if missing_ids:
for f in active_layer.getFeatures(QgsFeatureRequest(missing_ids)):
if not f.geometry().isGeosValid():
new_features.append(f)
active_layer.deleteFeatures(active_layer.selectedFeatureIds())
for f in result_layer.getFeatures():
new_features.extend(QgsVectorLayerUtils.
makeFeaturesCompatible([f], active_layer))
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(tr("Error adding processed features back into the layer."))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
active_layer.endEditCommand()
if ok and new_feature_ids:
active_layer.selectByIds(new_feature_ids)
elif not ok:
active_layer.rollBack()
return ok, results
except QgsProcessingException as e:
active_layer.endEditCommand()
active_layer.rollBack()
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing', Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)), fatalError=True)
return False, {}
class AddCurrentStationsLayerAlgorithm(QgsProcessingAlgorithm):
PrmCurrentStationsLayer = 'CurrentStationsLayer'
PrmCurrentPredictionsLayer = 'CurrentPredictionsLayer'
# boilerplate methods
def name(self):
return 'addcurrentstationslayer'
def displayName(self):
return tr('Add Current Stations Layer')
def helpUrl(self):
return ''
def createInstance(self):
return AddCurrentStationsLayerAlgorithm()
# Set up this algorithm
def initAlgorithm(self, config):
self.addParameter(
QgsProcessingParameterFeatureSink(
self.PrmCurrentStationsLayer, tr('Current stations layer'),
QgsProcessing.TypeVectorPoint,
os.path.join(layerStoragePath(), 'current_stations.gpkg')))
self.addParameter(
QgsProcessingParameterFeatureSink(
self.PrmCurrentPredictionsLayer,
tr('Current predictions layer'), QgsProcessing.TypeVectorPoint,
os.path.join(layerStoragePath(), 'current_predictions.gpkg')))
self.feedback = QgsProcessingFeedback()
def processAlgorithm(self, parameters, context, feedback):
self.context = context
self.feedback = feedback
self.parameters = parameters
try:
os.mkdir(layerStoragePath())
except FileExistsError:
pass
current_dest_id = self.getCurrentStations()
predictions_dest_id = self.getCurrentPredictions()
return {
self.PrmCurrentStationsLayer: current_dest_id,
self.PrmCurrentPredictionsLayer: predictions_dest_id
}
def baseStationFields(self):
fields = QgsFields()
fields.append(QgsField("station", QVariant.String, '', 16))
fields.append(QgsField("id", QVariant.String, '', 12))
fields.append(QgsField("name", QVariant.String))
fields.append(QgsField("type", QVariant.String, '', 1))
fields.append(QgsField("timeZoneId", QVariant.String, '', 32))
fields.append(QgsField("timeZoneUTC", QVariant.String, '', 32))
fields.append(QgsField("refStation", QVariant.String, '', 12))
return fields
def basePredictionFields(self):
fields = QgsFields()
fields.append(QgsField("station", QVariant.String, '', 16))
fields.append(QgsField("depth", QVariant.Double))
fields.append(QgsField("time", QVariant.DateTime))
fields.append(QgsField("value", QVariant.Double)
) # signed value, on flood/ebb dimension for current
fields.append(QgsField("type", QVariant.String))
return fields
def currentPredictionFields(self):
fields = self.basePredictionFields()
fields.append(QgsField("dir", QVariant.Double))
fields.append(QgsField(
"magnitude", QVariant.Double)) # value along direction if known
return fields
def getCurrentPredictions(self):
if currentPredictionsLayer() != None:
raise QgsProcessingException(
tr('Existing current layers must be removed before creating new ones.'
))
(predictionsSink, predictions_dest_id) = self.parameterAsSink(
self.parameters, self.PrmCurrentPredictionsLayer, self.context,
self.currentPredictionFields(), QgsWkbTypes.Point, epsg4326)
if self.context.willLoadLayerOnCompletion(predictions_dest_id):
proc = CurrentPredictionsStylePostProcessor.create(
tr('Current Predictions'), 'current_predictions.qml',
CurrentPredictionsLayerType)
self.context.layerToLoadOnCompletionDetails(
predictions_dest_id).setPostProcessor(proc)
return predictions_dest_id
def getCurrentStations(self):
if currentStationsLayer() != None:
raise QgsProcessingException(
tr('Existing current layers must be removed before creating new ones.'
))
self.feedback.pushInfo(
"Requesting metadata for NOAA current stations...")
url = 'https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi/stations.xml?type=currentpredictions&expand=currentpredictionoffsets'
r = requests.get(url, timeout=30)
if r.status_code != 200:
raise QgsProcessingException(
tr('Request failed with status {}').format(r.status_code))
content = r.text
if len(content) == 0:
return
self.feedback.pushInfo('Read {} bytes'.format(len(content)))
# obtain our current stations output sink
fields = self.baseStationFields()
fields.append(QgsField("bin", QVariant.String, '', 4))
fields.append(QgsField("depth", QVariant.Double))
fields.append(QgsField("depthType", QVariant.String, '', 1))
fields.append(QgsField("surface", QVariant.Int))
fields.append(QgsField("meanFloodDir", QVariant.Double))
fields.append(QgsField("meanEbbDir", QVariant.Double))
fields.append(QgsField("mfcTimeAdjMin", QVariant.Double))
fields.append(QgsField("sbeTimeAdjMin", QVariant.Double))
fields.append(QgsField("mecTimeAdjMin", QVariant.Double))
fields.append(QgsField("sbfTimeAdjMin", QVariant.Double))
fields.append(QgsField("mfcAmpAdj", QVariant.Double))
fields.append(QgsField("mecAmpAdj", QVariant.Double))
(currentSink, current_dest_id) = self.parameterAsSink(
self.parameters, self.PrmCurrentStationsLayer, self.context,
fields, QgsWkbTypes.Point, epsg4326)
# This script converts a stations XML result obtained from this URL:
# https://api.tidesandcurrents.noaa.gov/mdapi/prod/webapi/stations.xml?type=currentpredictions&expand=currentpredictionoffsets
# by including only the bin of minimum depth for each station and eliminating weak/variable stations.
# The flood and ebb directions are captured from the metadata also.
self.feedback.pushInfo("Parsing metadata...")
# Parse the XML file into a DOM up front
xmlparse = ET.fromstring(content)
# Get the list of stations
root = xmlparse
stations = root.findall('Station')
# Get a map that will track the metadata for the least-depth bin for each station.
# Also maintain a map by station key.
surfaceMap = {}
stationMap = {}
# Loop over all stations and index them. If filtering for shallowest at each location,
# maintain the minimum depth station for each id in surfaceMap.
for s in stations:
stationId = s.find('id').text
stationBin = s.find('currbin').text
# Skip weak/variable type stations
if s.find('type').text == 'W':
continue
stationMap[stationId + '_' + stationBin] = s
# If we find that we already saw this station, check its depth and only update the station map
# if the newly found bin is shallower
lastStation = surfaceMap.get(stationId)
if lastStation and lastStation.find('depth').text and s.find(
'depth').text:
lastDepth = float(lastStation.find('depth').text)
newDepth = float(s.find('depth').text)
if newDepth >= lastDepth:
continue
surfaceMap[stationId] = s
if self.context.willLoadLayerOnCompletion(current_dest_id):
proc = CurrentStationsStylePostProcessor.create(
tr('Current Stations'), 'current_stations.qml',
CurrentStationsLayerType)
self.context.layerToLoadOnCompletionDetails(
current_dest_id).setPostProcessor(proc)
# Now build the features for all stations in the map.
progress_count = 0
tzl = TimeZoneLookup()
for key, s in stationMap.items():
cpo = s.find('currentpredictionoffsets')
f = QgsFeature(fields)
lng = float(s.find('lng').text)
lat = float(s.find('lat').text)
geom = QgsGeometry(QgsPoint(lng, lat))
f.setGeometry(geom)
f['station'] = key
f['id'] = s.find('id').text
f['bin'] = s.find('currbin').text
f['name'] = s.find('name').text
f['type'] = s.find('type').text
f['depth'] = parseFloatNullable(s.find('depth').text)
f['depthType'] = s.find('depthType').text
f['surface'] = 1 if surfaceMap.get(f['id']) == s else 0
refStationId = cpo.find('refStationId').text
if refStationId:
f['refStation'] = refStationId + '_' + cpo.find(
'refStationBin').text
(f['timeZoneId'],
f['timeZoneUTC']) = tzl.getZoneByCoordinates(lat, lng)
f['meanFloodDir'] = parseFloatNullable(
cpo.find('meanFloodDir').text)
f['meanEbbDir'] = parseFloatNullable(cpo.find('meanEbbDir').text)
f['mfcTimeAdjMin'] = parseFloatNullable(
cpo.find('mfcTimeAdjMin').text)
f['sbeTimeAdjMin'] = parseFloatNullable(
cpo.find('sbeTimeAdjMin').text)
f['mecTimeAdjMin'] = parseFloatNullable(
cpo.find('mecTimeAdjMin').text)
f['sbfTimeAdjMin'] = parseFloatNullable(
cpo.find('sbfTimeAdjMin').text)
f['mfcAmpAdj'] = parseFloatNullable(cpo.find('mfcAmpAdj').text)
f['mecAmpAdj'] = parseFloatNullable(cpo.find('mecAmpAdj').text)
currentSink.addFeature(f)
progress_count += 1
self.feedback.setProgress(100 * progress_count / len(stationMap))
return current_dest_id
def mergeDataSources2Vrt(self,
dataSources,
outFile,
union=False,
relative=False,
schema=False,
feedback=None):
'''Function to do the work of merging datasources in a single vrt format
@param data_sources: Array of path strings
@param outfile: the output vrt file to generate
@param relative: Write relative flag. DOES NOT relativise paths. They have to be already relative
@param schema: Schema flag
@return: vrt in string format
'''
if feedback is None:
feedback = QgsProcessingFeedback()
vrt = '<OGRVRTDataSource>'
if union:
vrt += '<OGRVRTUnionLayer name="UnionedLayer">'
total = 100.0 / len(dataSources) if dataSources else 1
for current, layer in enumerate(dataSources):
if feedback.isCanceled():
break
feedback.setProgress(int(current * total))
inFile = layer.source()
srcDS = ogr.Open(inFile, 0)
if srcDS is None:
raise QgsProcessingException(
self.tr('Invalid datasource: {}'.format(inFile)))
if schema:
inFile = '@dummy@'
for layer in srcDS:
layerDef = layer.GetLayerDefn()
layerName = layerDef.GetName()
vrt += '<OGRVRTLayer name="{}">'.format(self.XmlEsc(layerName))
vrt += '<SrcDataSource relativeToVRT="{}" shared="{}">{}</SrcDataSource>'.format(
1 if relative else 0, not schema, self.XmlEsc(inFile))
if schema:
vrt += '<SrcLayer>@dummy@</SrcLayer>'
else:
vrt += '<SrcLayer>{}</SrcLayer>'.format(
self.XmlEsc(layerName))
vrt += '<GeometryType>{}</GeometryType>'.format(
self.GeomType2Name(layerDef.GetGeomType()))
crs = layer.GetSpatialRef()
if crs is not None:
vrt += '<LayerSRS>{}</LayerSRS>'.format(
self.XmlEsc(crs.ExportToWkt()))
# Process all the fields.
for fieldIdx in range(layerDef.GetFieldCount()):
fieldDef = layerDef.GetFieldDefn(fieldIdx)
vrt += '<Field name="{}" type="{}"'.format(
self.XmlEsc(fieldDef.GetName()),
self.fieldType2Name(fieldDef.GetType()))
if not schema:
vrt += ' src="{}"'.format(
self.XmlEsc(fieldDef.GetName()))
if fieldDef.GetWidth() > 0:
vrt += ' width="{}"'.format(fieldDef.GetWidth())
if fieldDef.GetPrecision() > 0:
vrt += ' precision="{}"'.format(
fieldDef.GetPrecision())
vrt += '/>'
vrt += '</OGRVRTLayer>'
srcDS.Destroy()
if union:
vrt += '</OGRVRTUnionLayer>'
vrt += '</OGRVRTDataSource>'
#TODO: pretty-print XML
if outFile is not None:
with codecs.open(outFile, 'w') as f:
f.write(vrt)
return vrt
def processAlgorithm(self, parameters, context: QgsProcessingContext,
feedback: QgsProcessingFeedback):
"""Here is where the processing itself takes place."""
feedback.setProgress(0)
# init params
reach_layer = self.parameterAsVectorLayer(parameters, self.REACH_LAYER,
context)
wastewater_node_layer = self.parameterAsVectorLayer(
parameters, self.WASTEWATER_NODE_LAYER, context)
value_expression = self.parameterAsExpression(parameters,
self.VALUE_EXPRESSION,
context)
reach_pk_name = self.parameterAsFields(parameters, self.REACH_PK_NAME,
context)[0]
node_pk_name = self.parameterAsFields(parameters, self.NODE_PK_NAME,
context)[0]
node_from_fk_name = self.parameterAsFields(parameters,
self.NODE_FROM_FK_NAME,
context)[0]
node_to_fk_name = self.parameterAsFields(parameters,
self.NODE_TO_FK_NAME,
context)[0]
branch_behavior = self.parameterAsEnum(parameters,
self.BRANCH_BEHAVIOR, context)
create_loop_layer = self.parameterAsBool(parameters,
self.CREATE_LOOP_LAYER,
context)
if branch_behavior == 0:
aggregate_method = lambda values: min(values) if values else 0
elif branch_behavior == 1:
aggregate_method = lambda values: max(values) if values else 0
elif branch_behavior == 2:
aggregate_method = lambda values: statistics.mean(
values) if values else 0
else:
aggregate_method = lambda values: feedback.pushError(
'Aggregate method not implemented')
# create feature sink
fields = wastewater_node_layer.fields()
fields.append(QgsField('value', QVariant.Double))
(sink, dest_id) = self.parameterAsSink(parameters, self.OUTPUT,
context, fields,
QgsWkbTypes.Point,
reach_layer.sourceCrs())
loop_sink = None
loop_dest_id = None
if create_loop_layer:
(loop_sink,
loop_dest_id) = self.parameterAsSink(parameters, self.LOOP_OUTPUT,
context, fields,
QgsWkbTypes.Point,
reach_layer.sourceCrs())
if sink is None:
raise QgsProcessingException(
self.invalidSinkError(parameters, self.OUTPUT))
feature_count = reach_layer.featureCount()
reaches_by_from_node = dict()
reaches_by_id = dict()
expression = QgsExpression(value_expression)
context = QgsExpressionContext(
QgsExpressionContextUtils.globalProjectLayerScopes(reach_layer))
expression.prepare(context)
progress = 0
feedback.setProgressText(self.tr('Indexing reaches'))
for reach in reach_layer.getFeatures(QgsFeatureRequest()):
if reach[node_from_fk_name] == NULL:
continue
context.setFeature(reach)
value = expression.evaluate(context)
reach_obj = Reach(reach[node_from_fk_name], reach[node_to_fk_name],
value, reach.geometry())
reaches_by_from_node.setdefault(reach_obj.from_id,
[]).append(reach_obj)
reaches_by_id[reach[reach_pk_name]] = reach_obj
feedback.setProgress(progress / feature_count * 10)
progress += 1
loop_nodes = []
current_feature = 0
calculated_values = {}
feedback.setProgressText(self.tr('Analyzing network'))
for node in wastewater_node_layer.getFeatures():
from_node_id = node[node_pk_name]
processed_nodes = []
times = []
if from_node_id in reaches_by_from_node.keys():
for reach in reaches_by_from_node[from_node_id]:
times.append(
self.calculate_branch(reach, reaches_by_from_node,
reaches_by_id,
list(processed_nodes),
calculated_values,
aggregate_method, loop_nodes,
feedback))
if times:
time = aggregate_method(times)
else:
time = 0
current_feature += 1
calculated_values[node[node_pk_name]] = time
new_node = QgsFeature(node)
new_node.setFields(fields)
new_node.setAttributes(node.attributes() + [time])
sink.addFeature(new_node, QgsFeatureSink.FastInsert)
if create_loop_layer and from_node_id in loop_nodes:
loop_sink.addFeature(node, QgsFeatureSink.FastInsert)
feedback.setProgress(10 + current_feature / feature_count * 90)
result = {self.OUTPUT: dest_id}
if create_loop_layer:
result[self.LOOP_OUTPUT] = loop_dest_id
return result
