def execute(self, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a GeoAlgorithmExecutionException in case anything goes
wrong.
"""
if feedback is None:
feedback = QgsProcessingFeedback()
self.model = model
try:
self.setOutputCRS()
self.resolveOutputs()
self.evaluateParameterValues()
self.runPreExecutionScript(feedback)
self.processAlgorithm(feedback)
feedback.setProgress(100)
self.convertUnsupportedFormats(feedback)
self.runPostExecutionScript(feedback)
except GeoAlgorithmExecutionException as gaee:
lines = [self.tr("Error while executing algorithm")]
lines.append(traceback.format_exc())
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, gaee.msg)
raise GeoAlgorithmExecutionException(gaee.msg, lines, gaee)
except Exception as e:
# If something goes wrong and is not caught in the
# algorithm, we catch it here and wrap it
lines = [self.tr("Uncaught error while executing algorithm")]
lines.append(traceback.format_exc())
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, lines)
raise GeoAlgorithmExecutionException(str(e) + self.tr("\nSee log for more details"), lines, e)
def executeAlgorithm(alg, parameters, context=None, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a QgsProcessingException in case anything goes
wrong.
:param parameters:
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
#self.model = model
#self.setOutputCRS()
#self.resolveOutputs()
#self.evaluateParameterValues()
#self.runPreExecutionScript(feedback)
result, ok = alg.run(parameters, context, feedback)
#self.processAlgorithm(parameters, context, feedback)
feedback.setProgress(100)
return result, ok
def handleAlgorithmResults(alg, context, feedback=None, showResults=True):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context, typeHint=details.layerTypeHint)
if layer is not None:
set_layer_name(layer, details)
style = None
if details.outputName:
style = RenderingStyles.getStyle(alg.id(), details.outputName)
if style is None:
if layer.type() == QgsMapLayer.RasterLayer:
style = ProcessingConfig.getSetting(ProcessingConfig.RASTER_STYLE)
else:
if layer.geometryType() == QgsWkbTypes.PointGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POINT_STYLE)
elif layer.geometryType() == QgsWkbTypes.LineGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_LINE_STYLE)
else:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POLYGON_STYLE)
if style:
layer.loadNamedStyle(style)
details.project.addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
if details.postProcessor():
details.postProcessor().postProcessLayer(layer, context, feedback)
else:
wrongLayers.append(str(l))
except Exception:
QgsMessageLog.logMessage(QCoreApplication.translate('Postprocessing', "Error loading result layer:") + "\n" + traceback.format_exc(), 'Processing', Qgis.Critical)
wrongLayers.append(str(l))
i += 1
feedback.setProgress(100)
if wrongLayers:
msg = QCoreApplication.translate('Postprocessing', "The following layers were not correctly generated.")
msg += "<ul>" + "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += QCoreApplication.translate('Postprocessing', "You can check the 'Log Messages Panel' in QGIS main window to find more information about the execution of the algorithm.")
feedback.reportError(msg)
return len(wrongLayers) == 0
def __init__(self, algname=None):
QgsProcessingFeedback.__init__(self)
self.msg = []
self.progressMessageBar = \
iface.messageBar().createMessage(self.tr('Executing algorithm <i>{0}</i>'.format(algname if algname else '')))
self.progress = QProgressBar()
self.progressChanged.connect(self.progress.setValue)
self.progress.setMaximum(100)
self.progress.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
self.progressMessageBar.layout().addWidget(self.progress)
self.message_bar_item = iface.messageBar().pushWidget(self.progressMessageBar,
Qgis.Info)
def runGdal(commands, feedback=None):
if feedback is None:
feedback = QgsProcessingFeedback()
envval = os.getenv('PATH')
# We need to give some extra hints to get things picked up on OS X
isDarwin = False
try:
isDarwin = platform.system() == 'Darwin'
except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526
pass
if isDarwin and os.path.isfile(os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")):
# Looks like there's a bundled gdal. Let's use it.
os.environ['PATH'] = "{}{}{}".format(os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep, envval)
os.environ['DYLD_LIBRARY_PATH'] = os.path.join(QgsApplication.prefixPath(), "lib")
else:
# Other platforms should use default gdal finder codepath
settings = QgsSettings()
path = settings.value('/GdalTools/gdalPath', '')
if not path.lower() in envval.lower().split(os.pathsep):
envval += '{}{}'.format(os.pathsep, path)
os.putenv('PATH', envval)
fused_command = ' '.join([str(c) for c in commands])
QgsMessageLog.logMessage(fused_command, 'Processing', Qgis.Info)
feedback.pushInfo('GDAL command:')
feedback.pushCommandInfo(fused_command)
feedback.pushInfo('GDAL command output:')
success = False
retry_count = 0
while not success:
loglines = []
loglines.append('GDAL execution console output')
try:
with subprocess.Popen(
fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
) as proc:
for line in proc.stdout:
feedback.pushConsoleInfo(line)
loglines.append(line)
success = True
except IOError as e:
if retry_count < 5:
retry_count += 1
else:
raise IOError(
str(e) + u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}'.format(
len(loglines), u'\n'.join(loglines[-10:])))
QgsMessageLog.logMessage('\n'.join(loglines), 'Processing', Qgis.Info)
GdalUtils.consoleOutput = loglines
def executeAlgorithm(alg, parameters, context=None, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a GeoAlgorithmExecutionException in case anything goes
wrong.
:param parameters:
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext()
#self.model = model
try:
#self.setOutputCRS()
#self.resolveOutputs()
#self.evaluateParameterValues()
#self.runPreExecutionScript(feedback)
result = alg.run(parameters, context, feedback)
#self.processAlgorithm(parameters, context, feedback)
feedback.setProgress(100)
return result
#self.convertUnsupportedFormats(context, feedback)
#self.runPostExecutionScript(feedback)
except GeoAlgorithmExecutionException as gaee:
lines = [self.tr('Error while executing algorithm')]
lines = []
lines.append(traceback.format_exc())
feedback.reportError(gaee.msg)
QgsMessageLog.logMessage(gaee.msg, self.tr('Processing'), QgsMessageLog.CRITICAL)
raise GeoAlgorithmExecutionException(gaee.msg, lines, gaee)
#except Exception as e:
# If something goes wrong and is not caught in the
# algorithm, we catch it here and wrap it
#lines = [self.tr('Uncaught error while executing algorithm')]
# lines = []
# lines.append(traceback.format_exc())
#QgsMessageLog.logMessage('\n'.join(lines), self.tr('Processing'), QgsMessageLog.CRITICAL)
#raise GeoAlgorithmExecutionException(str(e) + self.tr('\nSee log for more details'), lines, e)
def helpUrl(self):
return QgsHelp.helpUrl("processing_algs/{}/{}".format(
self.provider().id(), self.id())).toString()
def runalg(alg, feedback=None):
"""Executes a given algorithm, showing its progress in the
progress object passed along.
Return true if everything went OK, false if the algorithm
could not be completed.
"""
if feedback is None:
feedback = QgsProcessingFeedback()
try:
alg.execute(feedback)
return True
except GeoAlgorithmExecutionException as e:
ProcessingLog.addToLog(sys.exc_info()[0], ProcessingLog.LOG_ERROR)
if feedback is not None:
feedback.reportError(e.msg)
return False
def execute(alg, parameters, context=None, feedback=None):
"""Executes a given algorithm, showing its progress in the
progress object passed along.
Return true if everything went OK, false if the algorithm
could not be completed.
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
try:
results, ok = alg.run(parameters, context, feedback)
return ok, results
except QgsProcessingException as e:
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing', Qgis.Critical)
if feedback is not None:
feedback.reportError(e.msg)
return False, {}
def execute(self, parameters, context=None, feedback=None, model=None):
"""The method to use to call a processing algorithm.
Although the body of the algorithm is in processAlgorithm(),
it should be called using this method, since it performs
some additional operations.
Raises a QgsProcessingException in case anything goes
wrong.
:param parameters:
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
self.model = model
try:
self.setOutputCRS()
self.resolveOutputs()
self.runPreExecutionScript(feedback)
self.processAlgorithm(parameters, context, feedback)
feedback.setProgress(100)
self.convertUnsupportedFormats(context, feedback)
self.runPostExecutionScript(feedback)
except QgsProcessingException as gaee:
lines = [self.tr('Error while executing algorithm')]
lines.append(traceback.format_exc())
QgsMessageLog.logMessage(gaee.msg, self.tr('Processing'), QgsMessageLog.CRITICAL)
raise QgsProcessingException(gaee.msg, lines, gaee)
except Exception as e:
# If something goes wrong and is not caught in the
# algorithm, we catch it here and wrap it
lines = [self.tr('Uncaught error while executing algorithm')]
lines.append(traceback.format_exc())
QgsMessageLog.logMessage('\n'.join(lines), self.tr('Processing'), QgsMessageLog.CRITICAL)
raise QgsProcessingException(str(e) + self.tr('\nSee log for more details'), lines, e)
def execute(commands, feedback=None):
cmds = []
cmds.append(os.path.join(mpichDirectory(), "mpiexec"))
processes = ProcessingConfig.getSetting(TAUDEM_PROCESSES)
if int(processes) <= 0:
processes = 1
cmds.append("-n")
cmds.append(processes)
cmds.extend(commands)
if feedback is None:
feedback = QgsProcessingFeedback()
fused_command = " ".join([str(c) for c in cmds])
QgsMessageLog.logMessage(fused_command, "Processing", QgsMessageLog.INFO)
feedback.pushInfo("TauDEM command:")
feedback.pushCommandInfo(fused_command)
feedback.pushInfo("TauDEM command output:")
loglines = []
with subprocess.Popen(fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True) as proc:
try:
for line in iter(proc.stdout.readline, ""):
feedback.pushConsoleInfo(line)
loglines.append(line)
except:
pass
if ProcessingConfig.getSetting(TAUDEM_VERBOSE):
QgsMessageLog.logMessage("\n".join(loglines), "Processing", QgsMessageLog.INFO)
def handleAlgorithmResults(alg, feedback=None, showResults=True):
wrongLayers = []
htmlResults = False
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers'))
i = 0
for out in alg.outputs:
feedback.setProgress(100 * i / float(len(alg.outputs)))
if out.hidden or not out.open:
continue
if isinstance(out, (OutputRaster, OutputVector, OutputTable)):
try:
if hasattr(out, "layer") and out.layer is not None:
out.layer.setName(out.description)
QgsProject.instance().addMapLayers([out.layer])
else:
if ProcessingConfig.getSetting(
ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
name = os.path.basename(out.value)
else:
name = out.description
dataobjects.load(out.value, name, alg.crs,
RenderingStyles.getStyle(alg.commandLineName(),
out.name))
except Exception:
ProcessingLog.addToLog(ProcessingLog.LOG_ERROR,
"Error loading result layer:\n" + traceback.format_exc())
wrongLayers.append(out.description)
elif isinstance(out, OutputHTML):
ProcessingResults.addResult(out.description, out.value)
htmlResults = True
i += 1
QApplication.restoreOverrideCursor()
if wrongLayers:
msg = "The following layers were not correctly generated.<ul>"
msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += "You can check the log messages to find more information about the execution of the algorithm"
feedback.reportError(msg)
if showResults and htmlResults and not wrongLayers:
dlg = ResultsDialog()
dlg.exec_()
return len(wrongLayers) == 0
def refreshFeedback():
# refresh feedback track to not carry "bias" to next execution
workflow.feedback.progressChanged.disconnect(self.setProgress)
del workflow.feedback
workflow.feedback = QgsProcessingFeedback()
workflow.feedback.progressChanged.connect(self.setProgress)
def handleAlgorithmResults(alg, context, feedback=None, showResults=True, parameters={}):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context, typeHint=details.layerTypeHint)
if layer is not None:
set_layer_name(layer, details)
'''If running a model, the execution will arrive here when an algorithm that is part of
that model is executed. We check if its output is a final otuput of the model, and
adapt the output name accordingly'''
outputName = details.outputName
expcontext = QgsExpressionContext()
scope = QgsExpressionContextScope()
expcontext.appendScope(scope)
for out in alg.outputDefinitions():
outValue = parameters[out.name()]
if hasattr(outValue, "sink"):
outValue = outValue.sink.valueAsString(expcontext)[0]
else:
outValue = str(outValue)
if outValue == l:
outputName = out.name()
break
style = None
if outputName:
style = RenderingStyles.getStyle(alg.id(), outputName)
if style is None:
if layer.type() == QgsMapLayer.RasterLayer:
style = ProcessingConfig.getSetting(ProcessingConfig.RASTER_STYLE)
else:
if layer.geometryType() == QgsWkbTypes.PointGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POINT_STYLE)
elif layer.geometryType() == QgsWkbTypes.LineGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_LINE_STYLE)
else:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POLYGON_STYLE)
if style:
layer.loadNamedStyle(style)
details.project.addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
if details.postProcessor():
details.postProcessor().postProcessLayer(layer, context, feedback)
else:
wrongLayers.append(str(l))
except Exception:
QgsMessageLog.logMessage(QCoreApplication.translate('Postprocessing', "Error loading result layer:") + "\n" + traceback.format_exc(), 'Processing', Qgis.Critical)
wrongLayers.append(str(l))
i += 1
feedback.setProgress(100)
if wrongLayers:
msg = QCoreApplication.translate('Postprocessing', "The following layers were not correctly generated.")
msg += "<ul>" + "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += QCoreApplication.translate('Postprocessing', "You can check the 'Log Messages Panel' in QGIS main window to find more information about the execution of the algorithm.")
feedback.reportError(msg)
return len(wrongLayers) == 0
def handleAlgorithmResults(alg, context, feedback=None, showResults=True):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context)
if layer is not None:
layer.setName(details.name)
details.project.addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
else:
name = details.name
if ProcessingConfig.getSetting(
ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
name = os.path.basename(l)
dataobjects.load(l, name, alg.crs,
RenderingStyles.getStyle(alg.id(), l))
except Exception:
QgsMessageLog.logMessage("Error loading result layer:\n" + traceback.format_exc(), 'Processing', QgsMessageLog.CRITICAL)
#wrongLayers.append(out.description())
wrongLayers.append(l)
# for out in alg.outputs:
# feedback.setProgress(100 * i / float(len(alg.outputs)))
# if out.flags() & QgsProcessingParameterDefinition.FlagHidden or not out.open:
# continue
# if isinstance(out, (OutputRaster, OutputVector, OutputTable)):
# try:
# layer = QgsProcessingUtils.mapLayerFromString(out.value, context)
# if layer:
# layer.setName(out.description)
# QgsProject.instance().addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
# else:
# if ProcessingConfig.getSetting(
# ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
# name = os.path.basename(out.value)
# else:
# name = out.description
#
# isRaster = True if isinstance(out, OutputRaster) else False
# dataobjects.load(out.value, name, alg.crs,
# RenderingStyles.getStyle(alg.id(), out.name),
# isRaster)
# except Exception:
# QgsMessageLog.logMessage("Error loading result layer:\n" + traceback.format_exc(), 'Processing', QgsMessageLog.CRITICAL)
# wrongLayers.append(out.description)
# elif isinstance(out, OutputHTML):
# resultsList.addResult(alg.icon(), out.description, out.value)
i += 1
QApplication.restoreOverrideCursor()
if wrongLayers:
msg = "The following layers were not correctly generated.<ul>"
msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += "You can check the log messages to find more information about the execution of the algorithm"
feedback.reportError(msg)
return len(wrongLayers) == 0
def check_for_unsupported_property(name, # pylint:disable=too-many-branches
symbol,
feedback: QgsProcessingFeedback,
sink):
"""
Checks for properties of ESRI symbols which have no equivalent in QGIS,
and warns
"""
for c in symbol.children():
StyleToQgisXml.check_for_unsupported_property(name, c, feedback, sink)
try:
if symbol.random:
feedback.reportError(
'Warning: random marker fills are not supported by QGIS (considering sponsoring this feature!)')
if sink:
f = QgsFeature()
f.setAttributes([name, 'Random marker fills not supported by QGIS'])
sink.addFeature(f)
except AttributeError:
pass
if isinstance(symbol, (MarkerFillSymbolLayer, PictureFillSymbolLayer)):
if symbol.offset_x or symbol.offset_y:
feedback.reportError(
'Warning: marker fill offset X or Y is not supported by QGIS (considering sponsoring this feature!)')
if sink:
f = QgsFeature()
f.setAttributes([name, 'Marker fill offset X or Y not supported by QGIS'])
sink.addFeature(f)
if isinstance(symbol, PictureFillSymbolLayer):
if symbol.separation_x or symbol.separation_y:
feedback.reportError(
'Warning: picture fill separation X or Y is not supported by QGIS (considering sponsoring this feature!)')
if sink:
f = QgsFeature()
f.setAttributes([name, 'Picture fill separation X or Y not supported by QGIS'])
sink.addFeature(f)
if isinstance(symbol, HashLineSymbolLayer):
feedback.reportError(
'QGIS does not have a hash line symbol type (considering sponsoring this feature!)')
if sink:
f = QgsFeature()
f.setAttributes([name, 'Hash line symbols are not supported by QGIS'])
sink.addFeature(f)
try:
if symbol.halo:
feedback.reportError(
'Halos are not supported by QGIS (considering sponsoring this feature!)')
if sink:
f = QgsFeature()
f.setAttributes([name, 'Marker halos not supported by QGIS'])
sink.addFeature(f)
except AttributeError:
pass
def runGdal(commands, feedback=None):
if feedback is None:
feedback = QgsProcessingFeedback()
envval = os.getenv('PATH')
# We need to give some extra hints to get things picked up on OS X
isDarwin = False
try:
isDarwin = platform.system() == 'Darwin'
except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526
pass
if isDarwin and os.path.isfile(
os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")):
# Looks like there's a bundled gdal. Let's use it.
os.environ['PATH'] = "{}{}{}".format(
os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep,
envval)
os.environ['DYLD_LIBRARY_PATH'] = os.path.join(
QgsApplication.prefixPath(), "lib")
else:
# Other platforms should use default gdal finder codepath
settings = QgsSettings()
path = settings.value('/GdalTools/gdalPath', '')
if not path.lower() in envval.lower().split(os.pathsep):
envval += '{}{}'.format(os.pathsep, path)
os.putenv('PATH', envval)
fused_command = ' '.join([str(c) for c in commands])
QgsMessageLog.logMessage(fused_command, 'Processing',
QgsMessageLog.INFO)
feedback.pushInfo('GDAL command:')
feedback.pushCommandInfo(fused_command)
feedback.pushInfo('GDAL command output:')
success = False
retry_count = 0
while not success:
loglines = []
loglines.append('GDAL execution console output')
try:
with subprocess.Popen(
fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
) as proc:
for line in proc.stdout:
feedback.pushConsoleInfo(line)
loglines.append(line)
success = True
except IOError as e:
if retry_count < 5:
retry_count += 1
else:
raise IOError(
e.message +
u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}'
.format(len(loglines), u'\n'.join(loglines[-10:])))
QgsMessageLog.logMessage('\n'.join(loglines), 'Processing',
QgsMessageLog.INFO)
GdalUtils.consoleOutput = loglines
def processAlgorithm(self, parameters, context,
feedback: QgsProcessingFeedback):
# Input / Output
source = self.parameterAsSource(parameters, self.INPUT, context)
(sink, dest_id) = self.create_output_sink(parameters, context,
source.sourceCrs())
# Filter parameters
observation_type_indices = self.parameterAsEnums(
parameters, self.OBSERVATION_TYPE, context)
observation_types = list(
map(lambda i: self.OBSERVATION_TYPES[i][0],
observation_type_indices))
from_date = None
from_date_string = self.parameterAsString(parameters, self.FROM_DATE,
context)
if from_date_string:
from_date = datetime.fromisoformat(from_date_string)
to_date = None
to_date_string = self.parameterAsString(parameters, self.TO_DATE,
context)
if to_date_string:
to_date = datetime.fromisoformat(to_date_string)
fire_connection_string = self.settings.value("fire_connection_string")
fireDb = FireDb(fire_connection_string, debug=True)
features = list(source.getFeatures())
total_num_features = len(features)
total_num_features_processed = 0
# for current, feature in enumerate(features):
for feature in features:
if feedback.isCanceled():
return {}
wkt = feature.geometry().asWkt().upper()
geometry = Geometry(wkt)
observations = fireDb.hent_observationer_naer_geometri(
geometri=geometry,
afstand=0,
tid_fra=from_date,
tid_til=to_date)
pid_list = self.get_pids_from_observations(observations)
geometriobjekter = self.get_geometriobjekter_from_pids(
fireDb, pid_list)
idents = self.get_idents_from_pids(fireDb, pid_list)
feedback.setProgressText(
"Fandt {antal} observationer".format(antal=len(observations)))
feedback.setProgressText("Fandt {antal} geometriobjekter".format(
antal=len(geometriobjekter)))
feedback.setProgressText(
"Fandt {antal} idents".format(antal=len(idents)))
for current, observation in enumerate(observations):
observation_type_id = observation.observationstypeid
if observation_type_id in observation_types:
feature = self.create_feature_from_observation(
observation, geometriobjekter, idents, feedback)
if feature:
sink.addFeature(feature, QgsFeatureSink.FastInsert)
total_num_features_processed = total_num_features_processed + 1
feedback.setProgress(total_num_features_processed /
total_num_features)
if feedback.isCanceled():
return {}
apply_theme = self.parameterAsBool(parameters, self.APPLY_THEME,
context)
if apply_theme:
style_file = os.path.join(os.path.dirname(__file__), "..",
"styles", "observation.qml")
alg_params = {"INPUT": dest_id, "STYLE": style_file}
processing.run(
"qgis:setstyleforvectorlayer",
alg_params,
context=context,
feedback=feedback,
is_child_algorithm=True,
)
return {self.OUTPUT: dest_id}
def testReadOgr(self):
"""
Test reading vector inputs
"""
alg = RAlgorithm(
description_file=os.path.join(test_data_path, 'test_vectorin.rsx'))
alg.initAlgorithm()
context = QgsProcessingContext()
feedback = QgsProcessingFeedback()
script = alg.build_import_commands(
{'Layer': os.path.join(test_data_path, 'lines.shp')}, context,
feedback)
USE_NEW_API = Qgis.QGIS_VERSION_INT >= 30900 and hasattr(
QgsProcessingAlgorithm,
'parameterAsCompatibleSourceLayerPathAndLayerName')
if USE_NEW_API:
self.assertEqual(
script[0], 'Layer <- readOGR("{}")'.format(
os.path.join(test_data_path, 'lines.shp')))
else:
self.assertEqual(
script[0], 'Layer <- readOGR("{}")'.format(
os.path.join(test_data_path, 'lines.shp')))
script = alg.build_import_commands(
{
'Layer':
os.path.join(test_data_path, 'lines.shp').replace('/', '\\')
}, context, feedback)
if USE_NEW_API:
self.assertEqual(
script[0], 'Layer <- readOGR("{}")'.format(
os.path.join(test_data_path, 'lines.shp')))
else:
self.assertEqual(
script[0], 'Layer <- readOGR("{}")'.format(
os.path.join(test_data_path, 'lines.shp')))
vl = QgsVectorLayer(
os.path.join(test_data_path, 'test_gpkg.gpkg') +
'|layername=points')
self.assertTrue(vl.isValid())
vl2 = QgsVectorLayer(
os.path.join(test_data_path, 'test_gpkg.gpkg') +
'|layername=lines')
self.assertTrue(vl2.isValid())
script = alg.build_import_commands({
'Layer': vl,
'Layer2': vl2
}, context, feedback)
if USE_NEW_API:
# use the newer api and avoid unnecessary layer translation
self.assertEqual(script, [
'Layer <- readOGR("{}", layer="points")'.format(
os.path.join(test_data_path, 'test_gpkg.gpkg')),
'Layer2 <- readOGR("{}", layer="lines")'.format(
os.path.join(test_data_path, 'test_gpkg.gpkg'))
])
else:
# older version, forced to use inefficient api
self.assertIn('Layer <- readOGR("/tmp', script[0])
self.assertIn('Layer2 <- readOGR("/tmp', script[1])
def handleAlgorithmResults(alg, context, feedback=None, showResults=True):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(
QCoreApplication.translate('Postprocessing',
'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(
100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context)
if layer is not None:
if not ProcessingConfig.getSetting(
ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
layer.setName(details.name)
style = None
if details.outputName:
style = RenderingStyles.getStyle(alg.id(),
details.outputName)
if style is None:
if layer.type() == QgsMapLayer.RasterLayer:
style = ProcessingConfig.getSetting(
ProcessingConfig.RASTER_STYLE)
else:
if layer.geometryType() == QgsWkbTypes.PointGeometry:
style = ProcessingConfig.getSetting(
ProcessingConfig.VECTOR_POINT_STYLE)
elif layer.geometryType() == QgsWkbTypes.LineGeometry:
style = ProcessingConfig.getSetting(
ProcessingConfig.VECTOR_LINE_STYLE)
else:
style = ProcessingConfig.getSetting(
ProcessingConfig.VECTOR_POLYGON_STYLE)
if style:
layer.loadNamedStyle(style)
details.project.addMapLayer(
context.temporaryLayerStore().takeMapLayer(layer))
else:
wrongLayers.append(str(l))
except Exception:
QgsMessageLog.logMessage(
QCoreApplication.translate(
'Postprocessing', "Error loading result layer:") + "\n" +
traceback.format_exc(), 'Processing', Qgis.Critical)
wrongLayers.append(str(l))
i += 1
feedback.setProgress(100)
if wrongLayers:
msg = QCoreApplication.translate(
'Postprocessing',
"The following layers were not correctly generated.")
msg += "<ul>" + "".join(["<li>%s</li>" % lay
for lay in wrongLayers]) + "</ul>"
msg += QCoreApplication.translate(
'Postprocessing',
"You can check the 'Log Messages Panel' in QGIS main window to find more information about the execution of the algorithm."
)
feedback.reportError(msg)
return len(wrongLayers) == 0
# Initialize processing
Processing.initialize()
# Add Processing providers
reg = app.processingRegistry()
# lizsync provider
from lizsync.processing.provider import LizsyncProvider
reg.addProvider(LizsyncProvider())
# Native QGIS provider
# reg.addProvider(QgsNativeAlgorithms())
# Get parameters
input_alg = sys.argv[1]
parameters = sys.argv[2]
print(input_alg)
print(parameters)
input_params = json.loads(parameters)
# Run Alg
from qgis.core import QgsProcessingFeedback
feedback = QgsProcessingFeedback()
from processing import run as processing_run
res = processing_run(input_alg, input_params, feedback=feedback)
print("RESULT = %s" % json.dumps(res))
# Exit
app.exitQgis()
def testInputs(self):
"""
Test creation of script with algorithm inputs
"""
alg = RAlgorithm(description_file=os.path.join(test_data_path,
'test_algorithm_2.rsx'))
alg.initAlgorithm()
context = QgsProcessingContext()
feedback = QgsProcessingFeedback()
# enum evaluation
script = alg.build_import_commands({'in_enum': 0}, context, feedback)
self.assertIn('in_enum <- 0', script)
# boolean evaluation
script = alg.build_import_commands({'in_bool': True}, context,
feedback)
self.assertIn('in_bool <- TRUE', script)
script = alg.build_import_commands({'in_bool': False}, context,
feedback)
self.assertIn('in_bool <- FALSE', script)
# number evaluation
script = alg.build_import_commands({'in_number': None}, context,
feedback)
self.assertIn('in_number <- NULL', script)
script = alg.build_import_commands({'in_number': 5}, context, feedback)
self.assertIn('in_number <- 5.0', script)
script = alg.build_import_commands({'in_number': 5.5}, context,
feedback)
self.assertIn('in_number <- 5.5', script)
# folder destination
script = alg.build_import_commands(
{'param_folder_dest': '/tmp/processing/test_algorithm_2_r/'},
context, feedback)
# file destination
script = alg.build_import_commands(
{
'param_html_dest':
'/tmp/processing/test_algorithm_2_r/dest.html'
}, context, feedback)
self.assertIn(
'param_html_dest <- "/tmp/processing/test_algorithm_2_r/dest.html"',
script)
script = alg.build_import_commands(
{
'param_file_dest':
'/tmp/processing/test_algorithm_2_r/dest.file'
}, context, feedback)
self.assertIn(
'param_file_dest <- "/tmp/processing/test_algorithm_2_r/dest.file"',
script)
script = alg.build_import_commands(
{'param_csv_dest': '/tmp/processing/test_algorithm_2_r/dest.csv'},
context, feedback)
self.assertIn(
'param_csv_dest <- "/tmp/processing/test_algorithm_2_r/dest.csv"',
script)
def handleAlgorithmResults(alg, context, feedback=None, showResults=True):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(
QCoreApplication.translate('Postprocessing',
'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
feedback.setProgress(100 * i /
float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context)
if layer is not None:
layer.setName(details.name)
details.project.addMapLayer(
context.temporaryLayerStore().takeMapLayer(layer))
else:
name = details.name
if ProcessingConfig.getSetting(
ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
name = os.path.basename(l)
dataobjects.load(l, name, alg.crs,
RenderingStyles.getStyle(alg.id(), l))
except Exception:
QgsMessageLog.logMessage(
"Error loading result layer:\n" + traceback.format_exc(),
'Processing', QgsMessageLog.CRITICAL)
#wrongLayers.append(out.description())
wrongLayers.append(l)
# for out in alg.outputs:
# feedback.setProgress(100 * i / float(len(alg.outputs)))
# if out.flags() & QgsProcessingParameterDefinition.FlagHidden or not out.open:
# continue
# if isinstance(out, (OutputRaster, OutputVector, OutputTable)):
# try:
# layer = QgsProcessingUtils.mapLayerFromString(out.value, context)
# if layer:
# layer.setName(out.description)
# QgsProject.instance().addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
# else:
# if ProcessingConfig.getSetting(
# ProcessingConfig.USE_FILENAME_AS_LAYER_NAME):
# name = os.path.basename(out.value)
# else:
# name = out.description
#
# isRaster = True if isinstance(out, OutputRaster) else False
# dataobjects.load(out.value, name, alg.crs,
# RenderingStyles.getStyle(alg.id(), out.name),
# isRaster)
# except Exception:
# QgsMessageLog.logMessage("Error loading result layer:\n" + traceback.format_exc(), 'Processing', QgsMessageLog.CRITICAL)
# wrongLayers.append(out.description)
# elif isinstance(out, OutputHTML):
# resultsList.addResult(alg.icon(), out.description, out.value)
# i += 1
QApplication.restoreOverrideCursor()
if wrongLayers:
msg = "The following layers were not correctly generated.<ul>"
msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += "You can check the log messages to find more information about the execution of the algorithm"
feedback.reportError(msg)
return len(wrongLayers) == 0
def testOgr2PostGis(self):
context = QgsProcessingContext()
feedback = QgsProcessingFeedback()
source = os.path.join(testDataPath, 'polys.gml')
source_with_space = os.path.join(testDataPath, 'filename with spaces.gml')
alg = OgrToPostGis()
alg.initAlgorithm()
self.assertEqual(
alg.getConsoleCommands({'INPUT': source}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source_with_space}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 "' + source_with_space + '" filename_with_spaces '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.filename_with_spaces -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'HOST': 'google.com'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=google.com port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'PORT': 3333}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=3333 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'USER': '******'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public user=kevin_bacon" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'DBNAME': 'secret_stuff'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 dbname=secret_stuff active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'PASSWORD': '******'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 password=passw0rd active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'SCHEMA': 'desktop'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=desktop" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln desktop.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'TABLE': 'out_table'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.out_table -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'PK': ''}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'PK': 'new_fid'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=new_fid -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'PK': '',
'PRIMARY_KEY': 'objectid'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=objectid -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'PK': 'new_id',
'PRIMARY_KEY': 'objectid'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=new_id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'GEOCOLUMN': 'my_geom'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=my_geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'DIM': 1}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=3 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'SIMPLIFY': 5}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -simplify 5 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'SEGMENTIZE': 4}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -segmentize 4 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'SPAT': QgsRectangle(1, 2, 3, 4)}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -spat 1.0 2.0 3.0 4.0 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELDS': ['f1', 'f2']}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 -select "f1,f2" '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'WHERE': '0=1'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -where "0=1" -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'GT': 2}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -gt 2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'OVERWRITE': False}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'APPEND': True}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-append -overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'ADDFIELDS': True}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-addfields -overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'LAUNDER': True}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-lco LAUNDER=NO -overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'INDEX': True}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-lco SPATIAL_INDEX=OFF -overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'SKIPFAILURES': True}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -skipfailures -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'PROMOTETOMULTI': False}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'PRECISION': False}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI -lco PRECISION=NO'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'OPTIONS': 'blah'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI blah'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'SHAPE_ENCODING': 'blah'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES --config SHAPE_ENCODING "blah" -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'GTYPE': 4}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -nlt LINESTRING -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'A_SRS': 'EPSG:3111'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -a_srs EPSG:3111 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'A_SRS': QgsCoordinateReferenceSystem('EPSG:3111')}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -a_srs EPSG:3111 -nlt PROMOTE_TO_MULTI'])
custom_crs = 'proj4: +proj=utm +zone=36 +south +a=6378249.145 +b=6356514.966398753 +towgs84=-143,-90,-294,0,0,0,0 +units=m +no_defs'
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'A_SRS': custom_crs}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -a_srs EPSG:20936 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'T_SRS': 'EPSG:3111'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -t_srs EPSG:3111 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'T_SRS': QgsCoordinateReferenceSystem('EPSG:3111')}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -t_srs EPSG:3111 -nlt PROMOTE_TO_MULTI'])
custom_crs = 'proj4: +proj=utm +zone=36 +south +a=6378249.145 +b=6356514.966398753 +towgs84=-143,-90,-294,0,0,0,0 +units=m +no_defs'
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'T_SRS': custom_crs}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -t_srs EPSG:20936 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'S_SRS': 'EPSG:3111'}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -s_srs EPSG:3111 -nlt PROMOTE_TO_MULTI'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'S_SRS': QgsCoordinateReferenceSystem('EPSG:3111')}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -s_srs EPSG:3111 -nlt PROMOTE_TO_MULTI'])
custom_crs = 'proj4: +proj=utm +zone=36 +south +a=6378249.145 +b=6356514.966398753 +towgs84=-143,-90,-294,0,0,0,0 +units=m +no_defs'
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'S_SRS': custom_crs}, context, feedback),
['ogr2ogr',
'-progress --config PG_USE_COPY YES -f PostgreSQL "PG:host=localhost port=5432 active_schema=public" '
'-lco DIM=2 ' + source + ' polys2 '
'-overwrite -lco GEOMETRY_NAME=geom -lco FID=id -nln public.polys2 -s_srs EPSG:20936 -nlt PROMOTE_TO_MULTI'])
def testDissolve(self):
context = QgsProcessingContext()
feedback = QgsProcessingFeedback()
source = os.path.join(testDataPath, 'polys.gml')
source_with_space = os.path.join(testDataPath, 'filename with spaces.gml')
alg = Dissolve()
alg.initAlgorithm()
with tempfile.TemporaryDirectory() as outdir:
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry FROM \'polys2\'" ' +
'-f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry, my_field FROM \'polys2\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source_with_space,
'FIELD': 'my_field',
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
'"' + source_with_space + '" ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry, my_field FROM \'filename_with_spaces\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'GEOMETRY': 'the_geom',
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(the_geom) AS the_geom, my_field FROM \'polys2\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'KEEP_ATTRIBUTES': False,
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry, my_field FROM \'polys2\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'KEEP_ATTRIBUTES': False,
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry FROM \'polys2\'" ' +
'-f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'EXPLODE_COLLECTIONS': True,
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry, my_field FROM \'polys2\' ' +
'GROUP BY my_field" -explodecollections -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'COUNT_FEATURES': True,
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry, my_field, COUNT(geometry) AS count FROM \'polys2\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'COUNT_FEATURES': True,
'GEOMETRY': 'the_geom',
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(the_geom) AS the_geom, my_field, COUNT(the_geom) AS count FROM \'polys2\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'COMPUTE_AREA': True,
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry, my_field, SUM(ST_Area(geometry)) AS area, ' +
'ST_Perimeter(ST_Union(geometry)) AS perimeter FROM \'polys2\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'COMPUTE_AREA': True,
'GEOMETRY': 'the_geom',
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(the_geom) AS the_geom, my_field, SUM(ST_Area(the_geom)) AS area, ' +
'ST_Perimeter(ST_Union(the_geom)) AS perimeter FROM \'polys2\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'COMPUTE_STATISTICS': True,
'STATISTICS_ATTRIBUTE': 'my_val',
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry, my_field, ' +
'SUM(my_val) AS sum, MIN(my_val) AS min, MAX(my_val) AS max, AVG(my_val) AS avg FROM \'polys2\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
# compute stats without stats attribute, and vice versa (should be ignored)
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'COMPUTE_STATISTICS': True,
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry, my_field FROM \'polys2\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'STATISTICS_ATTRIBUTE': 'my_val',
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry, my_field FROM \'polys2\' ' +
'GROUP BY my_field" -f "ESRI Shapefile"'])
self.assertEqual(
alg.getConsoleCommands({'INPUT': source,
'FIELD': 'my_field',
'OPTIONS': 'my opts',
'OUTPUT': outdir + '/check.shp'}, context, feedback),
['ogr2ogr',
outdir + '/check.shp ' +
source + ' ' +
'-nlt PROMOTE_TO_MULTI -dialect sqlite -sql "SELECT ST_Union(geometry) AS geometry, my_field FROM \'polys2\' ' +
'GROUP BY my_field" "my opts" -f "ESRI Shapefile"'])
def initialize_feedback(self):
self.progress.setValue(0)
self.progress.setVisible(False)
self.feedback = QgsProcessingFeedback()
self.feedback.progressChanged.connect(self.progress_changed)
self.buttonBox.button(QDialogButtonBox.Ok).setEnabled(True)
def generate_od_routes(
network_layer: QgsVectorLayer,
origin_layer: QgsVectorLayer,
poi_layer: QgsVectorLayer,
size_field: str,
class_field: str,
work_layer: QgsVectorLayer = None,
work_size_field: str = None,
school_layer: QgsVectorLayer = None,
school_size_field: str = None,
origin_weight_field: str = None,
socio_data=None,
health_data=None,
diversity_data=None,
join_on: str = None,
max_distance: int = 25000,
return_layer: bool = True,
return_raw: bool = False,
feedback: QgsProcessingFeedback = None,
) -> QgsVectorLayer:
"""
Shortest path algorithm based on Dijkstra's algorithm
:param network_layer: road network
:param points_layer: combined from to points
:param relations_data: tabular from to id data
:param origin_field: name of from field
:param destination_field: name of to field
:param max_distance: maximum distance/cost
:param crs: output layer crs
"""
if not network_layer.wkbType() & QgsWkbTypes.LineString:
raise Exception('Network layer must be of type LineString')
crs = network_layer.crs()
## prepare graph
director = QgsVectorLayerDirector(
source=network_layer,
directionFieldId=-1,
directDirectionValue='',
reverseDirectionValue='',
bothDirectionValue='',
defaultDirection=QgsVectorLayerDirector.DirectionBoth,
)
# First strategy is for actual shortest distance calculation
director.addStrategy(QgsNetworkDistanceStrategy()) # 0
# Second strategy is a hack to be able to recover the edge id
director.addStrategy(SaveFidStrategy()) # 1
builder = QgsGraphBuilder(crs)
## spatial index
poi_sidx = QgsSpatialIndex(poi_layer)
work_sidx = QgsSpatialIndex(work_layer)
school_sidx = QgsSpatialIndex(school_layer)
## prepare points
orig_n = len(origin_layer)
poi_n = len(poi_layer)
work_n = len(work_layer) if work_layer else 0
school_n = len(school_layer) if school_layer else 0
dest_n = poi_n + work_n + school_n
orig_points = [None] * orig_n
orig_sizes = np.zeros(orig_n, dtype=float)
if socio_data:
orig_socio = np.zeros(orig_n, dtype=float)
dest_points = [None] * dest_n
dest_sizes = [None] * dest_n
dest_fids = [None] * dest_n
dest_cats = [None] * dest_n
orig_id_field = 'deso'
for i, feat in enumerate(origin_layer.getFeatures()):
orig_points[i] = feat.geometry().asPoint()
orig_sizes[i] = feat[size_field] * (
feat[origin_weight_field] if origin_weight_field else 1
)
if socio_data:
orig_socio[i] = socio_data[
feat[orig_id_field]
] # FIXME: check if all origins have data
if socio_data:
orig_socio = orig_socio / np.mean(orig_socio)
orig_sizes *= orig_socio
for i, feat in enumerate(poi_layer.getFeatures()):
dest_points[i] = feat.geometry().asPoint()
dest_fids[i] = feat.id()
dest_sizes[i] = 1 # TODO: dest size
dest_cats[i] = poi_class_map.get(feat[class_field])
if work_layer:
for i, feat in enumerate(work_layer.getFeatures(), start=poi_n):
dest_points[i] = feat.geometry().asPoint()
dest_fids[i] = feat.id()
dest_sizes[i] = feat[work_size_field] # TODO: dest size
dest_cats[i] = 'work'
if school_layer:
for i, feat in enumerate(school_layer.getFeatures(), start=(poi_n + work_n)):
dest_points[i] = feat.geometry().asPoint()
dest_fids[i] = feat.id()
dest_sizes[i] = feat[school_size_field] # TODO: dest size
dest_cats[i] = 'school'
# points = [origin.point for origin in origins_data] + [
# dest.point for dest in dests_data
# ]
if feedback is None:
feedback = QgsProcessingFeedback()
def progress(p):
if int(10 * p % 100) == 0:
print(f'{int(p):#3d}%')
feedback.progressChanged.connect(progress)
with timing('build network graph'):
tied_points = director.makeGraph(
builder, orig_points + dest_points, feedback=feedback
)
graph = builder.graph()
orig_tied_points = tied_points[:orig_n]
dest_tied_points = tied_points[orig_n:]
poi_tied_points = dest_tied_points[:poi_n]
work_tied_points = dest_tied_points[poi_n : poi_n + work_n]
school_tied_points = dest_tied_points[poi_n + work_n :]
dest_fid_to_tied_points = dict(zip(dest_fids, enumerate(dest_tied_points)))
poi_fid_to_tied_points = dict(zip(dest_fids[:poi_n], enumerate(poi_tied_points)))
work_fid_to_tied_points = dict(
zip(dest_fids[poi_n : poi_n + work_n], enumerate(work_tied_points, start=poi_n))
)
school_fid_to_tied_points = dict(
zip(
dest_fids[poi_n + work_n :],
enumerate(school_tied_points, start=poi_n + work_n),
)
)
orig_dests = [None] * orig_n
for i, point in enumerate(orig_points):
orig_dests[i] = (
[
poi_fid_to_tied_points[fid]
for fid in poi_sidx.nearestNeighbor(
point, neighbors=MAX_NEIGHBORS, maxDistance=max_distance
)
]
+ [
work_fid_to_tied_points[fid]
for fid in work_sidx.nearestNeighbor(
point, neighbors=MAX_NEIGHBORS, maxDistance=max_distance
)
]
+ [
school_fid_to_tied_points[fid]
for fid in school_sidx.nearestNeighbor(
point, neighbors=MAX_NEIGHBORS, maxDistance=max_distance
)
]
)
step = 100.0 / orig_n
time_dijkstra = 0.0
time_find = 0.0
time_route = 0.0
with timing('calculate connecting routes'):
routes = []
# for i, (origin_fid, dest_fids) in enumerate(od_data):
for i, (orig_point, dests) in enumerate(zip(orig_tied_points, orig_dests)):
origin_vertex_id = graph.findVertex(orig_point)
# Calculate the tree and cost using the distance strategy (#0)
ts = time()
(tree, cost) = QgsGraphAnalyzer.dijkstra(graph, origin_vertex_id, 0)
time_dijkstra += time() - ts
for j, dest_point in dests:
if feedback.isCanceled():
return
if dest_sizes[j] <= 0:
continue
category = dest_cats[j]
if category is None:
continue
ts = time()
dest_vertex_id = graph.findVertex(dest_point)
time_find += time() - ts
if tree[dest_vertex_id] != -1 and (
cost[dest_vertex_id] <= MAX_DISTANCE_M
or MAX_DISTANCE_M <= 0 # TODO: enable skipping max distance
):
route_distance = cost[dest_vertex_id]
# route_points = [graph.vertex(dest_vertex_id).point()]
cur_vertex_id = dest_vertex_id
route_fids = []
# Iterate the graph from dest to origin saving the edges
ts = time()
while cur_vertex_id != origin_vertex_id:
cur_edge = graph.edge(tree[cur_vertex_id])
# Here we recover the edge id through strategy #1
route_fids.append(cur_edge.cost(1))
cur_vertex_id = cur_edge.fromVertex()
# route_points.append(graph.vertex(cur_vertex_id).point())
time_route += time() - ts
# route_points.reverse()
# route_geom = QgsGeometry.fromPolylineXY(route_points))
# Hack to remove duplicate fids
route_fids = list(
dict.fromkeys(route_fids)
) # NOTE: requires python >= 3.7 for ordered dict FIXME: add python version check
route_fids.reverse()
# Calc
# TODO: Move to matrix and vectorize calculation using numpy
gravity_value = poi_gravity_values[category]
bike_params = mode_params_bike[category]
ebike_params = mode_params_ebike[category]
# NOTE: we include dest size in decay here
decay = dest_sizes[j] * math.exp(
gravity_value * route_distance / 1000.0
)
p_bike = sigmoid(*bike_params, route_distance)
p_ebike = sigmoid(*ebike_params, route_distance)
# TODO: use namedtuple or dataclass
routes.append(
Route(
i,
j,
category,
route_distance,
decay,
p_bike,
p_ebike,
route_fids,
)
)
feedback.setProgress(i * step)
print(f'dijkstra took: {time_dijkstra:#1.2f} sec')
print(f'find vertex took: {time_find:#1.2f} sec')
print(f'route took: {time_route:#1.2f} sec')
with timing('post process routes'):
alpha_bike = 0.8
alpha_ebke = 0.2
decay_sums = {cat: defaultdict(float) for cat in poi_categories}
bike_values = {cat: defaultdict(float) for cat in poi_categories}
ebike_values = {cat: defaultdict(float) for cat in poi_categories}
for route in routes:
# NOTE: dest size is included in decay
decay_sums[route.cat][route.i] += route.decay
for route in routes:
decay_sum = decay_sums[route.cat][route.i]
# TODO: add T_p and alpha_m
T_p = trip_generation[route.cat]
bike_value = (
T_p
* alpha_bike
* orig_sizes[route.i]
* route.p_bike
* route.decay
/ decay_sum
)
ebike_value = (
T_p
* alpha_ebke
* orig_sizes[route.i]
* route.p_ebike
* route.decay
/ decay_sum
)
for fid in route.net_fids:
bike_values[route.cat][fid] += float(bike_value)
ebike_values[route.cat][fid] += float(ebike_value)
# FIXME: Un-kludge this
with timing('create result features'):
fields = get_fields()
segments = []
for feature in network_layer.getFeatures():
fid = feature.id()
segment = QgsFeature(fields)
segment.setGeometry(QgsGeometry(feature.geometry()))
segment['network_fid'] = fid
flow = 0.0
for cat in poi_categories:
bike_field = f'{cat}_bike_value'
ebike_field = f'{cat}_ebike_value'
flow_bike = segment[bike_field] = bike_values[cat].get(fid)
flow_ebke = segment[ebike_field] = ebike_values[cat].get(fid)
if flow_bike is not None:
flow += flow_bike
if flow_ebke is not None:
flow += flow_ebke
segment['flow'] = flow
segment['lts'] = feature['lts']
segment['vgu'] = feature['vgu']
segment['R'] = flow * feature['ratio']
segments.append(segment)
if not return_layer:
if return_raw:
return segments, bike_values, ebike_values
return segments
with timing('create result layer'):
output_layer = QgsVectorLayer(
f'LineString?crs={crs.toWkt()}', 'segments', 'memory'
)
with edit(output_layer):
for field in fields:
output_layer.addAttribute(field)
output_layer.addFeatures(segments, flags=QgsFeatureSink.FastInsert)
return output_layer
def mergeDataSources2Vrt(self,
dataSources,
outFile,
union=False,
relative=False,
schema=False,
feedback=None):
'''Function to do the work of merging datasources in a single vrt format
@param data_sources: Array of path strings
@param outfile: the output vrt file to generate
@param relative: Write relative flag. DOES NOT relativise paths. They have to be already relative
@param schema: Schema flag
@return: vrt in string format
'''
if feedback is None:
feedback = QgsProcessingFeedback()
vrt = '<OGRVRTDataSource>'
if union:
vrt += '<OGRVRTUnionLayer name="UnionedLayer">'
total = 100.0 / len(dataSources) if dataSources else 1
for current, layer in enumerate(dataSources):
if feedback.isCanceled():
break
feedback.setProgress(int(current * total))
inFile = layer.source()
srcDS = ogr.Open(inFile, 0)
if srcDS is None:
raise QgsProcessingException(
self.tr('Invalid datasource: {}'.format(inFile)))
if schema:
inFile = '@dummy@'
for layer in srcDS:
layerDef = layer.GetLayerDefn()
layerName = layerDef.GetName()
vrt += '<OGRVRTLayer name="{}">'.format(self.XmlEsc(layerName))
vrt += '<SrcDataSource relativeToVRT="{}" shared="{}">{}</SrcDataSource>'.format(
1 if relative else 0, not schema, self.XmlEsc(inFile))
if schema:
vrt += '<SrcLayer>@dummy@</SrcLayer>'
else:
vrt += '<SrcLayer>{}</SrcLayer>'.format(
self.XmlEsc(layerName))
vrt += '<GeometryType>{}</GeometryType>'.format(
self.GeomType2Name(layerDef.GetGeomType()))
crs = layer.GetSpatialRef()
if crs is not None:
vrt += '<LayerSRS>{}</LayerSRS>'.format(
self.XmlEsc(crs.ExportToWkt()))
# Process all the fields.
for fieldIdx in range(layerDef.GetFieldCount()):
fieldDef = layerDef.GetFieldDefn(fieldIdx)
vrt += '<Field name="{}" type="{}"'.format(
self.XmlEsc(fieldDef.GetName()),
self.fieldType2Name(fieldDef.GetType()))
if not schema:
vrt += ' src="{}"'.format(
self.XmlEsc(fieldDef.GetName()))
if fieldDef.GetWidth() > 0:
vrt += ' width="{}"'.format(fieldDef.GetWidth())
if fieldDef.GetPrecision() > 0:
vrt += ' precision="{}"'.format(
fieldDef.GetPrecision())
vrt += '/>'
vrt += '</OGRVRTLayer>'
srcDS.Destroy()
if union:
vrt += '</OGRVRTUnionLayer>'
vrt += '</OGRVRTDataSource>'
#TODO: pretty-print XML
if outFile is not None:
with codecs.open(outFile, 'w') as f:
f.write(vrt)
return vrt
def execute_in_place_run(alg, active_layer, parameters, context=None, feedback=None, raise_exceptions=False):
"""Executes an algorithm modifying features in-place in the input layer.
The input layer must be editable or an exception is raised.
:param alg: algorithm to run
:type alg: QgsProcessingAlgorithm
:param active_layer: the editable layer
:type active_layer: QgsVectoLayer
:param parameters: parameters of the algorithm
:type parameters: dict
:param context: context, defaults to None
:param context: QgsProcessingContext, optional
:param feedback: feedback, defaults to None
:param feedback: QgsProcessingFeedback, optional
:raises QgsProcessingException: raised when the layer is not editable or the layer cannot be found in the current project
:return: a tuple with true if success and results
:rtype: tuple
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
if active_layer is None or not active_layer.isEditable():
raise QgsProcessingException(tr("Layer is not editable or layer is None."))
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."))
parameters['OUTPUT'] = 'memory:'
try:
new_feature_ids = []
active_layer.beginEditCommand(alg.name())
req = QgsFeatureRequest(QgsExpression(r"$id < 0"))
req.setFlags(QgsFeatureRequest.NoGeometry)
req.setSubsetOfAttributes([])
# Checks whether the algorithm has a processFeature method
if hasattr(alg, 'processFeature'): # in-place feature editing
# Make a clone or it will crash the second time the dialog
# is opened and run
alg = alg.create()
if not alg.prepare(parameters, context, feedback):
raise QgsProcessingException(tr("Could not prepare selected algorithm."))
# Check again for compatibility after prepare
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."))
field_idxs = range(len(active_layer.fields()))
feature_iterator = active_layer.getFeatures(QgsFeatureRequest(active_layer.selectedFeatureIds())) if parameters['INPUT'].selectedFeaturesOnly else active_layer.getFeatures()
for f in feature_iterator:
new_features = alg.processFeature(f, context, feedback)
new_features = make_features_compatible(new_features, active_layer)
if len(new_features) == 0:
active_layer.deleteFeature(f.id())
elif len(new_features) == 1:
new_f = new_features[0]
if not f.geometry().equals(new_f.geometry()):
active_layer.changeGeometry(f.id(), new_f.geometry())
if f.attributes() != new_f.attributes():
active_layer.changeAttributeValues(f.id(), dict(zip(field_idxs, new_f.attributes())), dict(zip(field_idxs, f.attributes())))
new_feature_ids.append(f.id())
else:
active_layer.deleteFeature(f.id())
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(tr("Error adding processed features back into the layer."))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
results, ok = {}, True
else: # Traditional 'run' with delete and add features cycle
results, ok = alg.run(parameters, context, feedback)
if ok:
result_layer = QgsProcessingUtils.mapLayerFromString(results['OUTPUT'], context)
# TODO: check if features have changed before delete/add cycle
active_layer.deleteFeatures(active_layer.selectedFeatureIds())
new_features = []
for f in result_layer.getFeatures():
new_features.extend(make_features_compatible([f], active_layer))
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(tr("Error adding processed features back into the layer."))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
active_layer.endEditCommand()
if ok and new_feature_ids:
active_layer.selectByIds(new_feature_ids)
elif not ok:
active_layer.rollBack()
return ok, results
except QgsProcessingException as e:
active_layer.endEditCommand()
active_layer.rollBack()
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing', Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)))
return False, {}
def runAlgorithm(algOrName,
parameters,
onFinish=None,
feedback=None,
context=None):
if isinstance(algOrName, QgsProcessingAlgorithm):
alg = algOrName
else:
alg = QgsApplication.processingRegistry().createAlgorithmById(
algOrName)
if feedback is None:
feedback = QgsProcessingFeedback()
if alg is None:
msg = Processing.tr('Error: Algorithm {0} not found\n').format(
algOrName)
feedback.reportError(msg)
raise QgsProcessingException(msg)
if context is None:
context = dataobjects.createContext(feedback)
if context.feedback() is None:
context.setFeedback(feedback)
ok, msg = alg.checkParameterValues(parameters, context)
if not ok:
msg = Processing.tr('Unable to execute algorithm\n{0}').format(msg)
feedback.reportError(msg)
raise QgsProcessingException(msg)
if not alg.validateInputCrs(parameters, context):
feedback.pushInfo(
Processing.
tr('Warning: Not all input layers use the same CRS.\nThis can cause unexpected results.'
))
ret, results = execute(alg, parameters, context, feedback)
if ret:
feedback.pushInfo(Processing.tr('Results: {}').format(results))
if onFinish is not None:
onFinish(alg, context, feedback)
else:
# auto convert layer references in results to map layers
for out in alg.outputDefinitions():
if out.name() not in results:
continue
if isinstance(out, (QgsProcessingOutputVectorLayer,
QgsProcessingOutputRasterLayer,
QgsProcessingOutputMapLayer)):
result = results[out.name()]
if not isinstance(result, QgsMapLayer):
layer = context.takeResultLayer(
result
) # transfer layer ownership out of context
if layer:
results[out.name(
)] = layer # replace layer string ref with actual layer (+ownership)
elif isinstance(out, QgsProcessingOutputMultipleLayers):
result = results[out.name()]
if result:
layers_result = []
for l in result:
if not isinstance(result, QgsMapLayer):
layer = context.takeResultLayer(
l
) # transfer layer ownership out of context
if layer:
layers_result.append(layer)
else:
layers_result.append(l)
else:
layers_result.append(l)
results[out.name(
)] = layers_result # replace layers strings ref with actual layers (+ownership)
else:
msg = Processing.tr("There were errors executing the algorithm.")
feedback.reportError(msg)
raise QgsProcessingException(msg)
if isinstance(feedback, MessageBarProgress):
feedback.close()
return results
def create_feature_from_observation(
self,
observation: Observation,
geometriobjekter: Dict[str, GeometriObjekt],
idents: Dict[str, str],
feedback: QgsProcessingFeedback,
):
observation_id = observation.objektid
fikspunkt1_id = observation.opstillingspunktid
fikspunkt1_ident = "uuid:" + fikspunkt1_id
if fikspunkt1_id in idents:
fikspunkt1_ident = idents[fikspunkt1_id]
fikspunkt2_id = observation.sigtepunktid
fikspunkt2_ident = "uuid:" + fikspunkt2_id
if fikspunkt2_id in idents:
fikspunkt2_ident = idents[fikspunkt2_id]
geometriobjekt1 = geometriobjekter[fikspunkt1_id]
geometriobjekt2 = geometriobjekter[fikspunkt2_id]
line_geometry = self.create_line_geometry_from_geometriobjekter(
geometriobjekt1, geometriobjekt2, feedback)
if line_geometry:
# create the feature
fet = QgsFeature()
fet.setGeometry(line_geometry)
# Felter, der skal gemmes på feature:
# [QgsField("observation_id", QVariant.String),
# QgsField("observation_type_id", QVariant.Double)
# QgsField("fikspunkt1_id", QVariant.String),
# QgsField("fikspunkt1_ident", QVariant.String),
# QgsField("fikspunkt2_id", QVariant.String),
# QgsField("fikspunkt2_ident", QVariant.String),
# QgsField("registrering_fra", QVariant.DateTime),
# QgsField("registrering_fra_iso", QVariant.String),
# QgsField("koteforskel", QVariant.Double),
# QgsField("nivellementslaengde", QVariant.Double),
# QgsField("antal_opstillinger", QVariant.Double), Value3
# QgsField("afstandsafhaengig_varians", QVariant.Double), (value5 for id=1, value4 for id=2)
# QgsField("afstandsuafhaengig_varians", QVariant.Double), (value6 for id=1, value5 for id=2)
# QgsField("Praecisionsnivellement", QVariant.Double)], (value7 for id=1, 0 for id=2)
observation_type_id = observation.observationstypeid
registrering_fra = QDateTime(observation.registreringfra)
registrering_fra_iso = registrering_fra.toString(Qt.ISODate)
koteforskel = observation.value1
nivellementslaengde = observation.value2
antal_opstillinger = observation.value3
if observation_type_id == 1:
afstandsafhaengig_varians = observation.value5
afstandsuafhaengig_varians = observation.value6
Praecisionsnivellement = observation.value7
elif observation_type_id == 2:
afstandsafhaengig_varians = observation.value4
afstandsuafhaengig_varians = observation.value5
Praecisionsnivellement = 0
else:
# Observationstypeid > 2
feedback.setProgressText(
"observation_type_id > 2 for observation med id = {id}. Springes over"
.format(id=observation_id))
return None
# create the feature
feature = QgsFeature()
feature.setGeometry(line_geometry)
feature.setAttributes([
observation_id,
observation_type_id,
fikspunkt1_id,
fikspunkt1_ident,
fikspunkt2_id,
fikspunkt2_ident,
registrering_fra,
registrering_fra_iso,
koteforskel,
nivellementslaengde,
antal_opstillinger,
afstandsafhaengig_varians,
afstandsuafhaengig_varians,
Praecisionsnivellement,
])
return feature
else:
# A geometry could not be established
feedback.setProgressText(
"En liniegeometri kunne IKKE opettes for observation med id = {id}"
.format(id=observation_id))
return None
def __init__(self, dialog):
QgsProcessingFeedback.__init__(self)
self.dialog = dialog
def execute_in_place_run(alg, active_layer, parameters, context=None, feedback=None, raise_exceptions=False):
"""Executes an algorithm modifying features in-place in the input layer.
The input layer must be editable or an exception is raised.
:param alg: algorithm to run
:type alg: QgsProcessingAlgorithm
:param active_layer: the editable layer
:type active_layer: QgsVectoLayer
:param parameters: parameters of the algorithm
:type parameters: dict
:param context: context, defaults to None
:param context: QgsProcessingContext, optional
:param feedback: feedback, defaults to None
:param feedback: QgsProcessingFeedback, optional
:raises QgsProcessingException: raised when the layer is not editable or the layer cannot be found in the current project
:return: a tuple with true if success and results
:rtype: tuple
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
if active_layer is None or not active_layer.isEditable():
raise QgsProcessingException(tr("Layer is not editable or layer is None."))
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."))
parameters['OUTPUT'] = 'memory:'
try:
new_feature_ids = []
active_layer.beginEditCommand(alg.displayName())
req = QgsFeatureRequest(QgsExpression(r"$id < 0"))
req.setFlags(QgsFeatureRequest.NoGeometry)
req.setSubsetOfAttributes([])
# Checks whether the algorithm has a processFeature method
if hasattr(alg, 'processFeature'): # in-place feature editing
# Make a clone or it will crash the second time the dialog
# is opened and run
alg = alg.create()
if not alg.prepare(parameters, context, feedback):
raise QgsProcessingException(tr("Could not prepare selected algorithm."))
# Check again for compatibility after prepare
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."))
field_idxs = range(len(active_layer.fields()))
feature_iterator = active_layer.getFeatures(QgsFeatureRequest(active_layer.selectedFeatureIds())) if parameters['INPUT'].selectedFeaturesOnly else active_layer.getFeatures()
step = 100 / len(active_layer.selectedFeatureIds()) if active_layer.selectedFeatureIds() else 1
for current, f in enumerate(feature_iterator):
feedback.setProgress(current * step)
if feedback.isCanceled():
break
# need a deep copy, because python processFeature implementations may return
# a shallow copy from processFeature
input_feature = QgsFeature(f)
new_features = alg.processFeature(input_feature, context, feedback)
new_features = QgsVectorLayerUtils.makeFeaturesCompatible(new_features, active_layer)
if len(new_features) == 0:
active_layer.deleteFeature(f.id())
elif len(new_features) == 1:
new_f = new_features[0]
if not f.geometry().equals(new_f.geometry()):
active_layer.changeGeometry(f.id(), new_f.geometry())
if f.attributes() != new_f.attributes():
active_layer.changeAttributeValues(f.id(), dict(zip(field_idxs, new_f.attributes())), dict(zip(field_idxs, f.attributes())))
new_feature_ids.append(f.id())
else:
active_layer.deleteFeature(f.id())
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(tr("Error adding processed features back into the layer."))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
results, ok = {}, True
else: # Traditional 'run' with delete and add features cycle
results, ok = alg.run(parameters, context, feedback)
if ok:
result_layer = QgsProcessingUtils.mapLayerFromString(results['OUTPUT'], context)
# TODO: check if features have changed before delete/add cycle
active_layer.deleteFeatures(active_layer.selectedFeatureIds())
new_features = []
for f in result_layer.getFeatures():
new_features.extend(QgsVectorLayerUtils.
makeFeaturesCompatible([f], active_layer))
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(tr("Error adding processed features back into the layer."))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
active_layer.endEditCommand()
if ok and new_feature_ids:
active_layer.selectByIds(new_feature_ids)
elif not ok:
active_layer.rollBack()
return ok, results
except QgsProcessingException as e:
active_layer.endEditCommand()
active_layer.rollBack()
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing', Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)))
return False, {}
def testOneSidedBuffer(self):
context = QgsProcessingContext()
feedback = QgsProcessingFeedback()
source = os.path.join(testDataPath, 'polys.gml')
source_with_space = os.path.join(testDataPath,
'filename with spaces.gml')
alg = OneSideBuffer()
alg.initAlgorithm()
with tempfile.TemporaryDirectory() as outdir:
self.assertEqual(
alg.getConsoleCommands(
{
'INPUT': source,
'DISTANCE': 5,
'OUTPUT': outdir + '/check.shp'
}, context, feedback),
[
'ogr2ogr', outdir + '/check.shp ' + source + ' ' +
'-dialect sqlite -sql "SELECT ST_SingleSidedBuffer(geometry, 5.0, 0) AS geometry,* FROM \\"polys2\\"" '
+ '-f "ESRI Shapefile"'
])
self.assertEqual(
alg.getConsoleCommands(
{
'INPUT': source,
'DISTANCE': 5,
'DISSOLVE': True,
'OUTPUT': outdir + '/check.shp'
}, context, feedback),
[
'ogr2ogr', outdir + '/check.shp ' + source + ' ' +
'-dialect sqlite -sql "SELECT ST_Union(ST_SingleSidedBuffer(geometry, 5.0, 0)) AS geometry,* FROM \\"polys2\\"" '
+ '-f "ESRI Shapefile"'
])
self.assertEqual(
alg.getConsoleCommands(
{
'INPUT': source,
'DISTANCE': 5,
'EXPLODE_COLLECTIONS': True,
'OUTPUT': outdir + '/check.shp'
}, context, feedback),
[
'ogr2ogr', outdir + '/check.shp ' + source + ' ' +
'-dialect sqlite -sql "SELECT ST_SingleSidedBuffer(geometry, 5.0, 0) AS geometry,* FROM \\"polys2\\"" '
+ '-explodecollections -f "ESRI Shapefile"'
])
self.assertEqual(
alg.getConsoleCommands(
{
'INPUT': source,
'DISTANCE': 5,
'FIELD': 'total population',
'OUTPUT': outdir + '/check.shp'
}, context, feedback),
[
'ogr2ogr', outdir + '/check.shp ' + source + ' ' +
'-dialect sqlite -sql "SELECT ST_Union(ST_SingleSidedBuffer(geometry, 5.0, 0)) AS geometry,* '
+
'FROM \\"polys2\\" GROUP BY \\"total population\\"" -f "ESRI Shapefile"'
])
def mergeDataSources2Vrt(self, dataSources, outFile, union=False, relative=False,
schema=False, feedback=None):
'''Function to do the work of merging datasources in a single vrt format
@param data_sources: Array of path strings
@param outfile: the output vrt file to generate
@param relative: Write relative flag. DOES NOT relativise paths. They have to be already relative
@param schema: Schema flag
@return: vrt in string format
'''
if feedback is None:
feedback = QgsProcessingFeedback()
vrt = '<OGRVRTDataSource>'
if union:
vrt += '<OGRVRTUnionLayer name="UnionedLayer">'
total = 100.0 / len(dataSources) if dataSources else 1
for current, layer in enumerate(dataSources):
if feedback.isCanceled():
break
feedback.setProgress(int(current * total))
inFile = layer.source()
srcDS = ogr.Open(inFile, 0)
if srcDS is None:
raise QgsProcessingException(
self.tr('Invalid datasource: {}'.format(inFile)))
if schema:
inFile = '@dummy@'
for layer in srcDS:
layerDef = layer.GetLayerDefn()
layerName = layerDef.GetName()
vrt += '<OGRVRTLayer name="{}">'.format(self.XmlEsc(layerName))
vrt += '<SrcDataSource relativeToVRT="{}" shared="{}">{}</SrcDataSource>'.format(1 if relative else 0, not schema, self.XmlEsc(inFile))
if schema:
vrt += '<SrcLayer>@dummy@</SrcLayer>'
else:
vrt += '<SrcLayer>{}</SrcLayer>'.format(self.XmlEsc(layerName))
vrt += '<GeometryType>{}</GeometryType>'.format(self.GeomType2Name(layerDef.GetGeomType()))
crs = layer.GetSpatialRef()
if crs is not None:
vrt += '<LayerSRS>{}</LayerSRS>'.format(self.XmlEsc(crs.ExportToWkt()))
# Process all the fields.
for fieldIdx in range(layerDef.GetFieldCount()):
fieldDef = layerDef.GetFieldDefn(fieldIdx)
vrt += '<Field name="{}" type="{}"'.format(self.XmlEsc(fieldDef.GetName()), self.fieldType2Name(fieldDef.GetType()))
if not schema:
vrt += ' src="{}"'.format(self.XmlEsc(fieldDef.GetName()))
if fieldDef.GetWidth() > 0:
vrt += ' width="{}"'.format(fieldDef.GetWidth())
if fieldDef.GetPrecision() > 0:
vrt += ' precision="{}"'.format(fieldDef.GetPrecision())
vrt += '/>'
vrt += '</OGRVRTLayer>'
srcDS.Destroy()
if union:
vrt += '</OGRVRTUnionLayer>'
vrt += '</OGRVRTDataSource>'
#TODO: pretty-print XML
if outFile is not None:
with codecs.open(outFile, 'w') as f:
f.write(vrt)
return vrt
def calculate_zonal_stats(callback,
zonal_layer,
points_layer,
join_fields,
output_layer_name,
discard_nonmatching=False,
predicates=('intersects', ),
summaries=('sum', )):
"""
Leveraging the QGIS processing algorithm 'Join attributes by location
(summary)', that is described in QGIS as follows:
This algorithm takes an input vector layer and creates a new vector layer
that is an extended version of the input one, with additional attributes in
its attribute table. The additional attributes and their values are taken
from a second vector layer. A spatial criteria is applied to select the
values from the second layer that are added to each feature from the first
layer in the resulting one. The algorithm calculates a statistical summary
for the values from matching features in the second layer (e.g. maximum
value, mean value, etc).
The full description of the algorithm can be obtained as follows:
processing.algorithmHelp('qgis:joinbylocationsummary') and it includes
the lists of predicates and summaries.
The code of the algorithm is here:
https://github.com/qgis/QGIS/blob
/483b4ff977e3d36b166fac792254c31e89e3aeae/python/plugins/processing/algs
/qgis/SpatialJoinSummary.py # NOQA
:param callback: function to be called once the aggregation is complete,
passing the output zonal layer as a parameter
:param zonal_layer: vector layer containing polygons (or its path)
:param points_layer: vector layer containing points (or its path)
:param join_fields: fields for which we want to calculate statistics
(e.g. structural)
:param output_layer_name: a memory layer will be produced, named
'memory:output_layer_name'
:param discard_nonmatching: discard records which could not be joined
(in our case, purge zones that contain no loss/damage points)
:param predicates: geometric predicates (default: 'intersects')
:param summaries: statistics to be calculated for each join field
(default: 'sum')
:returns: it waits until the task is complete or terminated, then it
calls the callback function, passing the output QgsVectorLayer as
parameter, or None in case of failure
"""
processing.Processing.initialize()
alg = QgsApplication.processingRegistry().algorithmById(
'qgis:joinbylocationsummary')
# make sure to use the actual lists of predicates and summaries as defined
# in the algorithm when it is instantiated
predicate_keys = [predicate[0] for predicate in alg.predicates]
PREDICATES = dict(zip(predicate_keys, range(len(predicate_keys))))
summary_keys = [statistic[0] for statistic in alg.statistics]
SUMMARIES = dict(zip(summary_keys, range(len(summary_keys))))
context = QgsProcessingContext()
feedback = QgsProcessingFeedback()
params = {
'DISCARD_NONMATCHING': discard_nonmatching,
'INPUT': zonal_layer,
'JOIN': points_layer,
'PREDICATE': [PREDICATES[predicate] for predicate in predicates],
'JOIN_FIELDS': join_fields,
'SUMMARIES': [SUMMARIES[summary] for summary in summaries],
'OUTPUT': 'memory:%s' % output_layer_name
}
task = QgsProcessingAlgRunnerTask(alg, params, context, feedback)
task.executed.connect(partial(task_finished, context, callback))
QgsApplication.taskManager().addTask(task)
while True:
# the user can "cancel" the task, interrupting this loop
QgsApplication.processEvents()
# status can be queued, onhold, running, complete, terminated
if task.status() > 2: # Complete or terminated
return
time.sleep(0.1)
def test_context(outputdir, data):
""" Context with Copy layer
"""
alg = _find_algorithm('pyqgiswps_test:testcopylayer')
inputs = {
p.name(): [parse_input_definition(p)]
for p in alg.parameterDefinitions()
}
outputs = {
p.name(): parse_output_definition(p)
for p in alg.outputDefinitions()
}
inputs['INPUT'][0].data = 'france_parts'
inputs['OUTPUT'][0].data = 'france_parts_2'
context = ProcessingContext(str(outputdir), 'france_parts.qgs')
feedback = QgsProcessingFeedback()
parameters = dict(
input_to_processing(ident, inp, alg, context)
for ident, inp in inputs.items())
assert isinstance(parameters['OUTPUT'], QgsProcessingOutputLayerDefinition)
destination = get_valid_filename(alg.id())
context.wms_url = "http://localhost/wms/MAP=test/{name}.qgs".format(
name=destination)
# Run algorithm
with chdir(outputdir):
results = run_algorithm(alg,
parameters=parameters,
feedback=feedback,
context=context,
outputs=outputs)
assert context.destination_project.count() == 1
# Save destination project
ok = context.write_result(context.workdir, destination)
assert ok
assert context.destination_project.fileName() == str(
outputdir / (destination + '.qgs'))
# WFS configuration inserted
WFSLayers = context.destination_project.readListEntry('WFSLayers', '/')[0]
assert len(WFSLayers) != 0
# All Vector Layers has been published in WFS
mapLayers = context.destination_project.mapLayers()
assert len(WFSLayers) == len([
lid for lid, lyr in mapLayers.items()
if lyr.type() == QgsMapLayer.VectorLayer
])
# Verifying th WFS configuration
for lid in WFSLayers:
lyr = context.destination_project.mapLayer(lid)
# Is the WFS layer id references a Map Layer
assert lyr
# Is the WFS layer id references a Vector Layer
assert lyr.type() == QgsMapLayer.VectorLayer
# The WFS layer precision is defined
assert context.destination_project.readNumEntry(
"WFSLayersPrecision", "/" + lid)[0] == 6
def handleAlgorithmResults(alg, context, feedback=None, showResults=True, parameters={}):
wrongLayers = []
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.setProgressText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers'))
i = 0
for l, details in context.layersToLoadOnCompletion().items():
if feedback.isCanceled():
return False
if len(context.layersToLoadOnCompletion()) > 2:
# only show progress feedback if we're loading a bunch of layers
feedback.setProgress(100 * i / float(len(context.layersToLoadOnCompletion())))
try:
layer = QgsProcessingUtils.mapLayerFromString(l, context, typeHint=details.layerTypeHint)
if layer is not None:
set_layer_name(layer, details)
'''If running a model, the execution will arrive here when an algorithm that is part of
that model is executed. We check if its output is a final otuput of the model, and
adapt the output name accordingly'''
outputName = details.outputName
expcontext = QgsExpressionContext()
scope = QgsExpressionContextScope()
expcontext.appendScope(scope)
for out in alg.outputDefinitions():
if out.name() not in parameters:
continue
outValue = parameters[out.name()]
if hasattr(outValue, "sink"):
outValue = outValue.sink.valueAsString(expcontext)[0]
else:
outValue = str(outValue)
if outValue == l:
outputName = out.name()
break
style = None
if outputName:
style = RenderingStyles.getStyle(alg.id(), outputName)
if style is None:
if layer.type() == QgsMapLayer.RasterLayer:
style = ProcessingConfig.getSetting(ProcessingConfig.RASTER_STYLE)
else:
if layer.geometryType() == QgsWkbTypes.PointGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POINT_STYLE)
elif layer.geometryType() == QgsWkbTypes.LineGeometry:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_LINE_STYLE)
else:
style = ProcessingConfig.getSetting(ProcessingConfig.VECTOR_POLYGON_STYLE)
if style:
layer.loadNamedStyle(style)
details.project.addMapLayer(context.temporaryLayerStore().takeMapLayer(layer))
if details.postProcessor():
details.postProcessor().postProcessLayer(layer, context, feedback)
else:
wrongLayers.append(str(l))
except Exception:
QgsMessageLog.logMessage(QCoreApplication.translate('Postprocessing', "Error loading result layer:") + "\n" + traceback.format_exc(), 'Processing', Qgis.Critical)
wrongLayers.append(str(l))
i += 1
feedback.setProgress(100)
if wrongLayers:
msg = QCoreApplication.translate('Postprocessing', "The following layers were not correctly generated.")
msg += "<ul>" + "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>"
msg += QCoreApplication.translate('Postprocessing', "You can check the 'Log Messages Panel' in QGIS main window to find more information about the execution of the algorithm.")
feedback.reportError(msg)
return len(wrongLayers) == 0
def main():
"""reads in a pre-processed groundwater layer, with recharge-discharge (CSV) and returns tif
Args:
state(str): state for which well elevations must be obtained
season(str): e.g. rech-96, disc-96
Returns:
None: well elevations with locations stored in CSV as SHP
"""
state = sys.argv[1]
season = sys.argv[2]
dataPath = root.joinpath("data","groundwater")
metaPath = root.joinpath("outputs","groundwater","csv",state+"_metadata.log")
outputsPath = root.joinpath("outputs","groundwater")
logging.basicConfig(level=logging.INFO,
format='%(asctime)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p',
handlers=[logging.FileHandler(str(metaPath))],
)
logging.info("get Recharge-Discharge for '%s' dataset",state)
# Initialize QGIS Application
QgsApplication.setPrefixPath("G:\\Users\\Craig\\miniconda3\\envs\\geo_env\\Library\\python\\qgis", True)
qgs = QgsApplication([], False)
qgs.initQgis()
# Append the path where QGIS processing plugin can be found
sys.path.append('G:\\Users\\Craig\\miniconda3\\envs\\geo_env\\Library\\python\\plugins')
import processing
from processing.core.Processing import Processing
Processing.initialize()
feedback = QgsProcessingFeedback()
QgsApplication.processingRegistry().addProvider(QgsNativeAlgorithms())
# Get file with recharge-discharge values from previous step
vectorPath = outputsPath.joinpath("shapefiles",state+"_processed_wRD.shp")
print(vectorPath,vectorPath.exists())
vLayer = QgsVectorLayer(str(vectorPath), 'well_rech_disc_layer', 'ogr') #.setSubsetString(season + " IS NOT NULL")
print("islayer valid:", vLayer.isValid())
# subset layer for the chosen season and choose only non null values
filter = "\"" + season + "\"" + " IS NOT NULL"
expr = QgsExpression(filter)
subset = vLayer.getFeatures(QgsFeatureRequest(expr))
vLayer.selectByIds([k.id() for k in subset]) # why didn't direct selection work? addFeature (false)
print(vLayer.selectedFeatureCount())
# write subsetted layer to shapefile
subsetPath = outputsPath.joinpath("shapefiles","noNulls",state+"_"+season+"_noNulls.shp")
_writer = QgsVectorFileWriter.writeAsVectorFormat(vLayer, str(subsetPath), "utf-8", vLayer.crs(), "ESRI Shapefile", onlySelected=True)
# import subsetted layer
subLayer = QgsVectorLayer(str(subsetPath), 'well_rech_disc_layer_nonulls', 'ogr')
print("is sub layer valid:", subLayer.isValid())
# declare params for grid layer
# https://gdal.org/tutorials/gdal_grid_tut.html
params = {
'INPUT':subLayer,
'POWER':2, # FOR gridinversedistance
# 'RADIUS':0.25, # FOR gridinversedistancenearestneighbor / gridlinear
# 'RADIUS_1':0.25, # FOR gridinversedistance / gridnearestneighbor / gridaverage
# 'RADIUS_2':0.25, # FOR gridinversedistance / gridnearestneighbor / gridaverage
'MAX_POINTS':12, # FOR gridinversedistance
'MIN_POINTS':1, # FOR gridinversedistance / gridaverage
'NODATA': -9999,
'Z_FIELD':season,
'OUTPUT':str(outputsPath.joinpath("tif","idw",state+"_"+season+"_grid_id_min_1_max_12_nonulls.tif"))
}
res = processing.run("gdal:gridinversedistance",params,feedback=feedback)
print(res['OUTPUT'])
def processAlgorithm(self, parameters: Dict[str, Any],
context: QgsProcessingContext,
feedback: QgsProcessingFeedback) -> Union[dict, Dict[str, Any]]:
result = dict()
canalsfield: str = self.parameterAsFields(parameters, self.CANALSFIELD, context)[0]
pointsfield: str = self.parameterAsFields(parameters, self.POINTSFIELD, context)[0]
typefield: str = self.parameterAsFields(parameters, self.TYPEFIELD, context)[0]
typevalue: str = self.parameterAsString(parameters, self.TYPEVALUE, context)
tolerancepoints: float = self.parameterAsDouble(parameters, self.TOLERANCEPOINTS, context)
tolerancecanals: float = self.parameterAsDouble(parameters, self.TOLERANCECANALS, context)
model_feedback = QgsProcessingMultiStepFeedback(3, feedback)
if feedback.isCanceled():
return result
snappedcanals_out_name = 'native:splitwithlines_1:{}'.format(self.SNAPPEDCANALS)
intersect_out_name = 'native:fieldcalculator_1:INTERSECTIONS'
points_with_uuid_name = 'qgis:advancedpythonfieldcalculator_1:{}'.format(self.POINTSWITHUUID)
proc_result = processing.run(self.snap_lines.model(),
{
'CANALS': parameters[self.CANALS],
'DELIVERYPOINTS': parameters[self.DELIVERYPOINTS],
'TOLERANCECANALS': tolerancecanals,
'TYPEFIELD': typefield,
'TYPEVALUE': typevalue,
'VERBOSE_LOG': True,
snappedcanals_out_name: 'TEMPORARY_OUTPUT',
intersect_out_name: 'TEMPORARY_OUTPUT',
points_with_uuid_name: 'TEMPORARY_OUTPUT',
},
context=context,
feedback=model_feedback,
is_child_algorithm=True)
# TODO SNAPPEDCANALS добавить в result в конце, когда они будут порезаны по пересечениям
print("Main processing complete")
model_feedback.setCurrentStep(1)
if feedback.isCanceled():
return result
snapped_canals: QgsVectorLayer = context.takeResultLayer(
proc_result[snappedcanals_out_name])
intersections: QgsVectorLayer = context.takeResultLayer(
proc_result[intersect_out_name])
points_with_uuid: QgsVectorLayer = context.takeResultLayer(
proc_result[points_with_uuid_name])
(snapped_canals_sink, snapped_canals_id) = self.parameterAsSink(parameters, self.SNAPPEDCANALS,
context, snapped_canals.fields(),
snapped_canals.wkbType(),
snapped_canals.sourceCrs())
(points_with_uuid_sink, points_with_uuid_id) = self.parameterAsSink(parameters, self.POINTSWITHUUID,
context, points_with_uuid.fields(),
points_with_uuid.wkbType(),
points_with_uuid.sourceCrs())
snapped_canals_sink.addFeatures(snapped_canals.getFeatures())
points_with_uuid_sink.addFeatures(points_with_uuid.getFeatures())
result.update({
self.SNAPPEDCANALS: snapped_canals_id,
self.POINTSWITHUUID: points_with_uuid_id,
})
snapped_points_layers = []
for line_name in points_with_uuid.uniqueValues(points_with_uuid.fields().indexFromName(pointsfield)):
line_expression = QgsExpression().createFieldEqualityExpression(canalsfield, line_name)
line_request = QgsFeatureRequest()
line_request.setFilterExpression(line_expression)
point_expression = QgsExpression().createFieldEqualityExpression(pointsfield, line_name)
point_request = QgsFeatureRequest()
point_request.setFilterExpression(point_expression)
subset_snapped_canals = snapped_canals.materialize(line_request, model_feedback)
subset_points_with_uuid = points_with_uuid.materialize(point_request, model_feedback)
snapped_points_layers.append(
processing.run("native:snapgeometries",
{
'INPUT': subset_points_with_uuid,
'REFERENCE_LAYER': subset_snapped_canals,
'TOLERANCE': tolerancepoints,
'BEHAVIOR': 1,
'OUTPUT': 'TEMPORARY_OUTPUT'
},
context=context,
# feedback=model_feedback,
is_child_algorithm=True)['OUTPUT']
)
print("Cycle complete")
model_feedback.setCurrentStep(2)
if feedback.isCanceled():
return result
snapped_points_layers.append(intersections)
snapped_points_id = processing.run("native:mergevectorlayers",
{
'LAYERS': snapped_points_layers,
'CRS': points_with_uuid.crs(),
'OUTPUT': 'TEMPORARY_OUTPUT'
},
context=context,
feedback=model_feedback,
is_child_algorithm=True)['OUTPUT']
snapped_points: QgsVectorLayer = context.takeResultLayer(snapped_points_id)
(snapped_points_sink, snapped_points_id) = self.parameterAsSink(parameters, self.SNAPPEDPOINTS,
context, snapped_points.fields(),
snapped_points.wkbType(),
snapped_points.sourceCrs())
snapped_points_sink.addFeatures(snapped_points.getFeatures())
print("Vectors merged")
model_feedback.setCurrentStep(3)
if feedback.isCanceled():
return result
result.update({
self.SNAPPEDPOINTS: snapped_points_id,
})
print(result)
return result
def runGdal(commands, feedback=None):
if feedback is None:
feedback = QgsProcessingFeedback()
envval = os.getenv('PATH')
# We need to give some extra hints to get things picked up on OS X
isDarwin = False
try:
isDarwin = platform.system() == 'Darwin'
except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526
pass
if isDarwin and os.path.isfile(
os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")):
# Looks like there's a bundled gdal. Let's use it.
os.environ['PATH'] = "{}{}{}".format(
os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep,
envval)
os.environ['DYLD_LIBRARY_PATH'] = os.path.join(
QgsApplication.prefixPath(), "lib")
else:
# Other platforms should use default gdal finder codepath
settings = QgsSettings()
path = settings.value('/GdalTools/gdalPath', '')
if not path.lower() in envval.lower().split(os.pathsep):
envval += '{}{}'.format(os.pathsep, path)
os.putenv('PATH', envval)
fused_command = ' '.join([str(c) for c in commands])
QgsMessageLog.logMessage(fused_command, 'Processing', Qgis.Info)
feedback.pushInfo(GdalUtils.tr('GDAL command:'))
feedback.pushCommandInfo(fused_command)
feedback.pushInfo(GdalUtils.tr('GDAL command output:'))
loglines = [GdalUtils.tr('GDAL execution console output')]
def on_stdout(ba):
val = ba.data().decode('UTF-8')
# catch progress reports
if val == '100 - done.':
on_stdout.progress = 100
feedback.setProgress(on_stdout.progress)
elif val in ('0', '10', '20', '30', '40', '50', '60', '70', '80',
'90'):
on_stdout.progress = int(val)
feedback.setProgress(on_stdout.progress)
elif val == '.':
on_stdout.progress += 2.5
feedback.setProgress(on_stdout.progress)
on_stdout.buffer += val
if on_stdout.buffer.endswith('\n') or on_stdout.buffer.endswith(
'\r'):
# flush buffer
feedback.pushConsoleInfo(on_stdout.buffer.rstrip())
loglines.append(on_stdout.buffer.rstrip())
on_stdout.buffer = ''
on_stdout.progress = 0
on_stdout.buffer = ''
def on_stderr(ba):
val = ba.data().decode('UTF-8')
on_stderr.buffer += val
if on_stderr.buffer.endswith('\n') or on_stderr.buffer.endswith(
'\r'):
# flush buffer
feedback.reportError(on_stderr.buffer.rstrip())
loglines.append(on_stderr.buffer.rstrip())
on_stderr.buffer = ''
on_stderr.buffer = ''
command, *arguments = QgsRunProcess.splitCommand(fused_command)
proc = QgsBlockingProcess(command, arguments)
proc.setStdOutHandler(on_stdout)
proc.setStdErrHandler(on_stderr)
res = proc.run(feedback)
if feedback.isCanceled() and res != 0:
feedback.pushInfo(
GdalUtils.tr('Process was canceled and did not complete'))
elif not feedback.isCanceled() and proc.exitStatus(
) == QProcess.CrashExit:
raise QgsProcessingException(
GdalUtils.tr('Process was unexpectedly terminated'))
elif res == 0:
feedback.pushInfo(GdalUtils.tr('Process completed successfully'))
else:
feedback.reportError(
GdalUtils.tr('Process returned error code {}').format(res))
return loglines
def runAlgorithm(algOrName, parameters, onFinish=None, feedback=None, context=None):
if isinstance(algOrName, QgsProcessingAlgorithm):
alg = algOrName
else:
alg = QgsApplication.processingRegistry().createAlgorithmById(algOrName)
if feedback is None:
feedback = QgsProcessingFeedback()
if alg is None:
msg = Processing.tr('Error: Algorithm {0} not found\n').format(algOrName)
feedback.reportError(msg)
raise QgsProcessingException(msg)
if context is None:
context = dataobjects.createContext(feedback)
if context.feedback() is None:
context.setFeedback(feedback)
ok, msg = alg.checkParameterValues(parameters, context)
if not ok:
msg = Processing.tr('Unable to execute algorithm\n{0}').format(msg)
feedback.reportError(msg)
raise QgsProcessingException(msg)
if not alg.validateInputCrs(parameters, context):
feedback.pushInfo(
Processing.tr('Warning: Not all input layers use the same CRS.\nThis can cause unexpected results.'))
ret, results = execute(alg, parameters, context, feedback)
if ret:
feedback.pushInfo(
Processing.tr('Results: {}').format(results))
if onFinish is not None:
onFinish(alg, context, feedback)
else:
# auto convert layer references in results to map layers
for out in alg.outputDefinitions():
if out.name() not in results:
continue
if isinstance(out, (QgsProcessingOutputVectorLayer, QgsProcessingOutputRasterLayer, QgsProcessingOutputMapLayer)):
result = results[out.name()]
if not isinstance(result, QgsMapLayer):
layer = context.takeResultLayer(result) # transfer layer ownership out of context
if layer:
results[out.name()] = layer # replace layer string ref with actual layer (+ownership)
elif isinstance(out, QgsProcessingOutputMultipleLayers):
result = results[out.name()]
if result:
layers_result = []
for l in result:
if not isinstance(result, QgsMapLayer):
layer = context.takeResultLayer(l) # transfer layer ownership out of context
if layer:
layers_result.append(layer)
else:
layers_result.append(l)
else:
layers_result.append(l)
results[out.name()] = layers_result # replace layers strings ref with actual layers (+ownership)
else:
msg = Processing.tr("There were errors executing the algorithm.")
feedback.reportError(msg)
raise QgsProcessingException(msg)
if isinstance(feedback, MessageBarProgress):
feedback.close()
return results
def testGdalTranslate(self):
context = QgsProcessingContext()
feedback = QgsProcessingFeedback()
source = os.path.join(testDataPath, 'dem.tif')
translate_alg = translate()
translate_alg.initAlgorithm()
# with no NODATA value
self.assertEqual(
translate_alg.getConsoleCommands(
{
'INPUT': source,
'OUTPUT': 'd:/temp/check.jpg'
}, context, feedback),
[
'gdal_translate',
'-ot Float32 -of JPEG ' + source + ' ' + 'd:/temp/check.jpg'
])
# with NODATA value
self.assertEqual(
translate_alg.getConsoleCommands(
{
'INPUT': source,
'NODATA': 9999,
'OUTPUT': 'd:/temp/check.jpg'
}, context, feedback),
[
'gdal_translate', '-a_nodata 9999.0 ' +
'-ot Float32 -of JPEG ' + source + ' ' + 'd:/temp/check.jpg'
])
# with "0" NODATA value
self.assertEqual(
translate_alg.getConsoleCommands(
{
'INPUT': source,
'NODATA': 0,
'OUTPUT': 'd:/temp/check.jpg'
}, context, feedback),
[
'gdal_translate', '-a_nodata 0.0 ' + '-ot Float32 -of JPEG ' +
source + ' ' + 'd:/temp/check.jpg'
])
# with target srs
self.assertEqual(
translate_alg.getConsoleCommands(
{
'INPUT': source,
'TARGET_CRS': 'EPSG:3111',
'OUTPUT': 'd:/temp/check.jpg'
}, context, feedback),
[
'gdal_translate', '-a_srs EPSG:3111 ' +
'-ot Float32 -of JPEG ' + source + ' ' + 'd:/temp/check.jpg'
])
# with target srs
self.assertEqual(
translate_alg.getConsoleCommands(
{
'INPUT': source,
'TARGET_CRS': 'EPSG:3111',
'OUTPUT': 'd:/temp/check.jpg'
}, context, feedback),
[
'gdal_translate', '-a_srs EPSG:3111 ' +
'-ot Float32 -of JPEG ' + source + ' ' + 'd:/temp/check.jpg'
])
# with copy subdatasets
self.assertEqual(
translate_alg.getConsoleCommands(
{
'INPUT': source,
'COPY_SUBDATASETS': True,
'OUTPUT': 'd:/temp/check.tif'
}, context, feedback), [
'gdal_translate', '-sds ' + '-ot Float32 -of GTiff ' +
source + ' ' + 'd:/temp/check.tif'
])
def execute_in_place_run(alg, parameters, context=None, feedback=None, raise_exceptions=False):
"""Executes an algorithm modifying features in-place in the input layer.
:param alg: algorithm to run
:type alg: QgsProcessingAlgorithm
:param parameters: parameters of the algorithm
:type parameters: dict
:param context: context, defaults to None
:type context: QgsProcessingContext, optional
:param feedback: feedback, defaults to None
:type feedback: QgsProcessingFeedback, optional
:param raise_exceptions: useful for testing, if True exceptions are raised, normally exceptions will be forwarded to the feedback
:type raise_exceptions: boo, default to False
:raises QgsProcessingException: raised when there is no active layer, or it cannot be made editable
:return: a tuple with true if success and results
:rtype: tuple
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
# Only feature based algs have sourceFlags
try:
if alg.sourceFlags() & QgsProcessingFeatureSource.FlagSkipGeometryValidityChecks:
context.setInvalidGeometryCheck(QgsFeatureRequest.GeometryNoCheck)
except AttributeError:
pass
active_layer = parameters['INPUT']
# Run some checks and prepare the layer for in-place execution by:
# - getting the active layer and checking that it is a vector
# - making the layer editable if it was not already
# - selecting all features if none was selected
# - checking in-place support for the active layer/alg/parameters
# If one of the check fails and raise_exceptions is True an exception
# is raised, else the execution is aborted and the error reported in
# the feedback
try:
if active_layer is None:
raise QgsProcessingException(tr("There is not active layer."))
if not isinstance(active_layer, QgsVectorLayer):
raise QgsProcessingException(tr("Active layer is not a vector layer."))
if not active_layer.isEditable():
if not active_layer.startEditing():
raise QgsProcessingException(tr("Active layer is not editable (and editing could not be turned on)."))
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."))
except QgsProcessingException as e:
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing', Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)), fatalError=True)
return False, {}
if not active_layer.selectedFeatureIds():
active_layer.selectAll()
# Make sure we are working on selected features only
parameters['INPUT'] = QgsProcessingFeatureSourceDefinition(active_layer.id(), True)
parameters['OUTPUT'] = 'memory:'
req = QgsFeatureRequest(QgsExpression(r"$id < 0"))
req.setFlags(QgsFeatureRequest.NoGeometry)
req.setSubsetOfAttributes([])
# Start the execution
# If anything goes wrong and raise_exceptions is True an exception
# is raised, else the execution is aborted and the error reported in
# the feedback
try:
new_feature_ids = []
active_layer.beginEditCommand(alg.displayName())
# Checks whether the algorithm has a processFeature method
if hasattr(alg, 'processFeature'): # in-place feature editing
# Make a clone or it will crash the second time the dialog
# is opened and run
alg = alg.create()
if not alg.prepare(parameters, context, feedback):
raise QgsProcessingException(tr("Could not prepare selected algorithm."))
# Check again for compatibility after prepare
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."))
field_idxs = range(len(active_layer.fields()))
iterator_req = QgsFeatureRequest(active_layer.selectedFeatureIds())
iterator_req.setInvalidGeometryCheck(context.invalidGeometryCheck())
feature_iterator = active_layer.getFeatures(iterator_req)
step = 100 / len(active_layer.selectedFeatureIds()) if active_layer.selectedFeatureIds() else 1
for current, f in enumerate(feature_iterator):
feedback.setProgress(current * step)
if feedback.isCanceled():
break
# need a deep copy, because python processFeature implementations may return
# a shallow copy from processFeature
input_feature = QgsFeature(f)
new_features = alg.processFeature(input_feature, context, feedback)
new_features = QgsVectorLayerUtils.makeFeaturesCompatible(new_features, active_layer)
if len(new_features) == 0:
active_layer.deleteFeature(f.id())
elif len(new_features) == 1:
new_f = new_features[0]
if not f.geometry().equals(new_f.geometry()):
active_layer.changeGeometry(f.id(), new_f.geometry())
if f.attributes() != new_f.attributes():
active_layer.changeAttributeValues(f.id(), dict(zip(field_idxs, new_f.attributes())), dict(zip(field_idxs, f.attributes())))
new_feature_ids.append(f.id())
else:
active_layer.deleteFeature(f.id())
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(tr("Error adding processed features back into the layer."))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
results, ok = {}, True
else: # Traditional 'run' with delete and add features cycle
# There is no way to know if some features have been skipped
# due to invalid geometries
if context.invalidGeometryCheck() == QgsFeatureRequest.GeometrySkipInvalid:
selected_ids = active_layer.selectedFeatureIds()
else:
selected_ids = []
results, ok = alg.run(parameters, context, feedback)
if ok:
result_layer = QgsProcessingUtils.mapLayerFromString(results['OUTPUT'], context)
# TODO: check if features have changed before delete/add cycle
new_features = []
# Check if there are any skipped features
if context.invalidGeometryCheck() == QgsFeatureRequest.GeometrySkipInvalid:
missing_ids = list(set(selected_ids) - set(result_layer.allFeatureIds()))
if missing_ids:
for f in active_layer.getFeatures(QgsFeatureRequest(missing_ids)):
if not f.geometry().isGeosValid():
new_features.append(f)
active_layer.deleteFeatures(active_layer.selectedFeatureIds())
for f in result_layer.getFeatures():
new_features.extend(QgsVectorLayerUtils.
makeFeaturesCompatible([f], active_layer))
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(tr("Error adding processed features back into the layer."))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
active_layer.endEditCommand()
if ok and new_feature_ids:
active_layer.selectByIds(new_feature_ids)
elif not ok:
active_layer.rollBack()
return ok, results
except QgsProcessingException as e:
active_layer.endEditCommand()
active_layer.rollBack()
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing', Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)), fatalError=True)
return False, {}
def get_missing_boundary_points_in_boundaries(self, db,
boundary_point_layer,
boundary_layer):
res = dict()
feedback = QgsProcessingFeedback()
extracted_vertices = processing.run("native:extractvertices", {
'INPUT': boundary_layer,
'OUTPUT': 'memory:'
},
feedback=feedback)
extracted_vertices_layer = extracted_vertices['OUTPUT']
# From vertices layer, get points with no overlap
overlapping_points = self.geometry.get_overlapping_points(
extracted_vertices_layer)
extracted_vertices_ids = [
feature.id() for feature in extracted_vertices_layer.getFeatures()
]
# Unpack list of lists into single list
overlapping_point_ids = [
item for sublist in overlapping_points for item in sublist
]
# Unpack list of lists into single list selecting only the first point
# per list. That means, discard overlapping points, and only keep one
cleaned_point_ids = [sublist[0] for sublist in overlapping_points]
# All vertices (even duplicated, due to the alg we use), minus all
# overlapping ids, plus only one of the overlapping ids
# This gets a list of all vertex ids with no duplicates
no_duplicate_ids = list(
set(extracted_vertices_ids) -
set(overlapping_point_ids)) + cleaned_point_ids
if boundary_point_layer.featureCount() == 0:
# Return all extracted and cleaned vertices
for feature in extracted_vertices_layer.getFeatures(
no_duplicate_ids):
if feature[db.names.T_ID_F] in res:
res[feature[db.names.T_ID_F]].append(feature.geometry())
else:
res[feature[db.names.T_ID_F]] = [feature.geometry()]
return res
index = QgsSpatialIndex(
boundary_point_layer.getFeatures(
QgsFeatureRequest().setSubsetOfAttributes([])), feedback)
for feature in extracted_vertices_layer.getFeatures(no_duplicate_ids):
if feature.hasGeometry():
geom = feature.geometry()
diff_geom = QgsGeometry(geom)
# Use a custom bbox to include very near but not exactly equal points
point_vert = {
'x': diff_geom.asPoint().x(),
'y': diff_geom.asPoint().y()
}
bbox = QgsRectangle(
QgsPointXY(point_vert['x'] - 0.0001,
point_vert['y'] - 0.0001),
QgsPointXY(point_vert['x'] + 0.0001,
point_vert['y'] + 0.0001))
intersects = index.intersects(bbox)
if not intersects:
if feature[db.names.T_ID_F] in res:
res[feature[db.names.T_ID_F]].append(diff_geom)
else:
res[feature[db.names.T_ID_F]] = [diff_geom]
return res
def check_algorithm(self, name, defs):
"""
Will run an algorithm definition and check if it generates the expected result
:param name: The identifier name used in the test output heading
:param defs: A python dict containing a test algorithm definition
"""
self.vector_layer_params = {}
QgsProject.instance().removeAllMapLayers()
params = self.load_params(defs['params'])
print('Running alg: "{}"'.format(defs['algorithm']))
alg = QgsApplication.processingRegistry().createAlgorithmById(
defs['algorithm'])
parameters = {}
if isinstance(params, list):
for param in zip(alg.parameterDefinitions(), params):
parameters[param[0].name()] = param[1]
else:
for k, p in list(params.items()):
parameters[k] = p
for r, p in list(defs['results'].items()):
if not 'in_place_result' in p or not p['in_place_result']:
parameters[r] = self.load_result_param(p)
expectFailure = False
if 'expectedFailure' in defs:
exec(('\n'.join(defs['expectedFailure'][:-1])), globals(),
locals())
expectFailure = eval(defs['expectedFailure'][-1])
# ignore user setting for invalid geometry handling
context = QgsProcessingContext()
context.setProject(QgsProject.instance())
if 'skipInvalid' in defs and defs['skipInvalid']:
context.setInvalidGeometryCheck(
QgsFeatureRequest.GeometrySkipInvalid)
feedback = QgsProcessingFeedback()
print('Algorithm parameters are {}'.format(parameters))
# first check that algorithm accepts the parameters we pass...
ok, msg = alg.checkParameterValues(parameters, context)
self.assertTrue(
ok,
'Algorithm failed checkParameterValues with result {}'.format(msg))
if expectFailure:
try:
results, ok = alg.run(parameters, context, feedback)
self.check_results(results, context, parameters,
defs['results'])
if ok:
raise _UnexpectedSuccess
except Exception:
pass
else:
results, ok = alg.run(parameters, context, feedback)
self.assertTrue(
ok, 'params: {}, results: {}'.format(parameters, results))
self.check_results(results, context, parameters, defs['results'])
def execute_in_place_run(alg,
parameters,
context=None,
feedback=None,
raise_exceptions=False):
"""Executes an algorithm modifying features in-place in the input layer.
:param alg: algorithm to run
:type alg: QgsProcessingAlgorithm
:param parameters: parameters of the algorithm
:type parameters: dict
:param context: context, defaults to None
:type context: QgsProcessingContext, optional
:param feedback: feedback, defaults to None
:type feedback: QgsProcessingFeedback, optional
:param raise_exceptions: useful for testing, if True exceptions are raised, normally exceptions will be forwarded to the feedback
:type raise_exceptions: boo, default to False
:raises QgsProcessingException: raised when there is no active layer, or it cannot be made editable
:return: a tuple with true if success and results
:rtype: tuple
"""
if feedback is None:
feedback = QgsProcessingFeedback()
if context is None:
context = dataobjects.createContext(feedback)
# Only feature based algs have sourceFlags
try:
if alg.sourceFlags(
) & QgsProcessingFeatureSource.FlagSkipGeometryValidityChecks:
context.setInvalidGeometryCheck(QgsFeatureRequest.GeometryNoCheck)
except AttributeError:
pass
active_layer = parameters['INPUT']
# Run some checks and prepare the layer for in-place execution by:
# - getting the active layer and checking that it is a vector
# - making the layer editable if it was not already
# - selecting all features if none was selected
# - checking in-place support for the active layer/alg/parameters
# If one of the check fails and raise_exceptions is True an exception
# is raised, else the execution is aborted and the error reported in
# the feedback
try:
if active_layer is None:
raise QgsProcessingException(tr("There is not active layer."))
if not isinstance(active_layer, QgsVectorLayer):
raise QgsProcessingException(
tr("Active layer is not a vector layer."))
if not active_layer.isEditable():
if not active_layer.startEditing():
raise QgsProcessingException(
tr("Active layer is not editable (and editing could not be turned on)."
))
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(
tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."
))
except QgsProcessingException as e:
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing',
Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)), fatalError=True)
return False, {}
if not active_layer.selectedFeatureIds():
active_layer.selectAll()
# Make sure we are working on selected features only
parameters['INPUT'] = QgsProcessingFeatureSourceDefinition(
active_layer.id(), True)
parameters['OUTPUT'] = 'memory:'
req = QgsFeatureRequest(QgsExpression(r"$id < 0"))
req.setFlags(QgsFeatureRequest.NoGeometry)
req.setSubsetOfAttributes([])
# Start the execution
# If anything goes wrong and raise_exceptions is True an exception
# is raised, else the execution is aborted and the error reported in
# the feedback
try:
new_feature_ids = []
active_layer.beginEditCommand(alg.displayName())
# Checks whether the algorithm has a processFeature method
if hasattr(alg, 'processFeature'): # in-place feature editing
# Make a clone or it will crash the second time the dialog
# is opened and run
alg = alg.create()
if not alg.prepare(parameters, context, feedback):
raise QgsProcessingException(
tr("Could not prepare selected algorithm."))
# Check again for compatibility after prepare
if not alg.supportInPlaceEdit(active_layer):
raise QgsProcessingException(
tr("Selected algorithm and parameter configuration are not compatible with in-place modifications."
))
field_idxs = range(len(active_layer.fields()))
iterator_req = QgsFeatureRequest(active_layer.selectedFeatureIds())
iterator_req.setInvalidGeometryCheck(
context.invalidGeometryCheck())
feature_iterator = active_layer.getFeatures(iterator_req)
step = 100 / len(active_layer.selectedFeatureIds()
) if active_layer.selectedFeatureIds() else 1
for current, f in enumerate(feature_iterator):
feedback.setProgress(current * step)
if feedback.isCanceled():
break
# need a deep copy, because python processFeature implementations may return
# a shallow copy from processFeature
input_feature = QgsFeature(f)
new_features = alg.processFeature(input_feature, context,
feedback)
new_features = QgsVectorLayerUtils.makeFeaturesCompatible(
new_features, active_layer)
if len(new_features) == 0:
active_layer.deleteFeature(f.id())
elif len(new_features) == 1:
new_f = new_features[0]
if not f.geometry().equals(new_f.geometry()):
active_layer.changeGeometry(f.id(), new_f.geometry())
if f.attributes() != new_f.attributes():
active_layer.changeAttributeValues(
f.id(), dict(zip(field_idxs, new_f.attributes())),
dict(zip(field_idxs, f.attributes())))
new_feature_ids.append(f.id())
else:
active_layer.deleteFeature(f.id())
# Get the new ids
old_ids = set(
[f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(
tr("Error adding processed features back into the layer."
))
new_ids = set(
[f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
results, ok = {}, True
else: # Traditional 'run' with delete and add features cycle
# There is no way to know if some features have been skipped
# due to invalid geometries
if context.invalidGeometryCheck(
) == QgsFeatureRequest.GeometrySkipInvalid:
selected_ids = active_layer.selectedFeatureIds()
else:
selected_ids = []
results, ok = alg.run(parameters, context, feedback)
if ok:
result_layer = QgsProcessingUtils.mapLayerFromString(
results['OUTPUT'], context)
# TODO: check if features have changed before delete/add cycle
new_features = []
# Check if there are any skipped features
if context.invalidGeometryCheck(
) == QgsFeatureRequest.GeometrySkipInvalid:
missing_ids = list(
set(selected_ids) - set(result_layer.allFeatureIds()))
if missing_ids:
for f in active_layer.getFeatures(
QgsFeatureRequest(missing_ids)):
if not f.geometry().isGeosValid():
new_features.append(f)
active_layer.deleteFeatures(active_layer.selectedFeatureIds())
for f in result_layer.getFeatures():
new_features.extend(
QgsVectorLayerUtils.makeFeaturesCompatible(
[f], active_layer))
# Get the new ids
old_ids = set([f.id() for f in active_layer.getFeatures(req)])
if not active_layer.addFeatures(new_features):
raise QgsProcessingException(
tr("Error adding processed features back into the layer."
))
new_ids = set([f.id() for f in active_layer.getFeatures(req)])
new_feature_ids += list(new_ids - old_ids)
active_layer.endEditCommand()
if ok and new_feature_ids:
active_layer.selectByIds(new_feature_ids)
elif not ok:
active_layer.rollBack()
return ok, results
except QgsProcessingException as e:
active_layer.endEditCommand()
active_layer.rollBack()
if raise_exceptions:
raise e
QgsMessageLog.logMessage(str(sys.exc_info()[0]), 'Processing',
Qgis.Critical)
if feedback is not None:
feedback.reportError(getattr(e, 'msg', str(e)), fatalError=True)
return False, {}
def __init__(self, dialog):
QgsProcessingFeedback.__init__(self)
self.dialog = dialog
def testFeatureSourceInput(self):
# create a memory layer and add to project and context
layer = QgsVectorLayer(
"Point?crs=epsg:3857&field=fldtxt:string&field=fldint:integer",
"testmem", "memory")
self.assertTrue(layer.isValid())
pr = layer.dataProvider()
f = QgsFeature()
f.setAttributes(["test", 123])
f.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(100, 200)))
f2 = QgsFeature()
f2.setAttributes(["test2", 457])
f2.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(110, 200)))
self.assertTrue(pr.addFeatures([f, f2]))
self.assertEqual(layer.featureCount(), 2)
# select first feature
layer.selectByIds([next(layer.getFeatures()).id()])
self.assertEqual(len(layer.selectedFeatureIds()), 1)
QgsProject.instance().addMapLayer(layer)
context = QgsProcessingContext()
context.setProject(QgsProject.instance())
alg = QgsApplication.processingRegistry().createAlgorithmById(
'grass7:v.buffer')
self.assertIsNotNone(alg)
temp_file = os.path.join(self.temp_dir, 'grass_output_sel.shp')
parameters = {
'input': QgsProcessingFeatureSourceDefinition('testmem', True),
'cats': '',
'where': '',
'type': [0, 1, 4],
'distance': 1,
'minordistance': None,
'angle': 0,
'column': None,
'scale': 1,
'tolerance': 0.01,
'-s': False,
'-c': False,
'-t': False,
'output': temp_file,
'GRASS_REGION_PARAMETER': None,
'GRASS_SNAP_TOLERANCE_PARAMETER': -1,
'GRASS_MIN_AREA_PARAMETER': 0.0001,
'GRASS_OUTPUT_TYPE_PARAMETER': 0,
'GRASS_VECTOR_DSCO': '',
'GRASS_VECTOR_LCO': ''
}
feedback = QgsProcessingFeedback()
results, ok = alg.run(parameters, context, feedback)
self.assertTrue(ok)
self.assertTrue(os.path.exists(temp_file))
# make sure that layer has correct features
res = QgsVectorLayer(temp_file, 'res')
self.assertTrue(res.isValid())
self.assertEqual(res.featureCount(), 1)
QgsProject.instance().removeMapLayer(layer)
