def runGdal(commands, feedback=None): if feedback is None: feedback = QgsProcessingFeedback() envval = os.getenv('PATH') # We need to give some extra hints to get things picked up on OS X isDarwin = False try: isDarwin = platform.system() == 'Darwin' except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526 pass if isDarwin and os.path.isfile(os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")): # Looks like there's a bundled gdal. Let's use it. os.environ['PATH'] = "{}{}{}".format(os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep, envval) os.environ['DYLD_LIBRARY_PATH'] = os.path.join(QgsApplication.prefixPath(), "lib") else: # Other platforms should use default gdal finder codepath settings = QgsSettings() path = settings.value('/GdalTools/gdalPath', '') if not path.lower() in envval.lower().split(os.pathsep): envval += '{}{}'.format(os.pathsep, path) os.putenv('PATH', envval) fused_command = ' '.join([str(c) for c in commands]) QgsMessageLog.logMessage(fused_command, 'Processing', Qgis.Info) feedback.pushInfo('GDAL command:') feedback.pushCommandInfo(fused_command) feedback.pushInfo('GDAL command output:') success = False retry_count = 0 while not success: loglines = [] loglines.append('GDAL execution console output') try: with subprocess.Popen( fused_command, shell=True, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, stderr=subprocess.STDOUT, universal_newlines=True, ) as proc: for line in proc.stdout: feedback.pushConsoleInfo(line) loglines.append(line) success = True except IOError as e: if retry_count < 5: retry_count += 1 else: raise IOError( str(e) + u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}'.format( len(loglines), u'\n'.join(loglines[-10:]))) QgsMessageLog.logMessage('\n'.join(loglines), 'Processing', Qgis.Info) GdalUtils.consoleOutput = loglines
def execute(commands, feedback=None): cmds = [] cmds.append(os.path.join(mpichDirectory(), "mpiexec")) processes = ProcessingConfig.getSetting(TAUDEM_PROCESSES) if int(processes) <= 0: processes = 1 cmds.append("-n") cmds.append(processes) cmds.extend(commands) if feedback is None: feedback = QgsProcessingFeedback() fused_command = " ".join([str(c) for c in cmds]) QgsMessageLog.logMessage(fused_command, "Processing", QgsMessageLog.INFO) feedback.pushInfo("TauDEM command:") feedback.pushCommandInfo(fused_command) feedback.pushInfo("TauDEM command output:") loglines = [] with subprocess.Popen(fused_command, shell=True, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, stderr=subprocess.STDOUT, universal_newlines=True) as proc: try: for line in iter(proc.stdout.readline, ""): feedback.pushConsoleInfo(line) loglines.append(line) except: pass if ProcessingConfig.getSetting(TAUDEM_VERBOSE): QgsMessageLog.logMessage("\n".join(loglines), "Processing", QgsMessageLog.INFO)
def runAlgorithm(algOrName, parameters, onFinish=None, feedback=None, context=None): if isinstance(algOrName, QgsProcessingAlgorithm): alg = algOrName else: alg = QgsApplication.processingRegistry().createAlgorithmById(algOrName) if feedback is None: feedback = QgsProcessingFeedback() if alg is None: msg = Processing.tr('Error: Algorithm {0} not found\n').format(algOrName) feedback.reportError(msg) raise QgsProcessingException(msg) if context is None: context = dataobjects.createContext(feedback) if context.feedback() is None: context.setFeedback(feedback) ok, msg = alg.checkParameterValues(parameters, context) if not ok: msg = Processing.tr('Unable to execute algorithm\n{0}').format(msg) feedback.reportError(msg) raise QgsProcessingException(msg) if not alg.validateInputCrs(parameters, context): feedback.pushInfo( Processing.tr('Warning: Not all input layers use the same CRS.\nThis can cause unexpected results.')) ret, results = execute(alg, parameters, context, feedback) if ret: feedback.pushInfo( Processing.tr('Results: {}').format(results)) if onFinish is not None: onFinish(alg, context, feedback) else: # auto convert layer references in results to map layers for out in alg.outputDefinitions(): if out.name() not in results: continue if isinstance(out, (QgsProcessingOutputVectorLayer, QgsProcessingOutputRasterLayer, QgsProcessingOutputMapLayer)): result = results[out.name()] if not isinstance(result, QgsMapLayer): layer = context.takeResultLayer(result) # transfer layer ownership out of context if layer: results[out.name()] = layer # replace layer string ref with actual layer (+ownership) elif isinstance(out, QgsProcessingOutputMultipleLayers): result = results[out.name()] if result: layers_result = [] for l in result: if not isinstance(result, QgsMapLayer): layer = context.takeResultLayer(l) # transfer layer ownership out of context if layer: layers_result.append(layer) else: layers_result.append(l) else: layers_result.append(l) results[out.name()] = layers_result # replace layers strings ref with actual layers (+ownership) else: msg = Processing.tr("There were errors executing the algorithm.") feedback.reportError(msg) raise QgsProcessingException(msg) if isinstance(feedback, MessageBarProgress): feedback.close() return results
def runAlgorithm(algOrName, parameters, onFinish=None, feedback=None, context=None): if isinstance(algOrName, QgsProcessingAlgorithm): alg = algOrName else: alg = QgsApplication.processingRegistry().createAlgorithmById( algOrName) if feedback is None: feedback = QgsProcessingFeedback() if alg is None: msg = Processing.tr('Error: Algorithm {0} not found\n').format( algOrName) feedback.reportError(msg) raise QgsProcessingException(msg) # check for any mandatory parameters which were not specified for param in alg.parameterDefinitions(): if param.name() not in parameters: if not param.flags( ) & QgsProcessingParameterDefinition.FlagOptional: msg = Processing.tr( 'Error: Missing parameter value for parameter {0}.' ).format(param.name()) feedback.reportError(msg) raise QgsProcessingException(msg) if context is None: context = dataobjects.createContext(feedback) if context.feedback() is None: context.setFeedback(feedback) ok, msg = alg.checkParameterValues(parameters, context) if not ok: msg = Processing.tr('Unable to execute algorithm\n{0}').format(msg) feedback.reportError(msg) raise QgsProcessingException(msg) if not alg.validateInputCrs(parameters, context): feedback.pushInfo( Processing. tr('Warning: Not all input layers use the same CRS.\nThis can cause unexpected results.' )) ret, results = execute(alg, parameters, context, feedback) if ret: feedback.pushInfo(Processing.tr('Results: {}').format(results)) if onFinish is not None: onFinish(alg, context, feedback) else: # auto convert layer references in results to map layers for out in alg.outputDefinitions(): if out.name() not in results: continue if isinstance(out, (QgsProcessingOutputVectorLayer, QgsProcessingOutputRasterLayer, QgsProcessingOutputMapLayer)): result = results[out.name()] if not isinstance(result, QgsMapLayer): layer = context.takeResultLayer( result ) # transfer layer ownership out of context if layer: results[out.name( )] = layer # replace layer string ref with actual layer (+ownership) elif isinstance(out, QgsProcessingOutputMultipleLayers): result = results[out.name()] if result: layers_result = [] for l in result: if not isinstance(result, QgsMapLayer): layer = context.takeResultLayer( l ) # transfer layer ownership out of context if layer: layers_result.append(layer) else: layers_result.append(l) else: layers_result.append(l) results[out.name( )] = layers_result # replace layers strings ref with actual layers (+ownership) else: msg = Processing.tr("There were errors executing the algorithm.") feedback.reportError(msg) raise QgsProcessingException(msg) if isinstance(feedback, MessageBarProgress): feedback.close() return results
def runGdal(commands, feedback=None): if feedback is None: feedback = QgsProcessingFeedback() envval = os.getenv('PATH') # We need to give some extra hints to get things picked up on OS X isDarwin = False try: isDarwin = platform.system() == 'Darwin' except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526 pass if isDarwin and os.path.isfile( os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")): # Looks like there's a bundled gdal. Let's use it. os.environ['PATH'] = "{}{}{}".format( os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep, envval) os.environ['DYLD_LIBRARY_PATH'] = os.path.join( QgsApplication.prefixPath(), "lib") else: # Other platforms should use default gdal finder codepath settings = QgsSettings() path = settings.value('/GdalTools/gdalPath', '') if not path.lower() in envval.lower().split(os.pathsep): envval += '{}{}'.format(os.pathsep, path) os.putenv('PATH', envval) fused_command = ' '.join([str(c) for c in commands]) QgsMessageLog.logMessage(fused_command, 'Processing', Qgis.Info) feedback.pushInfo('GDAL command:') feedback.pushCommandInfo(fused_command) feedback.pushInfo('GDAL command output:') success = False retry_count = 0 while not success: loglines = [] loglines.append('GDAL execution console output') try: with subprocess.Popen( fused_command, shell=True, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, stderr=subprocess.STDOUT, universal_newlines=True, ) as proc: for line in proc.stdout: feedback.pushConsoleInfo(line) loglines.append(line) success = True except IOError as e: if retry_count < 5: retry_count += 1 else: raise IOError( str(e) + u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}' .format(len(loglines), u'\n'.join(loglines[-10:]))) QgsMessageLog.logMessage('\n'.join(loglines), 'Processing', Qgis.Info) GdalUtils.consoleOutput = loglines
def runGdal(commands, feedback=None): if feedback is None: feedback = QgsProcessingFeedback() envval = os.getenv('PATH') # We need to give some extra hints to get things picked up on OS X isDarwin = False try: isDarwin = platform.system() == 'Darwin' except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526 pass if isDarwin and os.path.isfile( os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")): # Looks like there's a bundled gdal. Let's use it. os.environ['PATH'] = "{}{}{}".format( os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep, envval) os.environ['DYLD_LIBRARY_PATH'] = os.path.join( QgsApplication.prefixPath(), "lib") else: # Other platforms should use default gdal finder codepath settings = QgsSettings() path = settings.value('/GdalTools/gdalPath', '') if not path.lower() in envval.lower().split(os.pathsep): envval += '{}{}'.format(os.pathsep, path) os.putenv('PATH', envval) fused_command = ' '.join([str(c) for c in commands]) QgsMessageLog.logMessage(fused_command, 'Processing', Qgis.Info) feedback.pushInfo(GdalUtils.tr('GDAL command:')) feedback.pushCommandInfo(fused_command) feedback.pushInfo(GdalUtils.tr('GDAL command output:')) loglines = [GdalUtils.tr('GDAL execution console output')] def on_stdout(ba): val = ba.data().decode('UTF-8') # catch progress reports if val == '100 - done.': on_stdout.progress = 100 feedback.setProgress(on_stdout.progress) elif val in ('0', '10', '20', '30', '40', '50', '60', '70', '80', '90'): on_stdout.progress = int(val) feedback.setProgress(on_stdout.progress) elif val == '.': on_stdout.progress += 2.5 feedback.setProgress(on_stdout.progress) on_stdout.buffer += val if on_stdout.buffer.endswith('\n') or on_stdout.buffer.endswith( '\r'): # flush buffer feedback.pushConsoleInfo(on_stdout.buffer.rstrip()) loglines.append(on_stdout.buffer.rstrip()) on_stdout.buffer = '' on_stdout.progress = 0 on_stdout.buffer = '' def on_stderr(ba): val = ba.data().decode('UTF-8') on_stderr.buffer += val if on_stderr.buffer.endswith('\n') or on_stderr.buffer.endswith( '\r'): # flush buffer feedback.reportError(on_stderr.buffer.rstrip()) loglines.append(on_stderr.buffer.rstrip()) on_stderr.buffer = '' on_stderr.buffer = '' command, *arguments = QgsRunProcess.splitCommand(fused_command) proc = QgsBlockingProcess(command, arguments) proc.setStdOutHandler(on_stdout) proc.setStdErrHandler(on_stderr) res = proc.run(feedback) if feedback.isCanceled() and res != 0: feedback.pushInfo( GdalUtils.tr('Process was canceled and did not complete')) elif not feedback.isCanceled() and proc.exitStatus( ) == QProcess.CrashExit: raise QgsProcessingException( GdalUtils.tr('Process was unexpectedly terminated')) elif res == 0: feedback.pushInfo(GdalUtils.tr('Process completed successfully')) else: feedback.reportError( GdalUtils.tr('Process returned error code {}').format(res)) return loglines