def runGdal(commands, progress=None): if progress is None: progress = SilentProgress() envval = os.getenv('PATH') # We need to give some extra hints to get things picked up on OS X isDarwin = False try: isDarwin = platform.system() == 'Darwin' except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526 pass if isDarwin and os.path.isfile( os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")): # Looks like there's a bundled gdal. Let's use it. os.environ['PATH'] = "{}{}{}".format( os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep, envval) os.environ['DYLD_LIBRARY_PATH'] = os.path.join( QgsApplication.prefixPath(), "lib") else: # Other platforms should use default gdal finder codepath settings = QSettings() path = settings.value('/GdalTools/gdalPath', '') if not path.lower() in envval.lower().split(os.pathsep): envval += '{}{}'.format(os.pathsep, path) os.putenv('PATH', envval) fused_command = ' '.join([str(c) for c in commands]) ProcessingLog.addToLog(ProcessingLog.LOG_INFO, fused_command) progress.setInfo('GDAL command:') progress.setCommand(fused_command) progress.setInfo('GDAL command output:') success = False retry_count = 0 while success == False: loglines = [] loglines.append('GDAL execution console output') try: proc = subprocess.Popen( fused_command, shell=True, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, stderr=subprocess.STDOUT, universal_newlines=True, ).stdout for line in proc: progress.setConsoleInfo(line) loglines.append(line) success = True except IOError as e: if retry_count < 5: retry_count += 1 else: raise IOError( e.message + u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}' .format(len(loglines), u'\n'.join(loglines[-10:]))) ProcessingLog.addToLog(ProcessingLog.LOG_INFO, loglines) GdalUtils.consoleOutput = loglines
def runGdal(commands, progress=None): if progress is None: progress = SilentProgress() envval = os.getenv("PATH") # We need to give some extra hints to get things picked up on OS X isDarwin = False try: isDarwin = platform.system() == "Darwin" except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526 pass if isDarwin and os.path.isfile(os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")): # Looks like there's a bundled gdal. Let's use it. os.environ["PATH"] = "{}{}{}".format(os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep, envval) os.environ["DYLD_LIBRARY_PATH"] = os.path.join(QgsApplication.prefixPath(), "lib") else: # Other platforms should use default gdal finder codepath settings = QSettings() path = settings.value("/GdalTools/gdalPath", "") if not path.lower() in envval.lower().split(os.pathsep): envval += "{}{}".format(os.pathsep, path) os.putenv("PATH", envval) fused_command = " ".join([str(c) for c in commands]) ProcessingLog.addToLog(ProcessingLog.LOG_INFO, fused_command) progress.setInfo("GDAL command:") progress.setCommand(fused_command) progress.setInfo("GDAL command output:") success = False retry_count = 0 while success == False: loglines = [] loglines.append("GDAL execution console output") try: with subprocess.Popen( fused_command, shell=True, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, stderr=subprocess.STDOUT, universal_newlines=True, ) as proc: for line in proc.stdout: progress.setConsoleInfo(line) loglines.append(line) success = True except IOError as e: if retry_count < 5: retry_count += 1 else: raise IOError( e.message + u"\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}".format( len(loglines), u"\n".join(loglines[-10:]) ) ) ProcessingLog.addToLog(ProcessingLog.LOG_INFO, loglines) GdalUtils.consoleOutput = loglines
def handle(self, **kwargs): self.logger.debug('>>> Handler: 4...') vector_layer = kwargs['vector_layer'] plugin_name = kwargs['plugin_name'] if vector_layer: destination = 'memory:' if 'buffer_source' not in kwargs else kwargs[ 'buffer_source'] open_output_layer = True if 'open_output_layer' not in kwargs else kwargs[ 'open_output_layer'] buffer_layer = sp_buffering(plugin_name, SilentProgress(), vector_layer, destination=destination, open_output_layer=open_output_layer) if buffer_layer and buffer_layer.isValid(): kwargs['buffer_layer'] = buffer_layer self.next(**kwargs) else: self.error = 'Fail to create buffer, buffer is null or feature count is zero.' self.logger.fatal(self.error) raise RuntimeError(self.error) else: self.error = 'Failt to get grid layer.' self.logger.fatal(self.error) raise RuntimeError(self.error)
def execute(self, progress=SilentProgress(), model=None): """The method to use to call a processing algorithm. Although the body of the algorithm is in processAlgorithm(), it should be called using this method, since it performs some additional operations. Raises a GeoAlgorithmExecutionException in case anything goes wrong. """ self.model = model try: self.setOutputCRS() self.resolveOutputs() self.evaluateParameterValues() self.runPreExecutionScript(progress) self.processAlgorithm(progress) progress.setPercentage(100) self.convertUnsupportedFormats(progress) self.runPostExecutionScript(progress) except GeoAlgorithmExecutionException as gaee: lines = [self.tr('Error while executing algorithm')] lines.append(traceback.format_exc()) ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, gaee.msg) raise GeoAlgorithmExecutionException(gaee.msg, lines, gaee) except Exception as e: # If something goes wrong and is not caught in the # algorithm, we catch it here and wrap it lines = [self.tr('Uncaught error while executing algorithm')] lines.append(traceback.format_exc()) ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, lines) raise GeoAlgorithmExecutionException(str(e) + self.tr('\nSee log for more details'), lines, e)
def runalg(alg, progress=None): """Executes a given algorithm, showing its progress in the progress object passed along. Return true if everything went OK, false if the algorithm could not be completed. """ if progress is None: progress = SilentProgress() try: alg.execute(progress) return True except GeoAlgorithmExecutionException as e: ProcessingLog.addToLog(sys.exc_info()[0], ProcessingLog.LOG_ERROR) progress.error(e.msg) return False
def handleAlgorithmResults(alg, progress=None, showResults=True): wrongLayers = [] htmlResults = False if progress is None: progress = SilentProgress() progress.setText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers')) i = 0 for out in alg.outputs: progress.setPercentage(100 * i / float(len(alg.outputs))) if out.hidden or not out.open: continue if isinstance(out, (OutputRaster, OutputVector, OutputTable)): try: if hasattr(out, "layer") and out.layer is not None: out.layer.setLayerName(out.description) QgsMapLayerRegistry.instance().addMapLayers([out.layer]) else: if ProcessingConfig.getSetting( ProcessingConfig.USE_FILENAME_AS_LAYER_NAME): name = os.path.basename(out.value) else: name = out.description isRaster = True if isinstance(out, OutputRaster) else False dataobjects.load(out.value, name, alg.crs, RenderingStyles.getStyle(alg.commandLineName(), out.name), isRaster) except Exception: ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, "Error loading result layer:\n" + traceback.format_exc()) wrongLayers.append(out.description) elif isinstance(out, OutputFile) and out.ext == 'csv': try: dataobjects.load(out.value, name, alg.crs, RenderingStyles.getStyle(alg.commandLineName(), out.name)) except Exception: ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, "Error loading result layer:\n" + traceback.format_exc()) wrongLayers.append(out.description) elif isinstance(out, OutputHTML): ProcessingResults.addResult(out.description, out.value) htmlResults = True i += 1 QApplication.restoreOverrideCursor() if wrongLayers: msg = "The following layers were not correctly generated.<ul>" msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>" msg += "You can check the log messages to find more information about the execution of the algorithm" progress.error(msg) if showResults and htmlResults and not wrongLayers: dlg = ResultsDialog() dlg.exec_() return len(wrongLayers) == 0
def runalgIterating(alg, paramToIter, progress): # Generate all single-feature layers settings = QSettings() systemEncoding = settings.value('/UI/encoding', 'System') layerfile = alg.getParameterValue(paramToIter) layer = dataobjects.getObjectFromUri(layerfile, False) feat = QgsFeature() filelist = [] outputs = {} provider = layer.dataProvider() features = vector.features(layer) for feat in features: output = getTempFilename('shp') filelist.append(output) writer = QgsVectorFileWriter(output, systemEncoding, provider.fields(), provider.geometryType(), layer.crs()) writer.addFeature(feat) del writer # store output values to use them later as basenames for all outputs for out in alg.outputs: outputs[out.name] = out.value # now run all the algorithms i = 1 for f in filelist: alg.setParameterValue(paramToIter, f) for out in alg.outputs: filename = outputs[out.name] if filename: filename = filename[:filename.rfind('.')] + '_' + str(i) \ + filename[filename.rfind('.'):] out.value = filename progress.setText('Executing iteration ' + str(i) + '/' + str(len(filelist)) + '...') progress.setPercentage(i * 100 / len(filelist)) if UnthreadedAlgorithmExecutor.runalg(alg, SilentProgress()): Postprocessing.handleAlgorithmResults(alg, SilentProgress(), False) i += 1 else: return False return True
def handle(self, **kwargs): self.logger.debug('>>> Handler: 5...') torre_layer = kwargs['torre_layer'] if torre_layer: buffer_layer = sp_buffering(PLUGIN_VULNERABILIDADE, SilentProgress(), torre_layer) kwargs['buffer_layer_teste'] = buffer_layer self.next(**kwargs)
def handle(self, **kwargs): self.logger.error('>>> Handler: 5...') pc_layer = kwargs['pc_layer'] if pc_layer: buffer_pc_layer = sp_buffering(PLUGIN_VULNERABILIDADE, SilentProgress(), pc_layer) kwargs['buffer_pc_layer'] = buffer_pc_layer self.next(**kwargs)
def handle(self, **kwargs): self.logger.debug('>>> Handler: 4...') self.logger.debug(kwargs) vao_layer = kwargs['vao_layer'] if vao_layer: buffer_layer = calc(PLUGIN_VULNERABILIDADE, SilentProgress(), vao_layer) kwargs['buffer_layer'] = buffer_layer self.next(**kwargs)
def handle(self, **kwargs): self.logger.debug('>>> Handler: 4...') self.logger.debug(kwargs['line_layer']) line_layer = kwargs['clip_line_layers'] if line_layer: buffer_layer = sp_buffering(PLUGIN_VULNERABILIDADE, SilentProgress(), line_layer) kwargs['buffer_clip_layer'] = buffer_layer self.next(**kwargs)
def preprocess(self, layer): ''' Preprocesses the layer with the corresponding preprocess hook and returns the path to the resulting layer. If no preprocessing is performed, it returns the input layer itself ''' if not processingOk: return layer if layer.type() == layer.RasterLayer: try: hookFile = str(QSettings().value( "/OpenGeo/Settings/GeoServer/PreuploadRasterHook", "")) alg = self.getAlgorithmFromHookFile(hookFile) if (len(alg.parameters) == 1 and isinstance(alg.parameters[0], ParameterRaster) and len(alg.outputs) == 1 and isinstance(alg.outputs[0], OutputRaster)): alg.parameters[0].setValue(layer) if UnthreadedAlgorithmExecutor.runalg( alg, SilentProgress()): return load(alg.outputs[0].value) return layer except: return layer elif layer.type() == layer.VectorLayer: try: hookFile = str(QSettings().value( "/OpenGeo/Settings/GeoServer/PreuploadVectorHook", "")) alg = self.getAlgorithmFromHookFile(hookFile) if (len(alg.parameters) == 1 and isinstance(alg.parameters[0], ParameterVector) and len(alg.outputs) == 1 and isinstance(alg.outputs[0], OutputVector)): alg.parameters[0].setValue(layer) if UnthreadedAlgorithmExecutor.runalg( alg, SilentProgress()): return load(alg.outputs[0].value) return layer except: return layer
def handle(self, **kwargs): self.logger.debug('>>> Handler: 8...') self.logger.debug(kwargs['clip_line_layers']) line_layer = kwargs['clip_line_layers'] output_vector = kwargs['output_vector'] if line_layer: buffer_layer = buffer_line(PLUGIN_VULNERABILIDADE, SilentProgress(), line_layer, output_vector) kwargs['buffer_line_layer'] = buffer_layer self.next(**kwargs)
def runalg(alg, progress=None): """Executes a given algorithm, showing its progress in the progress object passed along. Return true if everything went OK, false if the algorithm could not be completed. """ try: alg.execute(progress or SilentProgress()) return True except GeoAlgorithmExecutionException as e: ProcessingLog.addToLog(sys.exc_info()[0], ProcessingLog.LOG_ERROR) progress.error(e.msg) return False
def runGdal(commands, progress=None): if progress is None: progress = SilentProgress() envval = os.getenv('PATH') # We need to give some extra hints to get things picked up on OS X isDarwin = False try: isDarwin = platform.system() == 'Darwin' except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526 pass if isDarwin and os.path.isfile(os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")): # Looks like there's a bundled gdal. Let's use it. os.environ['PATH'] = "{}{}{}".format(os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep, envval) os.environ['DYLD_LIBRARY_PATH'] = os.path.join(QgsApplication.prefixPath(), "lib") else: # Other platforms should use default gdal finder codepath settings = QSettings() path = settings.value('/GdalTools/gdalPath', '') if not path.lower() in envval.lower().split(os.pathsep): envval += '{}{}'.format(os.pathsep, path) os.putenv('PATH', envval) loglines = [] loglines.append('GDAL execution console output') fused_command = ''.join(['%s ' % c for c in commands]) progress.setInfo('GDAL command:') progress.setCommand(fused_command) proc = subprocess.Popen( fused_command, shell=True, stdout=subprocess.PIPE, stdin=open(os.devnull), stderr=subprocess.STDOUT, universal_newlines=True, ).stdout progress.setInfo('GDAL command output:') for line in iter(proc.readline, ''): progress.setConsoleInfo(line) loglines.append(line) ProcessingLog.addToLog(ProcessingLog.LOG_INFO, loglines) GdalUtils.consoleOutput = loglines
def handleAlgorithmResults(alg, progress=None, showResults=True): wrongLayers = [] htmlResults = False if progress is None: progress = SilentProgress() progress.setText(QCoreApplication.translate('Postprocessing', 'Loading resulting layers')) i = 0 for out in alg.outputs: progress.setPercentage(100 * i / float(len(alg.outputs))) if out.hidden or not out.open: continue if isinstance(out, (OutputRaster, OutputVector, OutputTable)): try: if hasattr(out, "layer") and out.layer is not None: out.layer.setLayerName(out.description) QgsMapLayerRegistry.instance().addMapLayers([out.layer]) else: if ProcessingConfig.getSetting( ProcessingConfig.USE_FILENAME_AS_LAYER_NAME): name = os.path.basename(out.value) else: name = out.description isRaster = True if isinstance(out, OutputRaster) else False dataobjects.load(out.value, name, alg.crs, RenderingStyles.getStyle(alg.commandLineName(), out.name), isRaster) except Exception: ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, "Error loading result layer:\n" + traceback.format_exc()) wrongLayers.append(out.description) elif isinstance(out, OutputHTML): ProcessingResults.addResult(out.description, out.value) htmlResults = True i += 1 QApplication.restoreOverrideCursor() if wrongLayers: msg = "The following layers were not correctly generated.<ul>" msg += "".join(["<li>%s</li>" % lay for lay in wrongLayers]) + "</ul>" msg += "You can check the log messages to find more information about the execution of the algorithm" progress.error(msg) if showResults and htmlResults and not wrongLayers: dlg = ResultsDialog() dlg.exec_() return len(wrongLayers) == 0
def getInstalledVersion(runOtb=False): global _installedVersion global _installedVersionFound if _installedVersionFound and not runOtb: return _installedVersion if otbPath() is None or otbLibPath() is None: _installedVersionFound = False return None commands = [os.path.join(otbPath(), "otbcli_Smoothing")] progress = SilentProgress() out = executeOtb(commands, progress, False) for line in out: if "version" in line: _installedVersionFound = True _installedVersion = line.split("version")[-1].strip() break return _installedVersion
def execute(self, progress=SilentProgress(), model=None): """The method to use to call a processing algorithm. Although the body of the algorithm is in processAlgorithm(), it should be called using this method, since it performs some additional operations. Raises a GeoAlgorithmExecutionException in case anything goes wrong. """ self.model = model try: self.setOutputCRS() self.resolveTemporaryOutputs() self.resolveDataObjects() self.checkOutputFileExtensions() self.runPreExecutionScript(progress) self.processAlgorithm(progress) progress.setPercentage(100) self.convertUnsupportedFormats(progress) self.runPostExecutionScript(progress) except GeoAlgorithmExecutionException as gaee: lines = [self.tr('Uncaught error while executing algorithm')] lines.append(traceback.format_exc()) ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, gaee.msg) raise GeoAlgorithmExecutionException(gaee.msg, lines, gaee) except Exception as e: # If something goes wrong and is not caught in the # algorithm, we catch it here and wrap it lines = [self.tr('Uncaught error while executing algorithm')] lines.append(traceback.format_exc()) ProcessingLog.addToLog(ProcessingLog.LOG_ERROR, lines) try: message = unicode(e) except UnicodeDecodeError: # Try with the 'replace' mode (requires e.message instead of e!) message = unicode(e.message, 'utf-8', 'replace') raise GeoAlgorithmExecutionException( message + self.tr(' \nSee log for more details'), lines, e)
def runAlgorithm(algOrName, onFinish, *args): if isinstance(algOrName, GeoAlgorithm): alg = algOrName else: alg = Processing.getAlgorithm(algOrName) if alg is None: print 'Error: Algorithm not found\n' return if len(args) != alg.getVisibleParametersCount() \ + alg.getVisibleOutputsCount(): print 'Error: Wrong number of parameters' processing.alghelp(algOrName) return alg = alg.getCopy() if isinstance(args, dict): # Set params by name for (name, value) in args.items(): if alg.getParameterFromName(name).setValue(value): continue if alg.getOutputFromName(name).setValue(value): continue print 'Error: Wrong parameter value %s for parameter %s.' \ % (value, name) return else: i = 0 for param in alg.parameters: if not param.hidden: if not param.setValue(args[i]): print 'Error: Wrong parameter value: ' \ + unicode(args[i]) return i = i + 1 for output in alg.outputs: if not output.hidden: if not output.setValue(args[i]): print 'Error: Wrong output value: ' + unicode(args[i]) return i = i + 1 msg = alg.checkParameterValuesBeforeExecuting() if msg: print 'Unable to execute algorithm\n' + msg return if not alg.checkInputCRS(): print 'Warning: Not all input layers use the same CRS.\n' \ + 'This can cause unexpected results.' ProcessingLog.addToLog(ProcessingLog.LOG_ALGORITHM, alg.getAsCommand()) # Don't set the wait cursor twice, because then when you # restore it, it will still be a wait cursor. cursor = QApplication.overrideCursor() if cursor is None or cursor == 0: QApplication.setOverrideCursor(QCursor(Qt.WaitCursor)) elif cursor.shape() != Qt.WaitCursor: QApplication.setOverrideCursor(QCursor(Qt.WaitCursor)) progress = SilentProgress() if interface.iface is not None : progress = MessageBarProgress() ret = UnthreadedAlgorithmExecutor.runalg(alg, progress) if onFinish is not None and ret: onFinish(alg, progress) QApplication.restoreOverrideCursor() progress.close() return alg
def runAlgorithm(algOrName, onFinish, *args): if isinstance(algOrName, GeoAlgorithm): alg = algOrName else: alg = Processing.getAlgorithm(algOrName) if alg is None: print 'Error: Algorithm not found\n' return if len(args) != alg.getVisibleParametersCount() \ + alg.getVisibleOutputsCount(): print 'Error: Wrong number of parameters' processing.alghelp(algOrName) return alg = alg.getCopy() if isinstance(args, dict): # Set params by name for (name, value) in args.items(): if alg.getParameterFromName(name).setValue(value): continue if alg.getOutputFromName(name).setValue(value): continue print 'Error: Wrong parameter value %s for parameter %s.' \ % (value, name) return else: i = 0 for param in alg.parameters: if not param.hidden: if not param.setValue(args[i]): print 'Error: Wrong parameter value: ' \ + unicode(args[i]) return i = i + 1 for output in alg.outputs: if not output.hidden: if not output.setValue(args[i]): print 'Error: Wrong output value: ' + unicode(args[i]) return i = i + 1 msg = alg.checkParameterValuesBeforeExecuting() if msg: print 'Unable to execute algorithm\n' + msg return if not alg.checkInputCRS(): print 'Warning: Not all input layers use the same CRS.\n' \ + 'This can cause unexpected results.' ProcessingLog.addToLog(ProcessingLog.LOG_ALGORITHM, alg.getAsCommand()) # Don't set the wait cursor twice, because then when you # restore it, it will still be a wait cursor. cursor = QApplication.overrideCursor() if cursor is None or cursor == 0: QApplication.setOverrideCursor(QCursor(Qt.WaitCursor)) elif cursor.shape() != Qt.WaitCursor: QApplication.setOverrideCursor(QCursor(Qt.WaitCursor)) progress = SilentProgress() if iface is not None : progress = MessageBarProgress() ret = UnthreadedAlgorithmExecutor.runalg(alg, progress) if onFinish is not None and ret: onFinish(alg, progress) QApplication.restoreOverrideCursor() progress.close() return alg
def runChole(commands, progress=None): cwd=os.path.dirname(__file__) + os.sep + 'Chloe2012' if progress is None: progress = SilentProgress() fused_command = ' '.join(commands) progress.setInfo('Command:') progress.setCommand(fused_command) progress.setInfo('Output:') # Execution Chloe with subprocess command system success = False retry_count = 0 while success == False: loglines = [] loglines.append('Execution console output :') try: process = subprocess.Popen( fused_command, shell=True, stdout=subprocess.PIPE, stdin=open(os.devnull), stderr=subprocess.STDOUT, universal_newlines=True, cwd=cwd, ) regex = re.compile(r'^\s*##\s*(?P<percentage>\d+)\s*\/\s*\d+\s*$') while True: output = process.stdout.readline() if output == '' and process.poll() is not None: break if output: line = output.strip() res = regex.search(line) if res: percentage = int(res.group('percentage')) progress.setPercentage(percentage) else: progress.setConsoleInfo(line) loglines.append(line) rc = process.poll() success = True except IOError as e: if retry_count < 5: retry_count += 1 else: raise IOError(e.message + u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}'.format(len(loglines), u'\n'.join(loglines[-10:]))) # Save log ProcessingLog.addToLog(ProcessingLog.LOG_INFO, loglines) ChloeUtils.consoleOutput = loglines
def execute(self, progress=SilentProgress(), model=None): self.collectNonTemporaryOutputs() GeoAlgorithm.execute(self, progress, model)