def runGdal(commands, feedback=None):
if feedback is None:
feedback = QgsProcessingFeedback()
envval = os.getenv('PATH')
# We need to give some extra hints to get things picked up on OS X
isDarwin = False
try:
isDarwin = platform.system() == 'Darwin'
except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526
pass
if isDarwin and os.path.isfile(
os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")):
# Looks like there's a bundled gdal. Let's use it.
os.environ['PATH'] = "{}{}{}".format(
os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep,
envval)
os.environ['DYLD_LIBRARY_PATH'] = os.path.join(
QgsApplication.prefixPath(), "lib")
else:
# Other platforms should use default gdal finder codepath
settings = QgsSettings()
path = settings.value('/GdalTools/gdalPath', '')
if not path.lower() in envval.lower().split(os.pathsep):
envval += '{}{}'.format(os.pathsep, path)
os.putenv('PATH', envval)
fused_command = ' '.join([str(c) for c in commands])
ProcessingLog.addToLog(ProcessingLog.LOG_INFO, fused_command)
feedback.pushInfo('GDAL command:')
feedback.pushCommandInfo(fused_command)
feedback.pushInfo('GDAL command output:')
success = False
retry_count = 0
while not success:
loglines = []
loglines.append('GDAL execution console output')
try:
with subprocess.Popen(
fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
) as proc:
for line in proc.stdout:
feedback.pushConsoleInfo(line)
loglines.append(line)
success = True
except IOError as e:
if retry_count < 5:
retry_count += 1
else:
raise IOError(
e.message +
u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}'
.format(len(loglines), u'\n'.join(loglines[-10:])))
ProcessingLog.addToLog(ProcessingLog.LOG_INFO, loglines)
GdalUtils.consoleOutput = loglines
def execute(commands, feedback=None):
if feedback is None:
feedback = QgsProcessingFeedback()
fused_command = ' '.join([str(c) for c in commands])
QgsMessageLog.logMessage(fused_command, 'Processing', Qgis.Info)
feedback.pushInfo('WhiteBoxTools command:')
feedback.pushCommandInfo(fused_command)
feedback.pushInfo('WhiteBoxTools command output:')
loglines = []
with subprocess.Popen(fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True) as proc:
try:
for line in iter(proc.stdout.readline, ''):
if '%' in line:
try:
feedback.setProgress(int(progressRegex.search(line).group(0)))
except:
pass
else:
feedback.pushConsoleInfo(line)
loglines.append(line)
except:
pass
if ProcessingConfig.getSetting(WHITEBOX_VERBOSE):
QgsMessageLog.logMessage('\n'.join(loglines), 'Processing', Qgis.Info)
def execute(command, feedback=None):
if feedback is None:
feedback = QgsProcessingFeedback()
fused_command = ' '.join([str(c) for c in commands])
QgsMessageLog.logMessage(fused_command, 'Processing', QgsMessageLog.INFO)
feedback.pushInfo('WhiteboxTools command:')
feedback.pushCommandInfo(fused_command)
feedback.pushInfo('WhiteboxTools command output:')
loglines = []
with subprocess.Popen(fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True) as proc:
try:
for line in proc.stdout:
feedback.pushConsoleInfo(line)
loglines.append(line)
except:
pass
if ProcessingConfig.getSetting(WHITEBOX_VERBOSE):
QgsMessageLog.logMessage(
'\n'.join(loglines), 'Processing', QgsMessageLog.INFO)
def execute(commands, feedback=None):
print(commands)
if feedback is None:
feedback = QgsProcessingFeedback()
fused_command = ' '.join([str(c) for c in commands])
QgsMessageLog.logMessage(fused_command, 'Processing', Qgis.Info)
feedback.pushInfo('FUSION command:')
feedback.pushCommandInfo(fused_command)
feedback.pushInfo('FUSION command output:')
loglines = []
with subprocess.Popen(fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True) as proc:
try:
for line in iter(proc.stdout.readline, ''):
feedback.pushConsoleInfo(line)
loglines.append(line)
except:
pass
if ProcessingConfig.getSetting(FUSION_VERBOSE):
QgsMessageLog.logMessage('\n'.join(loglines), 'Processing', Qgis.Info)
def runGdal(commands, feedback=None):
if feedback is None:
feedback = QgsProcessingFeedback()
envval = os.getenv('PATH')
# We need to give some extra hints to get things picked up on OS X
isDarwin = False
try:
isDarwin = platform.system() == 'Darwin'
except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526
pass
if isDarwin and os.path.isfile(os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")):
# Looks like there's a bundled gdal. Let's use it.
os.environ['PATH'] = "{}{}{}".format(os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep, envval)
os.environ['DYLD_LIBRARY_PATH'] = os.path.join(QgsApplication.prefixPath(), "lib")
else:
# Other platforms should use default gdal finder codepath
settings = QgsSettings()
path = settings.value('/GdalTools/gdalPath', '')
if not path.lower() in envval.lower().split(os.pathsep):
envval += '{}{}'.format(os.pathsep, path)
os.putenv('PATH', envval)
fused_command = ' '.join([str(c) for c in commands])
QgsMessageLog.logMessage(fused_command, 'Processing', Qgis.Info)
feedback.pushInfo('GDAL command:')
feedback.pushCommandInfo(fused_command)
feedback.pushInfo('GDAL command output:')
success = False
retry_count = 0
while not success:
loglines = []
loglines.append('GDAL execution console output')
try:
with subprocess.Popen(
fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True,
) as proc:
for line in proc.stdout:
feedback.pushConsoleInfo(line)
loglines.append(line)
success = True
except IOError as e:
if retry_count < 5:
retry_count += 1
else:
raise IOError(
str(e) + u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}'.format(
len(loglines), u'\n'.join(loglines[-10:])))
QgsMessageLog.logMessage('\n'.join(loglines), 'Processing', Qgis.Info)
GdalUtils.consoleOutput = loglines
def execute(commands, feedback=None):
cmds = []
cmds.append(os.path.join(mpichDirectory(), "mpiexec"))
processes = ProcessingConfig.getSetting(TAUDEM_PROCESSES)
if int(processes) <= 0:
processes = 1
cmds.append("-n")
cmds.append(processes)
cmds.extend(commands)
if feedback is None:
feedback = QgsProcessingFeedback()
fused_command = " ".join([str(c) for c in cmds])
# QgsMessageLog.logMessage(fused_command, "Processing", QgsMessageLog.INFO)
QgsMessageLog.logMessage(fused_command, "Processing", Qgis.Info)
feedback.pushInfo("TauDEM command:")
feedback.pushCommandInfo(fused_command)
feedback.pushInfo("TauDEM command output:")
loglines = []
with subprocess.Popen(fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True) as proc:
try:
for line in iter(proc.stdout.readline, ""):
feedback.pushConsoleInfo(line)
loglines.append(line)
except:
pass
if ProcessingConfig.getSetting(TAUDEM_VERBOSE):
# QgsMessageLog.logMessage("\n".join(loglines), "Processing", QgsMessageLog.INFO)
QgsMessageLog.logMessage("\n".join(loglines), "Processing", Qgis.Info)
def execute(commands, feedback=None):
command = prepareArguments(commands)
if feedback is None:
feedback = QgsProcessingFeedback()
feedback.pushInfo('pktools command:')
feedback.pushCommandInfo(command)
feedback.pushInfo('pktools command output:')
loglines = []
with subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True) as proc:
try:
for line in iter(proc.stdout.readline, ''):
feedback.pushConsoleInfo(line)
loglines.append(line)
except:
pass
def execute(commands, feedback=None):
cmds = []
cmds.append(os.path.join(mpichDirectory(), "mpiexec"))
processes = ProcessingConfig.getSetting(TAUDEM_PROCESSES)
if int(processes) <= 0:
processes = 1
cmds.append("-n")
cmds.append(processes)
cmds.extend(commands)
if feedback is None:
feedback = QgsProcessingFeedback()
fused_command = " ".join([str(c) for c in cmds])
QgsMessageLog.logMessage(fused_command, "Processing", QgsMessageLog.INFO)
feedback.pushInfo("TauDEM command:")
feedback.pushCommandInfo(fused_command)
feedback.pushInfo("TauDEM command output:")
loglines = []
with subprocess.Popen(fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
universal_newlines=True) as proc:
try:
for line in iter(proc.stdout.readline, ""):
feedback.pushConsoleInfo(line)
loglines.append(line)
except:
pass
if ProcessingConfig.getSetting(TAUDEM_VERBOSE):
QgsMessageLog.logMessage("\n".join(loglines), "Processing", QgsMessageLog.INFO)
def runGdal(commands, feedback=None):
if feedback is None:
feedback = QgsProcessingFeedback()
envval = os.getenv('PATH')
# We need to give some extra hints to get things picked up on OS X
isDarwin = False
try:
isDarwin = platform.system() == 'Darwin'
except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526
pass
if isDarwin and os.path.isfile(os.path.join(QgsApplication.prefixPath(), "bin", "gdalinfo")):
# Looks like there's a bundled gdal. Let's use it.
os.environ['PATH'] = "{}{}{}".format(os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep, envval)
os.environ['DYLD_LIBRARY_PATH'] = os.path.join(QgsApplication.prefixPath(), "lib")
else:
# Other platforms should use default gdal finder codepath
settings = QgsSettings()
path = settings.value('/GdalTools/gdalPath', '')
if not path.lower() in envval.lower().split(os.pathsep):
envval += '{}{}'.format(os.pathsep, path)
os.putenv('PATH', envval)
fused_command = ' '.join([str(c) for c in commands])
QgsMessageLog.logMessage(fused_command, 'Processing', Qgis.Info)
feedback.pushInfo(GdalUtils.tr('GDAL command:'))
feedback.pushCommandInfo(fused_command)
feedback.pushInfo(GdalUtils.tr('GDAL command output:'))
loglines = [GdalUtils.tr('GDAL execution console output')]
def on_stdout(ba):
val = ba.data().decode('UTF-8')
# catch progress reports
if val == '100 - done.':
on_stdout.progress = 100
feedback.setProgress(on_stdout.progress)
elif val in ('0', '10', '20', '30', '40', '50', '60', '70', '80', '90'):
on_stdout.progress = int(val)
feedback.setProgress(on_stdout.progress)
elif val == '.':
on_stdout.progress += 2.5
feedback.setProgress(on_stdout.progress)
on_stdout.buffer += val
if on_stdout.buffer.endswith('\n') or on_stdout.buffer.endswith('\r'):
# flush buffer
feedback.pushConsoleInfo(on_stdout.buffer.rstrip())
loglines.append(on_stdout.buffer.rstrip())
on_stdout.buffer = ''
on_stdout.progress = 0
on_stdout.buffer = ''
def on_stderr(ba):
val = ba.data().decode('UTF-8')
on_stderr.buffer += val
if on_stderr.buffer.endswith('\n') or on_stderr.buffer.endswith('\r'):
# flush buffer
feedback.reportError(on_stderr.buffer.rstrip())
loglines.append(on_stderr.buffer.rstrip())
on_stderr.buffer = ''
on_stderr.buffer = ''
command, *arguments = QgsRunProcess.splitCommand(fused_command)
proc = QgsBlockingProcess(command, arguments)
proc.setStdOutHandler(on_stdout)
proc.setStdErrHandler(on_stderr)
res = proc.run(feedback)
if feedback.isCanceled() and res != 0:
feedback.pushInfo(GdalUtils.tr('Process was canceled and did not complete'))
elif not feedback.isCanceled() and proc.exitStatus() == QProcess.CrashExit:
raise QgsProcessingException(GdalUtils.tr('Process was unexpectedly terminated'))
elif res == 0:
feedback.pushInfo(GdalUtils.tr('Process completed successfully'))
elif proc.processError() == QProcess.FailedToStart:
raise QgsProcessingException(GdalUtils.tr('Process {} failed to start. Either {} is missing, or you may have insufficient permissions to run the program.').format(command, command))
else:
feedback.reportError(GdalUtils.tr('Process returned error code {}').format(res))
return loglines
def runChloe(commands, feedback=None):
cwd = os.path.dirname(__file__) + os.sep + 'Chloe2012'
print('cwd : ' + cwd)
if feedback is None:
feedback = QgsProcessingFeedback()
envval = os.getenv('PATH')
# We need to give some extra hints to get things picked up on OS X
isDarwin = False
try:
isDarwin = platform.system() == 'Darwin'
except IOError: # https://travis-ci.org/m-kuhn/QGIS#L1493-L1526
pass
if isDarwin and os.path.isfile(
os.path.join(QgsApplication.prefixPath(), "bin", "chloeinfo")):
# Looks like there's a bundled chloe. Let's use it.
os.environ['PATH'] = "{}{}{}".format(
os.path.join(QgsApplication.prefixPath(), "bin"), os.pathsep,
envval)
os.environ['DYLD_LIBRARY_PATH'] = os.path.join(
QgsApplication.prefixPath(), "lib")
else:
# Other platforms should use default chloe finder codepath
settings = QgsSettings()
path = settings.value('/ChloeTools/chloePath', '')
if not path.lower() in envval.lower().split(os.pathsep):
envval += '{}{}'.format(os.pathsep, path)
os.putenv('PATH', envval)
fused_command = ' '.join([str(c) for c in commands])
QgsMessageLog.logMessage(fused_command, 'Processing', Qgis.Info)
feedback.pushInfo('CHLOE command:')
feedback.pushCommandInfo(fused_command)
feedback.pushInfo('CHLOE command output:')
success = False
retry_count = 0
while not success:
loglines = []
loglines.append('CHLOE execution console output')
print('step while')
try:
#print('runChloe')
#feedback.pushConsoleInfo('runChloe')
with subprocess.Popen(
fused_command,
shell=True,
stdout=subprocess.PIPE,
stdin=subprocess.DEVNULL,
stderr=subprocess.STDOUT,
#universal_newlines=True,
cwd=cwd) as proc:
for byte_line in proc.stdout:
line = byte_line.decode(
'utf8',
errors='backslashreplace').replace('\r', '')
feedback.pushConsoleInfo(line)
loglines.append(line)
success = True
except IOError as e:
if retry_count < 5:
print('retry ' + str(retry_count))
retry_count += 1
else:
raise IOError(
str(e) +
u'\nTried 5 times without success. Last iteration stopped after reading {} line(s).\nLast line(s):\n{}'
.format(len(loglines), u'\n'.join(loglines[-10:])))
QgsMessageLog.logMessage('\n'.join(loglines), 'Processing',
Qgis.Info)
ChloeUtils.consoleOutput = loglines
