def testDefaultScriptsFolder(self):
"""
Tests default user scripts folder
"""
self.assertTrue(RUtils.default_scripts_folder())
self.assertIn('rscripts', RUtils.default_scripts_folder())
self.assertTrue(os.path.exists(RUtils.default_scripts_folder()))
def testBuiltInPath(self):
"""
Tests built in scripts path
"""
self.assertTrue(RUtils.builtin_scripts_folder())
self.assertIn('builtin_scripts', RUtils.builtin_scripts_folder())
self.assertTrue(os.path.exists(RUtils.builtin_scripts_folder()))
def testScriptsFolders(self):
"""
Test script folders
"""
self.assertTrue(RUtils.script_folders())
self.assertIn(RUtils.default_scripts_folder(), RUtils.script_folders())
self.assertIn(RUtils.builtin_scripts_folder(), RUtils.script_folders())
def build_script_header_commands(self, _, __, ___):
"""
Builds the set of script startup commands for the algorithm
"""
commands = list()
# Just use main mirror
commands.append('options("repos"="{}")'.format(RUtils.package_repo()))
# Try to install packages if needed
if RUtils.use_user_library():
commands.append('.libPaths(\"' +
str(RUtils.r_library_folder()).replace('\\', '/') +
'\")')
packages = RUtils.get_required_packages(self.script)
packages.extend(['rgdal', 'raster'])
for p in packages:
commands.append('tryCatch(find.package("' + p +
'"), error=function(e) install.packages("' + p +
'", dependencies=TRUE))')
commands.append('library("raster")')
commands.append('library("rgdal")')
return commands
def process_metadata_line(self, line): # pylint: disable=too-many-return-statements
"""
Processes a "metadata" (##) line
"""
line = line.replace('#', '')
# special commands
# showplots is the older version, should be considere obsolete
if line.lower().strip().startswith('output_plots_to_html') or \
line.lower().strip().startswith('showplots'):
self.show_plots = True
self.addParameter(
QgsProcessingParameterFileDestination(
RAlgorithm.RPLOTS,
self.tr('R Plots'),
self.tr('HTML files (*.html)'),
optional=True))
return
# dontuserasterpackage is the older version, should be considere obsolete
if line.lower().strip().startswith('load_raster_using_rgdal') or \
line.lower().strip().startswith('dontuserasterpackage'):
self.r_templates.use_raster = False
return
if line.lower().strip().startswith('load_vector_using_rgdal'):
self.r_templates.use_sf = False
return
# passfilenames is the older version, should be considere obsolete
if line.lower().strip().startswith('pass_filenames') or\
line.lower().strip().startswith('passfilenames'):
self.pass_file_names = True
return
if line.lower().strip().startswith('dont_load_any_packages'):
self.r_templates.auto_load_packages = False
return
value, type_ = self.split_tokens(line)
if type_.lower().strip() == 'group':
self._group = value
return
if type_.lower().strip() == 'name':
self._name = self._display_name = value
self._name = RUtils.strip_special_characters(self._name.lower())
return
if type_.lower().strip() == 'display_name':
self._display_name = value
return
if type_.lower().strip() == 'github_install':
self.r_templates.install_github = True
self.r_templates.github_dependencies = value
return
# process enum with values and preparing its template
if "=enum literal" in RUtils.upgrade_parameter_line(line):
self.r_templates.add_literal_enum(value)
self.process_parameter_line(line)
def process_parameter_line(self, line):
"""
Processes a single script line representing a parameter
"""
value, _ = self.split_tokens(line)
description = RUtils.create_descriptive_name(value)
output = create_output_from_string(line)
if output is not None:
output.setName(value)
output.setDescription(description)
if issubclass(output.__class__, QgsProcessingOutputDefinition):
self.addOutput(output)
else:
# destination type parameter
self.addParameter(output)
else:
line = RUtils.upgrade_parameter_line(line)
# this is annoying, but required to work around a bug in early 3.8.0 versions
try:
param = getParameterFromString(line, context="")
except TypeError:
param = getParameterFromString(line)
if param is not None:
self.addParameter(param)
else:
self.error = self.tr('This script has a syntax error.\n'
'Problem with line: {0}').format(line)
def test_is_error_line(self):
"""
Test is_error_line
"""
self.assertFalse(RUtils.is_error_line('xxx yyy'))
self.assertTrue(RUtils.is_error_line('Error something went wrong'))
self.assertTrue(RUtils.is_error_line('Execution halted'))
def test_use_user_library(self):
"""
Test retrieving/setting the user library setting
"""
self.assertTrue(RUtils.use_user_library())
ProcessingConfig.setSettingValue(RUtils.R_USE_USER_LIB, False)
self.assertFalse(RUtils.use_user_library())
ProcessingConfig.setSettingValue(RUtils.R_USE_USER_LIB, True)
self.assertTrue(RUtils.use_user_library())
def test_package_repo(self):
"""
Test retrieving/setting the package repo
"""
self.assertEqual(RUtils.package_repo(), 'http://cran.at.r-project.org/')
ProcessingConfig.setSettingValue(RUtils.R_REPO, 'http://mirror.at.r-project.org/')
self.assertEqual(RUtils.package_repo(), 'http://mirror.at.r-project.org/')
ProcessingConfig.setSettingValue(RUtils.R_REPO, 'http://cran.at.r-project.org/')
self.assertEqual(RUtils.package_repo(), 'http://cran.at.r-project.org/')
def test_r_binary_folder(self):
"""
Test retrieving R binary folder
"""
self.assertFalse(RUtils.r_binary_folder())
ProcessingConfig.setSettingValue(RUtils.R_FOLDER, '/usr/local/bin')
self.assertEqual(RUtils.r_binary_folder(), '/usr/local/bin')
ProcessingConfig.setSettingValue(RUtils.R_FOLDER, None)
self.assertFalse(RUtils.r_binary_folder())
def test_library_folder(self):
"""
Test retrieving/setting the library folder
"""
self.assertIn('/profiles/default/processing/rlibs', RUtils.r_library_folder())
ProcessingConfig.setSettingValue(RUtils.R_LIBS_USER, '/usr/local')
self.assertEqual(RUtils.r_library_folder(), '/usr/local')
ProcessingConfig.setSettingValue(RUtils.R_LIBS_USER, None)
self.assertIn('/profiles/default/processing/rlibs', RUtils.r_library_folder())
def test_r_is_installed(self):
"""
Test checking that R is installed
"""
self.assertIsNone(RUtils.check_r_is_installed())
ProcessingConfig.setSettingValue(RUtils.R_FOLDER, '/home')
self.assertTrue(RUtils.check_r_is_installed())
self.assertIn('R is not installed', RUtils.check_r_is_installed())
ProcessingConfig.setSettingValue(RUtils.R_FOLDER, None)
self.assertIsNone(RUtils.check_r_is_installed())
def load(self):
"""
Called when first loading provider
"""
ProcessingConfig.settingIcons[self.name()] = self.icon()
ProcessingConfig.addSetting(
Setting(self.name(),
RUtils.RSCRIPTS_FOLDER,
self.tr('R scripts folder'),
RUtils.default_scripts_folder(),
valuetype=Setting.MULTIPLE_FOLDERS))
ProcessingConfig.addSetting(
Setting(
self.name(), RUtils.R_USE_USER_LIB,
self.tr('Use user library folder instead of system libraries'),
True))
ProcessingConfig.addSetting(
Setting(self.name(),
RUtils.R_LIBS_USER,
self.tr('User library folder'),
RUtils.r_library_folder(),
valuetype=Setting.FOLDER))
ProcessingConfig.addSetting(
Setting(self.name(),
RUtils.R_REPO,
self.tr('Package repository'),
"http://cran.at.r-project.org/",
valuetype=Setting.STRING))
ProcessingConfig.addSetting(
Setting(self.name(),
RUtils.R_FOLDER,
self.tr('R folder'),
RUtils.r_binary_folder(),
valuetype=Setting.FOLDER))
if RUtils.is_windows():
ProcessingConfig.addSetting(
Setting(self.name(), RUtils.R_USE64,
self.tr('Use 64 bit version'), False))
ProviderActions.registerProviderActions(self, self.actions)
ProviderContextMenuActions.registerProviderContextMenuActions(
self.contextMenuActions)
ProcessingConfig.readSettings()
self.refreshAlgorithms()
self.r_version = RUtils.get_r_version()
return True
def __init__(self, description_file, script=None):
super().__init__()
self.r_templates = RTemplates()
self.script = script
self._name = ''
self._display_name = ''
self._group = ''
self.description_file = os.path.realpath(
description_file) if description_file else None
self.error = None
self.commands = list()
self.is_user_script = False
if description_file:
self.is_user_script = not description_file.startswith(
RUtils.builtin_scripts_folder())
self.show_plots = False
self.pass_file_names = False
self.show_console_output = False
self.save_output_values = False
self.plots_filename = ''
self.output_values_filename = ''
self.results = {}
if self.script is not None:
self.load_from_string()
if self.description_file is not None:
self.load_from_file()
def process_metadata_line(self, line):
"""
Processes a "metadata" (##) line
"""
line = line.replace('#', '')
# special commands
if line.lower().strip().startswith('showplots'):
self.show_plots = True
self.addParameter(
QgsProcessingParameterFileDestination(
RAlgorithm.RPLOTS,
self.tr('R Plots'),
self.tr('HTML files (*.html)'),
optional=True))
return
if line.lower().strip().startswith('dontuserasterpackage'):
self.use_raster_package = False
return
if line.lower().strip().startswith('passfilenames'):
self.pass_file_names = True
return
value, type_ = self.split_tokens(line)
if type_.lower().strip() == 'group':
self._group = value
return
if type_.lower().strip() == 'name':
self._name = self._display_name = value
self._name = RUtils.strip_special_characters(self._name.lower())
return
self.process_parameter_line(line)
def saveScript(self, saveAs):
newPath = None
if self.filePath is None or saveAs:
scriptDir = RUtils.default_scripts_folder()
newPath, _ = QFileDialog.getSaveFileName(self,
self.tr("Save script"),
scriptDir,
self.tr("R scripts (*.rsx *.RSX)"))
if newPath:
if not newPath.lower().endswith(".rsx"):
newPath += ".rsx"
self.filePath = newPath
if self.filePath:
text = self.editor.text()
try:
with codecs.open(self.filePath, "w", encoding="utf-8") as f:
f.write(text)
except IOError as e:
QMessageBox.warning(self,
self.tr("I/O error"),
self.tr("Unable to save edits:\n{}").format(str(e))
)
return
self.setHasChanged(False)
QgsApplication.processingRegistry().providerById("r").refreshAlgorithms()
def processAlgorithm(self, parameters, context, feedback):
"""
Executes the algorithm
"""
self.results = {}
if RUtils.is_windows():
path = RUtils.r_binary_folder()
if path == '':
raise QgsProcessingException(
self.tr('R folder is not configured.\nPlease configure it '
'before running R scripts.'))
feedback.pushInfo(self.tr('R execution commands'))
output = RUtils.execute_r_algorithm(self, parameters, context,
feedback)
if self.show_plots:
html_filename = self.parameterAsFileOutput(parameters,
RAlgorithm.RPLOTS,
context)
if html_filename:
with open(html_filename, 'w') as f:
f.write('<html><img src="{}"/></html>'.format(
QUrl.fromLocalFile(self.plots_filename).toString()))
self.results[RAlgorithm.RPLOTS] = html_filename
if self.show_console_output:
html_filename = self.parameterAsFileOutput(
parameters, RAlgorithm.R_CONSOLE_OUTPUT, context)
if html_filename:
with open(html_filename, 'w') as f:
f.write(RUtils.html_formatted_console_output(output))
self.results[RAlgorithm.R_CONSOLE_OUTPUT] = html_filename
if self.save_output_values and self.output_values_filename:
with open(self.output_values_filename, 'r') as f:
lines = [line.strip() for line in f]
# get output values stored into the file
outputs = self.parse_output_values(iter(lines))
# merge output values into results
for k, v in outputs.items():
if k not in self.results:
self.results[k] = v
return self.results
def process_parameter_line(self, line):
"""
Processes a single script line representing a parameter
"""
value, _ = self.split_tokens(line)
description = RUtils.create_descriptive_name(value)
if not RUtils.is_valid_r_variable(value):
self.error = self.tr(
'This script has a syntax error in variable name.\n'
'"{1}" is not a valid variable name in R.'
'Problem with line: {0}').format(line, value)
output = create_output_from_string(line)
if output is not None:
output.setName(value)
output.setDescription(description)
if issubclass(output.__class__, QgsProcessingOutputDefinition):
self.addOutput(output)
self.save_output_values = True
else:
# destination type parameter
self.addParameter(output)
else:
line = RUtils.upgrade_parameter_line(line)
# this is necessary to remove the otherwise unknown keyword
line = line.replace("enum literal", "enum")
# this is annoying, but required to work around a bug in early 3.8.0 versions
try:
param = getParameterFromString(line, context="")
except TypeError:
param = getParameterFromString(line)
# set help parameter
if Qgis.QGIS_VERSION_INT >= 31600:
if self.descriptions is not None:
param.setHelp(self.descriptions.get(param.name()))
if param is not None:
self.addParameter(param)
else:
self.error = self.tr('This script has a syntax error.\n'
'Problem with line: {0}').format(line)
def loadAlgorithms(self):
"""
Called when provider must populate its available algorithms
"""
algs = []
for f in RUtils.script_folders():
algs.extend(self.load_scripts_from_folder(f))
for a in algs:
self.addAlgorithm(a)
def unload(self):
"""
Called when unloading provider
"""
ProcessingConfig.removeSetting(RUtils.RSCRIPTS_FOLDER)
ProcessingConfig.removeSetting(RUtils.R_LIBS_USER)
ProcessingConfig.removeSetting(RUtils.R_FOLDER)
if RUtils.is_windows():
ProcessingConfig.removeSetting(RUtils.R_USE64)
ProviderActions.deregisterProviderActions(self)
ProviderContextMenuActions.deregisterProviderContextMenuActions(self.contextMenuActions)
def canExecute(self):
"""
Returns True if the algorithm can be executed
"""
if self.error:
return False, self.error
msg = RUtils.check_r_is_installed()
if msg is not None:
return False, msg
return True, ''
def processAlgorithm(self, parameters, context, feedback):
"""
Executes the algorithm
"""
self.results = {}
if RUtils.is_windows():
path = RUtils.r_binary_folder()
if path == '':
raise QgsProcessingException(
self.tr('R folder is not configured.\nPlease configure it '
'before running R scripts.'))
feedback.pushInfo(self.tr('R execution commands'))
script_lines = self.build_r_script(parameters, context, feedback)
for line in script_lines:
feedback.pushCommandInfo(line)
output = RUtils.execute_r_algorithm(self, parameters, context,
feedback)
if self.show_plots:
html_filename = self.parameterAsFileOutput(parameters,
RAlgorithm.RPLOTS,
context)
if html_filename:
with open(html_filename, 'w') as f:
f.write('<html><img src="{}"/></html>'.format(
QUrl.fromLocalFile(self.plots_filename).toString()))
self.results[RAlgorithm.RPLOTS] = html_filename
if self.show_console_output:
html_filename = self.parameterAsFileOutput(
parameters, RAlgorithm.R_CONSOLE_OUTPUT, context)
if html_filename:
with open(html_filename, 'w') as f:
f.write(RUtils.html_formatted_console_output(output))
self.results[RAlgorithm.R_CONSOLE_OUTPUT] = html_filename
return self.results
def test_is_valid_r_variable(self):
"""
Test for strings to check if they are valid R variables.
"""
self.assertFalse(RUtils.is_valid_r_variable("var_name%"))
self.assertFalse(RUtils.is_valid_r_variable("2var_name"))
self.assertFalse(RUtils.is_valid_r_variable(".2var_name"))
self.assertFalse(RUtils.is_valid_r_variable("_var_name"))
self.assertTrue(RUtils.is_valid_r_variable("var_name2."))
self.assertTrue(RUtils.is_valid_r_variable(".var_name"))
self.assertTrue(RUtils.is_valid_r_variable("var.name"))
def process_parameter_line(self, line):
"""
Processes a single script line representing a parameter
"""
value, _ = self.split_tokens(line)
description = RUtils.create_descriptive_name(value)
if not RUtils.is_valid_r_variable(value):
self.add_error_message(
self.tr('This script has a syntax error in variable name.\n'
'"{1}" is not a valid variable name in R.'
'Problem with line: {0}').format(line, value))
output = create_output_from_string(line)
if output is not None:
output.setName(value)
output.setDescription(description)
if issubclass(output.__class__, QgsProcessingOutputDefinition):
self.addOutput(output)
self.save_output_values = True
else:
# destination type parameter
self.addParameter(output)
else:
param = create_parameter_from_string(line)
if param is not None:
# set help parameter
if Qgis.QGIS_VERSION_INT >= 31600:
if self.descriptions is not None:
param.setHelp(self.descriptions.get(param.name()))
self.addParameter(param)
else:
self.add_error_message(
self.tr('This script has a syntax error.\n'
'Problem with line: {0}').format(line))
def build_script_header_commands(self, script) -> List[str]:
"""
Builds the set of script startup commands for the algorithm, based on necessary packages,
github_install parameter and script analysis.
:param script: variable self.script from RAlgorithm
:return: list of str (commands)
"""
commands = []
# Just use main mirror
commands.append(self.set_option_repos(RUtils.package_repo()))
# Try to install packages if needed
if RUtils.use_user_library():
path_to_use = str(RUtils.r_library_folder()).replace('\\', '/')
commands.append(self.change_libPath(path_to_use))
packages = self.get_necessary_packages()
for p in packages:
commands.append(self.check_package_availability(p))
commands.append(self.load_package(p))
if self.install_github:
for dependency in self.github_dependencies:
commands.append(self.install_package_github(dependency))
packages_script = RUtils.get_required_packages(script)
for p in packages_script:
commands.append(self.check_package_availability(p))
commands.append(self.load_package(p))
return commands
def create_parameter_from_string(s: str):
"""
Tries to create an algorithm parameter from a line string
"""
if not ("|" in s and s.startswith("QgsProcessingParameter")):
s = RUtils.upgrade_parameter_line(s)
# this is necessary to remove the otherwise unknown keyword
s = s.replace("enum literal", "enum")
# this is annoying, but required to work around a bug in early 3.8.0 versions
try:
param = getParameterFromString(s, context="")
except TypeError:
param = getParameterFromString(s)
return param
def test_r_executable(self):
"""
Test retrieving R executable
"""
self.assertEqual(RUtils.path_to_r_executable(), 'R')
self.assertEqual(RUtils.path_to_r_executable(script_executable=True), 'Rscript')
ProcessingConfig.setSettingValue(RUtils.R_FOLDER, '/usr/local/bin')
self.assertEqual(RUtils.path_to_r_executable(), '/usr/local/bin/R')
self.assertEqual(RUtils.path_to_r_executable(script_executable=True), '/usr/local/bin/Rscript')
ProcessingConfig.setSettingValue(RUtils.R_FOLDER, None)
self.assertEqual(RUtils.path_to_r_executable(), 'R')
self.assertEqual(RUtils.path_to_r_executable(script_executable=True), 'Rscript')
def openScript(self):
if self.hasChanged:
ret = QMessageBox.warning(self,
self.tr("Unsaved changes"),
self.tr("There are unsaved changes in the script. Continue?"),
QMessageBox.Yes | QMessageBox.No, QMessageBox.No)
if ret == QMessageBox.No:
return
scriptDir = RUtils.default_scripts_folder()
fileName, _ = QFileDialog.getOpenFileName(self,
self.tr("Open script"),
scriptDir,
self.tr("R scripts (*.rsx *.RSX)"))
if fileName == "":
return
with OverrideCursor(Qt.WaitCursor):
self._loadFile(fileName)
def test_guess_r_binary_folder(self):
"""
Test guessing the R binary folder -- not much to do here, all the logic is Windows specific
"""
self.assertFalse(RUtils.guess_r_binary_folder())
def test_is_macos(self):
"""
Test is_macos
"""
self.assertFalse(RUtils.is_macos()) # suck it even more, MacOS users!
