def testDefaultScriptsFolder(self): """ Tests default user scripts folder """ self.assertTrue(RUtils.default_scripts_folder()) self.assertIn('rscripts', RUtils.default_scripts_folder()) self.assertTrue(os.path.exists(RUtils.default_scripts_folder()))
def testBuiltInPath(self): """ Tests built in scripts path """ self.assertTrue(RUtils.builtin_scripts_folder()) self.assertIn('builtin_scripts', RUtils.builtin_scripts_folder()) self.assertTrue(os.path.exists(RUtils.builtin_scripts_folder()))
def testScriptsFolders(self): """ Test script folders """ self.assertTrue(RUtils.script_folders()) self.assertIn(RUtils.default_scripts_folder(), RUtils.script_folders()) self.assertIn(RUtils.builtin_scripts_folder(), RUtils.script_folders())
def build_script_header_commands(self, _, __, ___): """ Builds the set of script startup commands for the algorithm """ commands = list() # Just use main mirror commands.append('options("repos"="{}")'.format(RUtils.package_repo())) # Try to install packages if needed if RUtils.use_user_library(): commands.append('.libPaths(\"' + str(RUtils.r_library_folder()).replace('\\', '/') + '\")') packages = RUtils.get_required_packages(self.script) packages.extend(['rgdal', 'raster']) for p in packages: commands.append('tryCatch(find.package("' + p + '"), error=function(e) install.packages("' + p + '", dependencies=TRUE))') commands.append('library("raster")') commands.append('library("rgdal")') return commands
def process_metadata_line(self, line): # pylint: disable=too-many-return-statements """ Processes a "metadata" (##) line """ line = line.replace('#', '') # special commands # showplots is the older version, should be considere obsolete if line.lower().strip().startswith('output_plots_to_html') or \ line.lower().strip().startswith('showplots'): self.show_plots = True self.addParameter( QgsProcessingParameterFileDestination( RAlgorithm.RPLOTS, self.tr('R Plots'), self.tr('HTML files (*.html)'), optional=True)) return # dontuserasterpackage is the older version, should be considere obsolete if line.lower().strip().startswith('load_raster_using_rgdal') or \ line.lower().strip().startswith('dontuserasterpackage'): self.r_templates.use_raster = False return if line.lower().strip().startswith('load_vector_using_rgdal'): self.r_templates.use_sf = False return # passfilenames is the older version, should be considere obsolete if line.lower().strip().startswith('pass_filenames') or\ line.lower().strip().startswith('passfilenames'): self.pass_file_names = True return if line.lower().strip().startswith('dont_load_any_packages'): self.r_templates.auto_load_packages = False return value, type_ = self.split_tokens(line) if type_.lower().strip() == 'group': self._group = value return if type_.lower().strip() == 'name': self._name = self._display_name = value self._name = RUtils.strip_special_characters(self._name.lower()) return if type_.lower().strip() == 'display_name': self._display_name = value return if type_.lower().strip() == 'github_install': self.r_templates.install_github = True self.r_templates.github_dependencies = value return # process enum with values and preparing its template if "=enum literal" in RUtils.upgrade_parameter_line(line): self.r_templates.add_literal_enum(value) self.process_parameter_line(line)
def process_parameter_line(self, line): """ Processes a single script line representing a parameter """ value, _ = self.split_tokens(line) description = RUtils.create_descriptive_name(value) output = create_output_from_string(line) if output is not None: output.setName(value) output.setDescription(description) if issubclass(output.__class__, QgsProcessingOutputDefinition): self.addOutput(output) else: # destination type parameter self.addParameter(output) else: line = RUtils.upgrade_parameter_line(line) # this is annoying, but required to work around a bug in early 3.8.0 versions try: param = getParameterFromString(line, context="") except TypeError: param = getParameterFromString(line) if param is not None: self.addParameter(param) else: self.error = self.tr('This script has a syntax error.\n' 'Problem with line: {0}').format(line)
def test_is_error_line(self): """ Test is_error_line """ self.assertFalse(RUtils.is_error_line('xxx yyy')) self.assertTrue(RUtils.is_error_line('Error something went wrong')) self.assertTrue(RUtils.is_error_line('Execution halted'))
def test_use_user_library(self): """ Test retrieving/setting the user library setting """ self.assertTrue(RUtils.use_user_library()) ProcessingConfig.setSettingValue(RUtils.R_USE_USER_LIB, False) self.assertFalse(RUtils.use_user_library()) ProcessingConfig.setSettingValue(RUtils.R_USE_USER_LIB, True) self.assertTrue(RUtils.use_user_library())
def test_package_repo(self): """ Test retrieving/setting the package repo """ self.assertEqual(RUtils.package_repo(), 'http://cran.at.r-project.org/') ProcessingConfig.setSettingValue(RUtils.R_REPO, 'http://mirror.at.r-project.org/') self.assertEqual(RUtils.package_repo(), 'http://mirror.at.r-project.org/') ProcessingConfig.setSettingValue(RUtils.R_REPO, 'http://cran.at.r-project.org/') self.assertEqual(RUtils.package_repo(), 'http://cran.at.r-project.org/')
def test_r_binary_folder(self): """ Test retrieving R binary folder """ self.assertFalse(RUtils.r_binary_folder()) ProcessingConfig.setSettingValue(RUtils.R_FOLDER, '/usr/local/bin') self.assertEqual(RUtils.r_binary_folder(), '/usr/local/bin') ProcessingConfig.setSettingValue(RUtils.R_FOLDER, None) self.assertFalse(RUtils.r_binary_folder())
def test_library_folder(self): """ Test retrieving/setting the library folder """ self.assertIn('/profiles/default/processing/rlibs', RUtils.r_library_folder()) ProcessingConfig.setSettingValue(RUtils.R_LIBS_USER, '/usr/local') self.assertEqual(RUtils.r_library_folder(), '/usr/local') ProcessingConfig.setSettingValue(RUtils.R_LIBS_USER, None) self.assertIn('/profiles/default/processing/rlibs', RUtils.r_library_folder())
def test_r_is_installed(self): """ Test checking that R is installed """ self.assertIsNone(RUtils.check_r_is_installed()) ProcessingConfig.setSettingValue(RUtils.R_FOLDER, '/home') self.assertTrue(RUtils.check_r_is_installed()) self.assertIn('R is not installed', RUtils.check_r_is_installed()) ProcessingConfig.setSettingValue(RUtils.R_FOLDER, None) self.assertIsNone(RUtils.check_r_is_installed())
def load(self): """ Called when first loading provider """ ProcessingConfig.settingIcons[self.name()] = self.icon() ProcessingConfig.addSetting( Setting(self.name(), RUtils.RSCRIPTS_FOLDER, self.tr('R scripts folder'), RUtils.default_scripts_folder(), valuetype=Setting.MULTIPLE_FOLDERS)) ProcessingConfig.addSetting( Setting( self.name(), RUtils.R_USE_USER_LIB, self.tr('Use user library folder instead of system libraries'), True)) ProcessingConfig.addSetting( Setting(self.name(), RUtils.R_LIBS_USER, self.tr('User library folder'), RUtils.r_library_folder(), valuetype=Setting.FOLDER)) ProcessingConfig.addSetting( Setting(self.name(), RUtils.R_REPO, self.tr('Package repository'), "http://cran.at.r-project.org/", valuetype=Setting.STRING)) ProcessingConfig.addSetting( Setting(self.name(), RUtils.R_FOLDER, self.tr('R folder'), RUtils.r_binary_folder(), valuetype=Setting.FOLDER)) if RUtils.is_windows(): ProcessingConfig.addSetting( Setting(self.name(), RUtils.R_USE64, self.tr('Use 64 bit version'), False)) ProviderActions.registerProviderActions(self, self.actions) ProviderContextMenuActions.registerProviderContextMenuActions( self.contextMenuActions) ProcessingConfig.readSettings() self.refreshAlgorithms() self.r_version = RUtils.get_r_version() return True
def __init__(self, description_file, script=None): super().__init__() self.r_templates = RTemplates() self.script = script self._name = '' self._display_name = '' self._group = '' self.description_file = os.path.realpath( description_file) if description_file else None self.error = None self.commands = list() self.is_user_script = False if description_file: self.is_user_script = not description_file.startswith( RUtils.builtin_scripts_folder()) self.show_plots = False self.pass_file_names = False self.show_console_output = False self.save_output_values = False self.plots_filename = '' self.output_values_filename = '' self.results = {} if self.script is not None: self.load_from_string() if self.description_file is not None: self.load_from_file()
def process_metadata_line(self, line): """ Processes a "metadata" (##) line """ line = line.replace('#', '') # special commands if line.lower().strip().startswith('showplots'): self.show_plots = True self.addParameter( QgsProcessingParameterFileDestination( RAlgorithm.RPLOTS, self.tr('R Plots'), self.tr('HTML files (*.html)'), optional=True)) return if line.lower().strip().startswith('dontuserasterpackage'): self.use_raster_package = False return if line.lower().strip().startswith('passfilenames'): self.pass_file_names = True return value, type_ = self.split_tokens(line) if type_.lower().strip() == 'group': self._group = value return if type_.lower().strip() == 'name': self._name = self._display_name = value self._name = RUtils.strip_special_characters(self._name.lower()) return self.process_parameter_line(line)
def saveScript(self, saveAs): newPath = None if self.filePath is None or saveAs: scriptDir = RUtils.default_scripts_folder() newPath, _ = QFileDialog.getSaveFileName(self, self.tr("Save script"), scriptDir, self.tr("R scripts (*.rsx *.RSX)")) if newPath: if not newPath.lower().endswith(".rsx"): newPath += ".rsx" self.filePath = newPath if self.filePath: text = self.editor.text() try: with codecs.open(self.filePath, "w", encoding="utf-8") as f: f.write(text) except IOError as e: QMessageBox.warning(self, self.tr("I/O error"), self.tr("Unable to save edits:\n{}").format(str(e)) ) return self.setHasChanged(False) QgsApplication.processingRegistry().providerById("r").refreshAlgorithms()
def processAlgorithm(self, parameters, context, feedback): """ Executes the algorithm """ self.results = {} if RUtils.is_windows(): path = RUtils.r_binary_folder() if path == '': raise QgsProcessingException( self.tr('R folder is not configured.\nPlease configure it ' 'before running R scripts.')) feedback.pushInfo(self.tr('R execution commands')) output = RUtils.execute_r_algorithm(self, parameters, context, feedback) if self.show_plots: html_filename = self.parameterAsFileOutput(parameters, RAlgorithm.RPLOTS, context) if html_filename: with open(html_filename, 'w') as f: f.write('<html><img src="{}"/></html>'.format( QUrl.fromLocalFile(self.plots_filename).toString())) self.results[RAlgorithm.RPLOTS] = html_filename if self.show_console_output: html_filename = self.parameterAsFileOutput( parameters, RAlgorithm.R_CONSOLE_OUTPUT, context) if html_filename: with open(html_filename, 'w') as f: f.write(RUtils.html_formatted_console_output(output)) self.results[RAlgorithm.R_CONSOLE_OUTPUT] = html_filename if self.save_output_values and self.output_values_filename: with open(self.output_values_filename, 'r') as f: lines = [line.strip() for line in f] # get output values stored into the file outputs = self.parse_output_values(iter(lines)) # merge output values into results for k, v in outputs.items(): if k not in self.results: self.results[k] = v return self.results
def process_parameter_line(self, line): """ Processes a single script line representing a parameter """ value, _ = self.split_tokens(line) description = RUtils.create_descriptive_name(value) if not RUtils.is_valid_r_variable(value): self.error = self.tr( 'This script has a syntax error in variable name.\n' '"{1}" is not a valid variable name in R.' 'Problem with line: {0}').format(line, value) output = create_output_from_string(line) if output is not None: output.setName(value) output.setDescription(description) if issubclass(output.__class__, QgsProcessingOutputDefinition): self.addOutput(output) self.save_output_values = True else: # destination type parameter self.addParameter(output) else: line = RUtils.upgrade_parameter_line(line) # this is necessary to remove the otherwise unknown keyword line = line.replace("enum literal", "enum") # this is annoying, but required to work around a bug in early 3.8.0 versions try: param = getParameterFromString(line, context="") except TypeError: param = getParameterFromString(line) # set help parameter if Qgis.QGIS_VERSION_INT >= 31600: if self.descriptions is not None: param.setHelp(self.descriptions.get(param.name())) if param is not None: self.addParameter(param) else: self.error = self.tr('This script has a syntax error.\n' 'Problem with line: {0}').format(line)
def loadAlgorithms(self): """ Called when provider must populate its available algorithms """ algs = [] for f in RUtils.script_folders(): algs.extend(self.load_scripts_from_folder(f)) for a in algs: self.addAlgorithm(a)
def unload(self): """ Called when unloading provider """ ProcessingConfig.removeSetting(RUtils.RSCRIPTS_FOLDER) ProcessingConfig.removeSetting(RUtils.R_LIBS_USER) ProcessingConfig.removeSetting(RUtils.R_FOLDER) if RUtils.is_windows(): ProcessingConfig.removeSetting(RUtils.R_USE64) ProviderActions.deregisterProviderActions(self) ProviderContextMenuActions.deregisterProviderContextMenuActions(self.contextMenuActions)
def canExecute(self): """ Returns True if the algorithm can be executed """ if self.error: return False, self.error msg = RUtils.check_r_is_installed() if msg is not None: return False, msg return True, ''
def processAlgorithm(self, parameters, context, feedback): """ Executes the algorithm """ self.results = {} if RUtils.is_windows(): path = RUtils.r_binary_folder() if path == '': raise QgsProcessingException( self.tr('R folder is not configured.\nPlease configure it ' 'before running R scripts.')) feedback.pushInfo(self.tr('R execution commands')) script_lines = self.build_r_script(parameters, context, feedback) for line in script_lines: feedback.pushCommandInfo(line) output = RUtils.execute_r_algorithm(self, parameters, context, feedback) if self.show_plots: html_filename = self.parameterAsFileOutput(parameters, RAlgorithm.RPLOTS, context) if html_filename: with open(html_filename, 'w') as f: f.write('<html><img src="{}"/></html>'.format( QUrl.fromLocalFile(self.plots_filename).toString())) self.results[RAlgorithm.RPLOTS] = html_filename if self.show_console_output: html_filename = self.parameterAsFileOutput( parameters, RAlgorithm.R_CONSOLE_OUTPUT, context) if html_filename: with open(html_filename, 'w') as f: f.write(RUtils.html_formatted_console_output(output)) self.results[RAlgorithm.R_CONSOLE_OUTPUT] = html_filename return self.results
def test_is_valid_r_variable(self): """ Test for strings to check if they are valid R variables. """ self.assertFalse(RUtils.is_valid_r_variable("var_name%")) self.assertFalse(RUtils.is_valid_r_variable("2var_name")) self.assertFalse(RUtils.is_valid_r_variable(".2var_name")) self.assertFalse(RUtils.is_valid_r_variable("_var_name")) self.assertTrue(RUtils.is_valid_r_variable("var_name2.")) self.assertTrue(RUtils.is_valid_r_variable(".var_name")) self.assertTrue(RUtils.is_valid_r_variable("var.name"))
def process_parameter_line(self, line): """ Processes a single script line representing a parameter """ value, _ = self.split_tokens(line) description = RUtils.create_descriptive_name(value) if not RUtils.is_valid_r_variable(value): self.add_error_message( self.tr('This script has a syntax error in variable name.\n' '"{1}" is not a valid variable name in R.' 'Problem with line: {0}').format(line, value)) output = create_output_from_string(line) if output is not None: output.setName(value) output.setDescription(description) if issubclass(output.__class__, QgsProcessingOutputDefinition): self.addOutput(output) self.save_output_values = True else: # destination type parameter self.addParameter(output) else: param = create_parameter_from_string(line) if param is not None: # set help parameter if Qgis.QGIS_VERSION_INT >= 31600: if self.descriptions is not None: param.setHelp(self.descriptions.get(param.name())) self.addParameter(param) else: self.add_error_message( self.tr('This script has a syntax error.\n' 'Problem with line: {0}').format(line))
def build_script_header_commands(self, script) -> List[str]: """ Builds the set of script startup commands for the algorithm, based on necessary packages, github_install parameter and script analysis. :param script: variable self.script from RAlgorithm :return: list of str (commands) """ commands = [] # Just use main mirror commands.append(self.set_option_repos(RUtils.package_repo())) # Try to install packages if needed if RUtils.use_user_library(): path_to_use = str(RUtils.r_library_folder()).replace('\\', '/') commands.append(self.change_libPath(path_to_use)) packages = self.get_necessary_packages() for p in packages: commands.append(self.check_package_availability(p)) commands.append(self.load_package(p)) if self.install_github: for dependency in self.github_dependencies: commands.append(self.install_package_github(dependency)) packages_script = RUtils.get_required_packages(script) for p in packages_script: commands.append(self.check_package_availability(p)) commands.append(self.load_package(p)) return commands
def create_parameter_from_string(s: str): """ Tries to create an algorithm parameter from a line string """ if not ("|" in s and s.startswith("QgsProcessingParameter")): s = RUtils.upgrade_parameter_line(s) # this is necessary to remove the otherwise unknown keyword s = s.replace("enum literal", "enum") # this is annoying, but required to work around a bug in early 3.8.0 versions try: param = getParameterFromString(s, context="") except TypeError: param = getParameterFromString(s) return param
def test_r_executable(self): """ Test retrieving R executable """ self.assertEqual(RUtils.path_to_r_executable(), 'R') self.assertEqual(RUtils.path_to_r_executable(script_executable=True), 'Rscript') ProcessingConfig.setSettingValue(RUtils.R_FOLDER, '/usr/local/bin') self.assertEqual(RUtils.path_to_r_executable(), '/usr/local/bin/R') self.assertEqual(RUtils.path_to_r_executable(script_executable=True), '/usr/local/bin/Rscript') ProcessingConfig.setSettingValue(RUtils.R_FOLDER, None) self.assertEqual(RUtils.path_to_r_executable(), 'R') self.assertEqual(RUtils.path_to_r_executable(script_executable=True), 'Rscript')
def openScript(self): if self.hasChanged: ret = QMessageBox.warning(self, self.tr("Unsaved changes"), self.tr("There are unsaved changes in the script. Continue?"), QMessageBox.Yes | QMessageBox.No, QMessageBox.No) if ret == QMessageBox.No: return scriptDir = RUtils.default_scripts_folder() fileName, _ = QFileDialog.getOpenFileName(self, self.tr("Open script"), scriptDir, self.tr("R scripts (*.rsx *.RSX)")) if fileName == "": return with OverrideCursor(Qt.WaitCursor): self._loadFile(fileName)
def test_guess_r_binary_folder(self): """ Test guessing the R binary folder -- not much to do here, all the logic is Windows specific """ self.assertFalse(RUtils.guess_r_binary_folder())
def test_is_macos(self): """ Test is_macos """ self.assertFalse(RUtils.is_macos()) # suck it even more, MacOS users!