def run_command(self): """ Run the command. Open a file to receive regression test results. Returns: None. Raises: RuntimeError: if a runtime input error occurs. """ warning_count = 0 logger = logging.getLogger(__name__) # Get data for the command pv_OutputFile = self.get_parameter_value('OutputFile') # Runtime checks on input pv_OutputFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_OutputFile, self))) if warning_count > 0: message = "There were " + str(warning_count) + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Open the regression test results file try: pv_OutputFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_OutputFile, self))) self.__open_new_regression_test_report_file(pv_OutputFile_absolute, False) # Do not append logger.info('Opened regression test results report file "' + pv_OutputFile_absolute + '"') except Exception as e: warning_count += 1 message = 'Unexpected error opening file "' + pv_OutputFile_absolute + '"' logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) if warning_count > 0: message = "There were " + str(warning_count) + " warnings processing the command." logger.warning(message) raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_batch(command_file, runtime_properties): """ Run in batch mode by processing the specific command file. Args: command_file (str): The name of the command file to run, absolute path or relative to the current folder. runtime_properties (dict): A dictionary of properties for the processor. Returns: None. """ logger = logging.getLogger(__name__) print('Running command file: ' + command_file) working_dir = os.getcwd() # Convert the command file to an absolute path if not already. logger.info("Working directory=" + working_dir) logger.info("Command file=" + command_file) command_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(working_dir, command_file)) logger.info("Command file (absolute)=" + command_file_absolute) # from geoprocessor.core.CommandFileRunner import CommandFileRunner CommandFileRunner = importlib.import_module( 'geoprocessor.core.CommandFileRunner') class_ = getattr(CommandFileRunner, 'CommandFileRunner') runner = class_() # Read the command file try: runner.read_command_file(command_file_absolute) except IOError as e: # Would be nice to use FileNotFoundError but that is Python 3.3+ message = 'Error: Command file "' + command_file_absolute + '" was not found.' print(message) logger.error(message, exc_info=True) return except Exception as e2: message = 'Error reading command file "' + command_file_absolute + '".' logger.error(message, exc_info=True) print(message) return # Run the command file try: # Pass the runtime properties to supplement default properties and those created in the command file runner.run_commands(env_properties=runtime_properties) logger.info("At end of gp.run_batch") except Exception as e: message = 'Error running command file.' print(message) logger.error(message, exc_info=True) return finally: StartRegressionTestResultsReport.close_regression_test_report_file() logger.info("GeoProcessor properties after running:") for property_name, property_value in runner.get_processor( ).properties.items(): logger.info(property_name + " = " + str(property_value)) print("See log file for more information.")
def run_command(self): """ Run the command. Write the GeoLayer to a spatial data file in KML format. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. pv_GeoLayerID = self.get_parameter_value("GeoLayerID") pv_OutputFile = self.get_parameter_value("OutputFile") pv_PlacemarkNameAttribute = self.get_parameter_value("PlacemarkNameAttribute") pv_PlacemarkDescriptionAttribute = self.get_parameter_value("PlacemarkDescriptionAttribute") # Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax output_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_OutputFile, self))) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_write_geolayer(pv_GeoLayerID, output_file_absolute): try: # Get the GeoLayer geolayer = self.command_processor.get_geolayer(pv_GeoLayerID) # Write the GeoLayer to a spatial data file in KML format # "Note that KML by specification uses only a single projection, EPSG:4326. All OGR KML output will be # presented in EPSG:4326. As such OGR will create layers in the correct coordinate system and transform # any geometries." - www.gdal.org/drv_kml.html qgis_util.write_qgsvectorlayer_to_kml(geolayer.qgs_vector_layer, output_file_absolute, "EPSG:4326", pv_PlacemarkNameAttribute, pv_PlacemarkDescriptionAttribute, "clampToGround") # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error writing GeoLayer {} to GeoJSON format.".format(pv_GeoLayerID) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log(CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format(self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def do_run(cls, line): """ Run the command file from the command line using syntax: run command-file """ if line is None: # Put this here to get rid of PyCharm warning about line not being used pass logger = logging.getLogger(__name__) command_file = line if command_file is None: print('Error: no command file specified for "run" command.') print('Running command file: ' + command_file) working_dir = os.getcwd() # Convert the command file to an absolute path if not already. command_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(working_dir, command_file)) # Make sure the file exists so ugly exception is not printed if not os.path.isfile(command_file_absolute): print("File does not exist, cannot run: " + command_file_absolute) return # from geoprocessor.core.CommandFileRunner import CommandFileRunner CommandFileRunner = importlib.import_module( 'geoprocessor.core.CommandFileRunner') class_ = getattr(CommandFileRunner, 'CommandFileRunner') runner = class_() # Read the command file try: runner.read_command_file(command_file_absolute) except IOError as e: # Would be nice to use FileNotFoundError but that is Python 3.3+ message = 'Error: Command file "' + command_file_absolute + '" was not found.' print(message) logger.error(message, exc_info=True) except Exception as e2: message = 'Error reading command file.' logger.error(message, exc_info=True) print(message) # Run the command file try: runner.run_commands() except Exception as e: message = 'Error running command file.' print(message) logger.error(message, exc_info=True) logger.info("GeoProcessor properties after running:") for property_name, property_value in runner.get_processor( ).properties.items(): logger.info(property_name + " = " + str(property_value)) print("See log file for more information.")
def run_command(self): """ Run the command. Restart the lof file with the given name. Subsequent logging messages will go to the file until another StartLog() command is run. Returns: Nothing. Raises: RuntimeError if any exception occurs running the command. """ logger = logging.getLogger(__name__) warning_count = 0 pv_LogFile = self.get_parameter_value('LogFile') log_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_LogFile))) try: # Change the GeoProcessor logger to use the specified file # - The initial application log file will be closed. log_util.reset_log_file_handler(log_file_absolute) # Add the log file to output self.command_processor.add_output_file(log_file_absolute) except Exception as e: warning_count += 1 message = 'Unexpected error (re)starting log file "' + log_file_absolute + '"' logger.exception(message, e) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "Check the old log file or command window for details.")) if warning_count > 0: message = "There were " + str(warning_count) + " warnings processing the command." raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Returns: Nothing. Raises: ValueError: if a runtime input error occurs. """ warning_count = 0 logger = logging.getLogger(__name__) # Get data for the command pv_SearchFolder = self.get_parameter_value('SearchFolder') pv_FilenamePattern = self.get_parameter_value('FilenamePattern') if pv_FilenamePattern is None or pv_FilenamePattern == "": # The pattern that is desired is test_*.gp, using globbing syntax pv_FilenamePattern = "[Tt][Ee][Ss][Tt]-*.gp" pv_OutputFile = self.get_parameter_value('OutputFile') # Runtime checks on input working_dir = self.command_processor.get_property('WorkingDir') logger.info('working_dir: "' + working_dir + '"') search_folder_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(working_dir, self.command_processor.expand_parameter_value(pv_SearchFolder, self))) search_folder_absolute_internal = io_util.verify_path_for_os(search_folder_absolute, always_use_forward_slashes=True) output_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(working_dir, self.command_processor.expand_parameter_value(pv_OutputFile, self))) output_file_absolute_internal = io_util.verify_path_for_os(output_file_absolute, always_use_forward_slashes=True) logger.info('search_folder_absolute: "' + search_folder_absolute + '"') logger.info('search_folder_absolute_internal: "' + search_folder_absolute_internal + '"') if not os.path.exists(search_folder_absolute_internal): message = 'The folder to search "' + search_folder_absolute + '" does not exist.' logger.warning(message) warning_count += 1 self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "Verify that the folder exists at the time the command is run.")) if warning_count > 0: message = "There were " + str(warning_count) + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Do the processing try: # Output folder is used when constructing filenames for command files to run output_folder_absolute = os.path.dirname(output_file_absolute_internal) logger.info('output_folder_absolute: "' + output_folder_absolute + '"') logger.info('output_folder_absolute_internal: "' + output_file_absolute_internal + '"') files = [] # List of files to match # Convert globbing-style wildcards to Pythonic regex logger.info('Filename pattern using globbing = "' + pv_FilenamePattern + '"') # filename_pattern_regex = pv_FilenamePattern.replace("*", "[^/]*") # Match any character of any length, making sure that special characters are dealt with filename_pattern_regex = pv_FilenamePattern.replace(".", "\\.") # So .gp is handled literally # The following is used to match between "test-" and ".gp" filename_pattern_regex = filename_pattern_regex.replace("*", ".*") logger.info('Filename pattern using Python regex = "' + filename_pattern_regex + '"') filename_pattern_regex_compiled = re.compile(filename_pattern_regex) CreateRegressionTestCommandFile.__get_matching_filenames_in_tree(files, search_folder_absolute_internal, filename_pattern_regex_compiled, pattern_string=filename_pattern_regex) # Sort the list files = sorted(files) # Open the output file... # TODO smalers 2018-10-20 decide whether to support append mode out = open(output_file_absolute_internal, "w") # Write a standard header to the file so that it is clear when the file was created io_util.print_standard_file_header(out, comment_line_prefix="#") # Include the setup command file if requested # logger.info('Adding commands from setup command file "' + setup_command_file_absolute + '"') # include_setup_file(out, setup_file_absolute, "setup") # Include the matching test cases # Python 2... # nl = os.linesep # newline character for operating system # Python 3 translates \n into the OS-specific end of line so os.linesep introduces extra end of lines nl = "\n" out.write("#" + nl) out.write("# The following " + str(len(files)) + " test cases will be run to compare results with expected results." + nl) out.write("# Individual log files are generally created for each test." + nl) # TODO smalers 2018-01-20 evaluate how to handle output table # - Currently the GeoProcessor does not have table commands table_param = "" # Use absolute path since each developer will regenerate this file. out.write('StartRegressionTestResultsReport(OutputFile="' + output_file_absolute + '.out.txt"' + table_param + ")" + nl) # Find the list of matching files... # logger.debug('output_folder_absolute"' + output_folder_absolute + '"') for a_file in files: logger.info('Adding command file "' + a_file + '"') # The command files to run are relative to the commands file being created. # - use the operating system separator command_file_to_run = io_util.verify_path_for_os( io_util.to_relative_path(output_folder_absolute, a_file)) # Determine if the command file has @expectedStatus in it. If so, define an ExpectedStatus # parameter for the command. logger.info('Relative path to command file is "' + command_file_to_run + '"') out.write('RunCommands(CommandFile="' + command_file_to_run + '"' + CreateRegressionTestCommandFile.__determine_expected_status_parameter(a_file) + ')' + nl) # TODO smalers 2018-12-30 Maybe the output file is relative to the working folder # output_file_relative = io_util.to_relative_path(working_dir, output_file_absolute) # out.write('WriteCommandSummaryToFile(OutputFile="' + output_file_relative + '.summary.html")' + nl) out.write('WriteCommandSummaryToFile(OutputFile="' + output_file_absolute + '.summary.html")' + nl) # TODO smalers 2018-01-28 evaluate whether to support end content # Include the end command file if requested # Message.printStatus ( 2, routine, "Adding commands from end command file \"" + EndCommandFile_full + "\"") # includeCommandFile ( out, EndCommandFile_full, "end" ); out.close() # Add the log file to output self.command_processor.add_output_file(output_file_absolute) except Exception as e: warning_count += 1 message = 'Unexpected error creating regression test command file "' + output_file_absolute + '"' traceback.print_exc(file=sys.stdout) logger.exception(message, e) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) if warning_count > 0: message = "There were " + str(warning_count) + " warnings processing the command." raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Create a file summarizing command log messages, status, etc. Returns: None. Raises: RuntimeError: if a runtime input error occurs. """ warning_count = 0 logger = logging.getLogger(__name__) # Get data for the command pv_OutputFile = self.get_parameter_value('OutputFile') # Runtime checks on input pv_OutputFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_OutputFile, self))) if warning_count > 0: message = "There were " + str( warning_count) + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Create the command summary file try: pv_OutputFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_OutputFile, self))) # Open the file and write command summary logger.info('Writing summary to "' + pv_OutputFile_absolute + '"') fp = open(pv_OutputFile_absolute, "w") self.write_file_header(fp) self.write_command_summary(fp) self.write_file_footer(fp) # Close the file fp.close() except Exception as e: warning_count += 1 message = 'Unexpected error writing file "' + pv_OutputFile_absolute + '"' logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) except: warning_count += 1 message = 'Unexpected error writing file "' + pv_OutputFile_absolute + '"' logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) if warning_count > 0: message = "There were " + str( warning_count) + " warnings processing the command." logger.warning(message) raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Run a separate command file and save results to the current processor. Returns: None. Raises: ValueError: if a runtime input error occurs. RuntimeError: if a runtime error occurs. """ # The following import is deferred until runtime because if included at the top of the module # it causes a circular dependency and the GeoProcessor won't load from geoprocessor.core.CommandFileRunner import CommandFileRunner warning_count = 0 logger = logging.getLogger(__name__) # Get data for the command pv_CommandFile = self.get_parameter_value('CommandFile') pv_ExpectedStatus = self.get_parameter_value('ExpectedStatus') expected_status = pv_ExpectedStatus if pv_ExpectedStatus == "": pv_ExpectedStatus = None # Default - was not specified in the command # Runtime checks on input pv_CommandFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_CommandFile, self))) if warning_count > 0: message = "There were " + str(warning_count) + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Write the output file try: command_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_CommandFile, self))) logger.info('Processing commands from file "' + command_file_absolute + '" using command file runner.') runner = CommandFileRunner() # This will set the initial working directory of the runner to that of the command file... file_found = True try: runner.read_command_file(command_file_absolute) except FileNotFoundError as e: warning_count += 1 message = 'File does not exist: "' + command_file_absolute + '"' self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "Confirm that the command file exists.")) # Set the following to skip code below file_found = False # If the command file is not enabled, don't need to initialize or process # TODO SAM 2013-04-20 Even if disabled, will still run discovery above # - need to disable discovery in this case is_enabled = runner.is_command_file_enabled() expected_status = str(CommandStatusType.SUCCESS) if pv_ExpectedStatus is not None: expected_status = pv_ExpectedStatus if not file_found: # Need to add logic to indicate a failed test self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord( CommandStatusType.FAILURE, "Command file does not exist.", "Confirm that the command file exists.")) # Set the results to fail test_pass_fail = self.__FAIL elif is_enabled: # TODO smalers, 2018-01-26 Java code set datastores here # TODO SAM 2010-09-30 Need to evaluate how to share properties - issue is that built-in properties are # handled explicitly whereas user-defined properties are in a list that can be easily shared. # Also, some properties like the working directory receive special treatment. # For now don't bite off the property issue runner.run_commands(env_properties=self.command_processor.env_properties) logger.info("Done running commands") # Total runtime for the commands # long run_time_total = TSCommandProcessorUtil.getRunTimeTotal(runner.getProcessor().getCommands()); # Set the CommandStatus for this command to the most severe status of the # commands file that was just run. max_severity = command_util.get_command_status_max_severity(runner.command_processor) logger.info("Max severity from commands = " + str(max_severity)) test_pass_fail = "????" # Status for the test, which is not always the same as max_severity if pv_ExpectedStatus is not None: expected_status_type = CommandStatusType.value_of(expected_status) if max_severity is expected_status_type: # Expected status matches the actual so consider this a success. # This should generally be used only when running a test that we expect to fail (e.g., run # obsolete command or testing handling of errors). self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.SUCCESS, "Severity for RunCommands (" + str(max_severity) + ") is max of commands in command file that was run - matches expected (" + expected_status + ") so RunCommands status=Success.", "Additional status messages are omitted to allow test to be success - " + "refer to log file if warning/failure.")) # TODO SAM 2008-07-09 Need to evaluate how to append all the log messages but still # have a successful status that shows in the displays. # DO NOT append the messages from the command because their status will cause the # error displays to show problem indicators. test_pass_fail = self.__PASS else: # Expected status and it does NOT match the actual status so this is a failure. self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.SUCCESS, "Severity for RunCommands (" + str(max_severity) + ") is max of commands in command file that was run - " + "does not match expected (" + expected_status + ") so RunCommands status=Failure.", "Check the command to confirm the expected status.")) # TODO SAM 2008-07-09 Need to evaluate how to append all the log messages but still # have a successful status that shows in the displays. # DO NOT append the messages from the command because their status will cause the # error displays to show problem indicators. test_pass_fail = self.__FAIL else: # TODO smalers 2018-01-28 evaluate whether this is needed given that success is default expected # status # Expected status is not specified self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord( max_severity, "Severity for RunCommands (" + str(max_severity) + ") is max of commands in command file that was run.", "Status messages from commands that were run are appended to RunCommand status messages.")) # Append the log records from the command file that was run. # The status contains lists of CommandLogRecord for each run mode. # For RunCommands() the log messages should be associated with the originating command, # not this RunCommand command logger.info("Appending log records") command_util.append_command_status_log_records( self.command_status, runner.command_processor.commands) if max_severity.value >= CommandStatusType.WARNING.value: test_pass_fail = self.__FAIL else: test_pass_fail = self.__PASS # Add a record to the regression test report... logger.info("Adding record to regression test report") run_time_total = 0 StartRegressionTestResultsReport.append_to_regression_test_report( is_enabled, run_time_total, test_pass_fail, expected_status, max_severity, command_file_absolute) # If it was requested to append the results to the calling processor, get # the results from the runner and do so... # if ( (AppendResults != null) && AppendResults.equalsIgnoreCase("true")) { # TSCommandProcessor processor2 = runner.getProcessor(); # Object o_tslist = processor2.getPropContents("TSResultsList"); # PropList request_params = new PropList ( "" ); # if ( o_tslist != null ) { # @SuppressWarnings("unchecked") # List<TS> tslist = (List<TS>)o_tslist; # int size = tslist.size() # TS ts; # for ( int i = 0; i < size; i++ ) { # ts = tslist.get(i); # request_params.setUsingObject( "TS", ts ); # processor.processRequest( "AppendTimeSeries", request_params ); # } # } # } logger.info("...done processing commands from file.") else: # Add a record to the regression report (the is_enabled value is what is important for the report # because the test is not actually run)... # TODO smalers 2018-01-26 finish... logger.info("Command file is not enabled") run_time_total = 0 test_pass_fail = "" max_severity = CommandStatusType.UNKNOWN StartRegressionTestResultsReport.append_to_regression_test_report( is_enabled, run_time_total, test_pass_fail, expected_status, max_severity, command_file_absolute) pass except Exception as e: warning_count += 1 message = 'Unexpected error running command file "' + pv_CommandFile_absolute + '"' traceback.print_exc(file=sys.stdout) logger.error(message, e, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) except: warning_count += 1 message = 'Unexpected error running command file "' + pv_CommandFile_absolute + '"' traceback.print_exc(file=sys.stdout) logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) if warning_count > 0: message = "There were " + str(warning_count) + " warnings processing the command." logger.warning(message) raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Download the file from the web and save it on the local computer. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values pv_URL = self.get_parameter_value("URL") pv_OutputFile = self.get_parameter_value("OutputFile", default_value=None) pv_Username = self.get_parameter_value("Username", default_value=None) pv_Password = self.get_parameter_value("Password", default_value=None) # Convert the pv_URL parameter to expand for ${Property} syntax. url_abs = self.command_processor.expand_parameter_value(pv_URL, self) # Convert the OutputFile parameter value relative path to an absolute path. Expand for ${Property} syntax. # If the OutputFile parameter is specified, continue. if pv_OutputFile: output_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_OutputFile, self))) # If the OutputFile parameter is NOT specified, continue. else: original_filename = io_util.get_filename( pv_URL) + io_util.get_extension(pv_URL) output_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( original_filename, self))) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_run_webget(output_file_absolute): try: # Get the output folder. output_folder = os.path.dirname(output_file_absolute) # Get the URL file and convert it into a request Response object # Authentication Reference: http://docs.python-requests.org/en/master/user/authentication/ r = requests.get(url_abs, auth=HTTPBasicAuth(pv_Username, pv_Password), verify=False, stream=True) # Get the filename of the URL and the output file url_filename = io_util.get_filename(url_abs) output_filename = io_util.get_filename(output_file_absolute) # Remove the output file if it already exists. if os.path.exists(output_file_absolute): os.remove(output_file_absolute) # If the URL file is a zip file, process as a zip file. if zip_util.is_zip_file_request(r): # Create an empty list to hold the files that were downloaded/extracted to the output folder. downloaded_files = [] with open( os.path.join(output_folder, "{}.zip".format(url_filename)), "wb") as downloaded_zip_file: downloaded_zip_file.write(r.content) downloaded_files.append("{}.zip".format(url_filename)) # Determine if the downloaded zip file(s) should be renamed. If the filename is %f then the # filenames of the downloaded products should be the same as the url filenames if not output_filename == '%f': self.__rename_files_in_a_folder( list_of_files=downloaded_files, folder_path=output_folder, new_filename=output_filename) else: # Download the file to the output folder. with open( os.path.join(output_folder, os.path.basename(url_abs)), "wb") as downloaded_file: downloaded_file.write(r.content) # Determine if the downloaded file should be renamed. If the filename is %f then the filename # of the downloaded product should be the same as the url filename if not output_filename == '%f': self.__rename_files_in_a_folder( list_of_files=[os.path.basename(url_abs)], folder_path=output_folder, new_filename=output_filename) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error downloading file from URL {}.".format( url_abs) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): # Obtain the parameter values. pv_Folder = self.get_parameter_value("Folder") pv_URL = self.get_parameter_value("URL") pv_ListFiles = self.get_parameter_value("ListFiles", default_value="True") pv_ListFolders = self.get_parameter_value("ListFolders", default_value="True") pv_ListProperty = self.get_parameter_value("ListProperty") pv_IncludePatterns = self.get_parameter_value("IncludePatterns", default_value="*") pv_ExcludePatterns = self.get_parameter_value("ExcludePatterns", default_value="''") # Convert the IncludeAttributes and ExcludeAttributes to lists. to_include = string_util.delimited_string_to_list(pv_IncludePatterns) to_exclude = string_util.delimited_string_to_list(pv_ExcludePatterns) # Convert the pv_ListFiles and pv_ListFolders to Boolean values. list_files_bool = string_util.str_to_bool(pv_ListFiles) list_dirs_bool = string_util.str_to_bool(pv_ListFolders) # Set the absolute paths for the Folder and the URL to None until proven to exist within the command process. folder_abs = None url_abs = None # If the input is a local folder. if pv_Folder: # Convert the Folder parameter value relative path to an absolute path and expand for ${Property} syntax. folder_abs = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_Folder, self))) # If the input is a url. if pv_URL: # Convert the URL parameter to expand for ${Property} syntax. url_abs = self.command_processor.expand_parameter_value( pv_URL, self) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_list_files(folder_abs, url_abs, list_files_bool, list_dirs_bool, pv_ListProperty): try: # If the input is a local folder. if pv_Folder: # Get a list of the files in the folder. files = [ f for f in os.listdir(folder_abs) if os.path.isfile(os.path.join(folder_abs, f)) ] # Get a list of directories in the folder. dirs = [ d for d in os.listdir(folder_abs) if os.path.isdir(os.path.join(folder_abs, d)) ] # If configured to list files and folders, continue. if list_files_bool and list_dirs_bool: # Filter the list of files and folders with regards to the IncludePatterns and ExcludePatterns. output_filtered = string_util.filter_list_of_strings( files + dirs, to_include, to_exclude) # If configured to list files, continue. elif list_files_bool: # Filter the list of files with regards to the IIncludePatterns and ExcludePatterns. output_filtered = string_util.filter_list_of_strings( files, to_include, to_exclude) # If configured to list folders, continue. elif list_dirs_bool: # Filter the list of folders with regards to the IIncludePatterns and ExcludePatterns. output_filtered = string_util.filter_list_of_strings( dirs, to_include, to_exclude) else: output_filtered = [] # Add the filtered list to the desired ListProperty. Sort the list alphabetically self.command_processor.set_property( pv_ListProperty, sorted(output_filtered, key=str.lower)) # If the input is a url. if pv_URL: # If configured to list files and folders, continue. if list_files_bool and list_dirs_bool: output_filtered = self.scrape_html( url_abs, True, True, to_include, to_exclude) # If configured to list files, continue. elif list_files_bool: output_filtered = self.scrape_html( url_abs, True, False, to_include, to_exclude) # If configured to list folders, continue. elif list_dirs_bool: output_filtered = self.scrape_html( url_abs, False, True, to_include, to_exclude) else: output_filtered = None # Add the filtered list to the desired ListProperty. self.command_processor.set_property( pv_ListProperty, sorted(output_filtered, key=str.lower)) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error running ListFiles command." recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Read all spatial data files within the folder. For each desired spatial data file (can be specified by the Subset_Pattern parameter), create a GeoLayer object, and add to the GeoProcessor's geolayer list. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. pv_SpatialDataFolder = self.get_parameter_value("SpatialDataFolder") pv_Subset_Pattern = self.get_parameter_value("Subset_Pattern") pv_GeoLayerID_prefix = self.get_parameter_value("GeoLayerID_prefix") # Convert the SpatialDataFolder parameter value relative path to an absolute path sd_folder_abs = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_SpatialDataFolder, self))) # Run the initial checks on the parameter values. Only continue if the checks passed. if self.__should_read_folder(sd_folder_abs): # Determine which files within the folder should be processed. All files will be processed if # pv_Subset_Pattern is set to None. Otherwise only files that match the given pattern will be processed. # Check that each file in the folder is: # 1. a file # 2. a spatial data file (ends in .shp or .geojson) # 3. follows the given pattern (if Subset_Pattern parameter value does not equal None) if pv_Subset_Pattern: spatial_data_files_abs = [os.path.join(sd_folder_abs, source_file) for source_file in glob.glob(os.path.join(sd_folder_abs, pv_Subset_Pattern)) if os.path.isfile(os.path.join(sd_folder_abs, source_file)) and (source_file.endswith(".shp") or source_file.endswith(".geojson"))] else: spatial_data_files_abs = [os.path.join(sd_folder_abs, source_file) for source_file in os.listdir(sd_folder_abs) if os.path.isfile(os.path.join(sd_folder_abs, source_file)) and (source_file.endswith(".shp") or source_file.endswith(".geojson"))] # Iterate through the desired spatial data files for spatial_data_file_absolute in spatial_data_files_abs: # Determine the GeoLayerID. if pv_GeoLayerID_prefix: geolayer_id = "{}_{}".format(pv_GeoLayerID_prefix, io_util.expand_formatter(spatial_data_file_absolute, '%f')) else: geolayer_id = io_util.expand_formatter(spatial_data_file_absolute, '%f') # Run the secondary checks on the parameter values. Only continue if the checks passed. if self.__should_read_geolayer(geolayer_id): try: # Create a QGSVectorLayer object with the GeoJSON SpatialDataFile qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_file(spatial_data_file_absolute) # Create a GeoLayer and add it to the geoprocessor's GeoLayers list geolayer_obj = GeoLayer(geolayer_id, qgs_vector_layer, spatial_data_file_absolute) self.command_processor.add_geolayer(geolayer_obj) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error reading GeoLayer {} from" \ " file {}.".format(geolayer_id, spatial_data_file_absolute) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log(CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format(self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Remove the specified file from the operating system. Returns: Nothing. Raises: RuntimeError: if a runtime input error occurs. """ warning_count = 0 logger = logging.getLogger(__name__) # Get data for the command pv_SourceFile = self.get_parameter_value('SourceFile') pv_IfSourceFileNotFound = self.get_parameter_value( 'IfSourceFileNotFound') if pv_IfSourceFileNotFound is None or pv_IfSourceFileNotFound == "": pv_IfSourceFileNotFound = 'Warn' # Default pv_RemoveIfFolder = self.get_parameter_value('RemoveIfFolder', default_value="False") pv_RemoveIfFolder = string_util.str_to_bool(pv_RemoveIfFolder) # Runtime checks on input pv_SourceFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_SourceFile, self))) if warning_count > 0: message = "There were " + str( warning_count) + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Remove the file try: input_count = 1 if not os.path.exists(pv_SourceFile_absolute): message = 'The source file does not exist: "' + pv_SourceFile_absolute + '"' if pv_IfSourceFileNotFound == 'Warn': warning_count += 1 self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord( CommandStatusType.WARNING, message, "Verify that the source exists at the time the command is run." )) logger.warning(message) elif pv_IfSourceFileNotFound == 'Fail': warning_count += 1 self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord( CommandStatusType.FAILURE, message, "Verify that the source exists at the time the command is run." )) logger.warning(message) input_count -= 1 if input_count == 1: try: logger.info('Removing file "' + pv_SourceFile_absolute + '"') os.remove(pv_SourceFile_absolute) except: if pv_RemoveIfFolder: shutil.rmtree(pv_SourceFile_absolute) except Exception as e: warning_count += 1 message = 'Unexpected error removing file "' + pv_SourceFile_absolute + '"' traceback.print_exc(file=sys.stdout) logger.exception(message, e) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) if warning_count > 0: message = "There were " + str( warning_count) + " warnings processing the command." raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Compare the input files. Returns: Nothing. Raises: RuntimeError: if a runtime input error occurs. """ warning_count = 0 logger = logging.getLogger(__name__) # Get runtime data for the command pv_InputFile1 = self.get_parameter_value('InputFile1') pv_InputFile2 = self.get_parameter_value('InputFile2') pv_CommentLineChar = self.get_parameter_value('CommentLineChar') if pv_CommentLineChar is None or pv_CommentLineChar == "": pv_CommentLineChar = "#" # Default value pv_MatchCase = self.get_parameter_value('MatchCase') if pv_MatchCase is None or pv_MatchCase == "": pv_MatchCase = True # Default value pv_IgnoreWhitespace = self.get_parameter_value('IgnoreWhitespace') if pv_IgnoreWhitespace is None or pv_IgnoreWhitespace == "": pv_IgnoreWhitespace = True # Default value pv_AllowedDiffCount = self.get_parameter_value('AllowedDiffCount') if pv_AllowedDiffCount is None or pv_AllowedDiffCount == "": allowed_diff_count = 0 # Default value else: allowed_diff_count = int(pv_AllowedDiffCount) # Convert IfDifferent and IfSame to internal types, Ignore will convert to None, which is OK pv_IfDifferent = self.get_parameter_value('IfDifferent') if pv_IfDifferent is None or pv_IfDifferent == "": pv_IfDifferent = "Ignore" # Default value pv_IfDifferent_CommandStatusType = CommandStatusType.value_of( pv_IfDifferent, True) pv_IfSame = self.get_parameter_value('IfSame') if pv_IfSame is None or pv_IfSame == "": pv_IfSame = "Ignore" # Default value pv_IfSame_CommandStatusType = CommandStatusType.value_of( pv_IfSame, True) # Runtime checks on input pv_InputFile1_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_InputFile1, self))) pv_InputFile2_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_InputFile2, self))) if warning_count > 0: message = "There were " + warning_count + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Do the processing diff_count = 0 line_count_compared = 0 try: input_count = 2 if not os.path.exists(pv_InputFile1_absolute): warning_count += 1 message = 'The first input file does not exist: "' + pv_InputFile1_absolute + '"' self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord( CommandStatusType.FAILURE, message, "Verify that the first input file exists at the time the command is run." )) logger.warning(message) input_count -= 1 if not os.path.exists(pv_InputFile2_absolute): warning_count += 1 message = 'The second input file does not exist: "' + pv_InputFile2_absolute + '"' self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord( CommandStatusType.FAILURE, message, "Verify that the second input file exists at the time the command is run." )) logger.warning(message) input_count -= 1 if input_count == 2: # Try to do the comparison, just inline the logic # Open the files... in1 = open(pv_InputFile1_absolute, 'r') in2 = open(pv_InputFile2_absolute, 'r') # Loop through the files, comparing non-comment lines... while True: # The following will discard comments and only return non-comment lines # Therefore, comparisons are made on chunks of non-comment lines. # TODO smalers 2018-01-08 Could make this comparison more intelligent if the # of comments varies iline1 = CompareFiles.__read_line(in1, pv_CommentLineChar, pv_IgnoreWhitespace) iline2 = CompareFiles.__read_line(in2, pv_CommentLineChar, pv_IgnoreWhitespace) if iline1 is None and iline2 is None: # Both are done at the same time... break # TODO smalers 2006-04-20 The following needs to handle comments at the end... if iline1 is None and iline2 is not None: # First file is done (second is not) so files are different... diff_count += 1 break if iline2 is None and iline1 is not None: # Second file is done (first is not) so files are different... diff_count += 1 break line_count_compared += 1 if pv_MatchCase: # Compare the lines as is since case-specific if iline1 != iline2: diff_count += 1 else: # Compare by ignoring case if iline1.upper() != iline2.upper(): diff_count += 1 # logger.debug('Compared:\n"' + iline1 + '"\n"' + iline2 + '"\ndiff_count=' + str(diff_count)) in1.close() in2.close() if line_count_compared == 0: line_count_compared = 1 # to avoid divide by zero below. logger.info("There are " + str(diff_count) + " lines that are different, " + '{:4f}'.format(100.0 * float(diff_count) / float(line_count_compared)) + '% (compared ' + str(line_count_compared) + ' lines).') except Exception as e: warning_count += 1 message = 'Unexpected error comparing file "' + pv_InputFile1_absolute + '" to "' + \ pv_InputFile2_absolute + '"' traceback.print_exc(file=sys.stdout) logger.exception(message, e) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) if diff_count > allowed_diff_count and \ ((pv_IfDifferent_CommandStatusType == CommandStatusType.WARNING) or (pv_IfDifferent_CommandStatusType == CommandStatusType.FAILURE)): message = "" + str(diff_count) + " lines were different, " + \ '{:4f}'.format(100.0*float(diff_count)/float(line_count_compared)) + \ "% (compared " + str(line_count_compared) + " lines)." if pv_IfDifferent_CommandStatusType == CommandStatusType.WARNING: logger.warning(message) else: logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord( pv_IfDifferent_CommandStatusType, message, "Check files because difference is not expected.")) raise RuntimeError(message) if diff_count == 0 and \ ((pv_IfSame_CommandStatusType == CommandStatusType.WARNING) or (pv_IfSame_CommandStatusType == CommandStatusType.FAILURE)): message = "No lines were different (the files are the same)." if pv_IfDifferent_CommandStatusType == CommandStatusType.WARNING: logger.warning(message) else: logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(pv_IfSame_CommandStatusType, message, "Check files because match is not expected.")) raise RuntimeError(message) if warning_count > 0: message = "There were " + str( warning_count) + " warnings processing the command." raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Open the connection to a database and store as a DataStore. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. The DatabasePort parameter value will be obtained later in the code. pv_DatabaseDialect = self.get_parameter_value("DatabaseDialect") pv_DatabaseServer = self.get_parameter_value("DatabaseServer") pv_DatabaseName = self.get_parameter_value("DatabaseName") pv_DatabaseUser = self.get_parameter_value("DatabaseUser") pv_DatabasePassword = self.get_parameter_value("DatabasePassword") pv_DatabasePort = self.get_parameter_value("DatabasePort") pv_DataStoreID = self.get_parameter_value("DataStoreID") pv_ConfigFile = self.get_parameter_value("ConfigFile") pv_IfDataStoreIDExists = self.get_parameter_value( "IfDataStoreIDExists", default_value="Replace") # Expand for ${Property} syntax. pv_DatabaseServer = self.command_processor.expand_parameter_value( pv_DatabaseServer, self) pv_DatabaseName = self.command_processor.expand_parameter_value( pv_DatabaseName, self) pv_DatabaseUser = self.command_processor.expand_parameter_value( pv_DatabaseUser, self) pv_DatabasePassword = self.command_processor.expand_parameter_value( pv_DatabasePassword, self) # Convert the File parameter value relative path to an absolute path and expand for ${Property} syntax. if pv_ConfigFile: pv_ConfigFile = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_ConfigFile, self))) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_open_datastore(pv_DataStoreID, pv_ConfigFile, pv_IfDataStoreIDExists): try: # If the DataStoreID already exists and the IfDataStoreIDExists parameter is set to "Open", continue. if self.command_processor.get_datastore( pv_DataStoreID) and pv_IfDataStoreIDExists.upper( ) == "OPEN": # Get the DataStore obj from the ID and open the connection. datastore_obj = self.command_processor.get_datastore( pv_DataStoreID) datastore_obj.open_db_connection() # If the "Configuration file configures datastore" method is used. elif pv_ConfigFile is not None: print( "WARNING: The 'Configuration file configures datastore' method is not currently enabled." ) # If the "Parameters configure datastore" method is used. else: # Create a new DataStore object and assign the DataStore ID. new_datastore = DataStore(pv_DataStoreID) # Assign the DataBase URI for the appropriate DataBase dialect. if pv_DatabaseDialect.upper() == "POSTGRESQL": new_datastore.get_db_uri_postgres( pv_DatabaseServer, pv_DatabaseName, pv_DatabaseUser, pv_DatabasePassword, pv_DatabasePort) # Open a connection to the database and add the DataStore object to the GeoProcessor. new_datastore.open_db_connection() self.command_processor.add_datastore(new_datastore) # Raise an exception if an unexpected error occurs during the process. except Exception as e: self.warning_count += 1 message = "Unexpected error opening DataStore {}.".format( pv_DataStoreID) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Write the GeoLayer to a delimited file. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the command parameter values. pv_GeoLayerID = self.get_parameter_value("GeoLayerID") pv_OutputFile = self.get_parameter_value("OutputFile") pv_OutputCRS = self.get_parameter_value("OutputCRS") pv_OutputGeometryFormat = self.get_parameter_value( "OutputGeometryFormat", default_value="XY").upper() pv_OutputDelimiter = self.get_parameter_value( "OutputDelimiter", default_value="COMMA").upper() # Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax output_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_OutputFile, self))) # Get the filename of the OutputFile with the path but without the extension. path, filename = os.path.split(output_file_absolute) path = os.path.split(output_file_absolute)[0] filename_wo_ext_path = os.path.join(path, os.path.splitext(filename)[0]) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_write_geolayer(pv_GeoLayerID, filename_wo_ext_path, pv_OutputCRS, pv_OutputGeometryFormat): try: # Get the GeoLayer geolayer = self.command_processor.get_geolayer(pv_GeoLayerID) # Get the current coordinate reference system (in EPSG code) of the current GeoLayer if pv_OutputCRS is None: pv_OutputCRS = geolayer.get_crs() # Write the GeoLayer to a delimited spatial data file. qgis_util.write_qgsvectorlayer_to_delimited_file( geolayer.qgs_vector_layer, filename_wo_ext_path, pv_OutputCRS, pv_OutputGeometryFormat, pv_OutputDelimiter) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error writing GeoLayer {} to delimited file format.".format( pv_GeoLayerID) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Write the GeoLayer to a spatial data file in GeoJSON format. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values except for the OutputCRS pv_GeoLayerID = self.get_parameter_value("GeoLayerID") pv_OutputPrecision = int(self.get_parameter_value("OutputPrecision", default_value=5)) pv_OutputFile = self.get_parameter_value("OutputFile") # Expand for ${Property} syntax. pv_GeoLayerID = self.command_processor.expand_parameter_value(pv_GeoLayerID, self) # Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax output_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_OutputFile, self))) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_write_geolayer(pv_GeoLayerID, output_file_absolute, pv_OutputPrecision): try: # Get the GeoLayer geolayer = self.command_processor.get_geolayer(pv_GeoLayerID) # Get the current coordinate reference system (in EPSG code) of the current GeoLayer geolayer_crs = geolayer.get_crs() # Obtain the parameter value of the OutputCRS pv_OutputCRS = self.get_parameter_value("OutputCRS", default_value=geolayer_crs) # Write the GeoLayer to a spatial data file in GeoJSONformat qgis_util.write_qgsvectorlayer_to_geojson(geolayer.qgs_vector_layer, output_file_absolute, pv_OutputCRS, pv_OutputPrecision) # Save the output file in the processor self.command_processor.add_output_file(output_file_absolute) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error writing GeoLayer {} to GeoJSON format.".format(pv_GeoLayerID) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log(CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format(self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Read the layer file from a GeoJSON file, create a GeoLayer object, and add to the GeoProcessor's geolayer list. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ logger = logging.getLogger(__name__) # Obtain the parameter values. pv_SpatialDataFile = self.get_parameter_value("SpatialDataFile") pv_GeoLayerID = self.get_parameter_value("GeoLayerID", default_value='%f') # Expand for ${Property} syntax. pv_GeoLayerID = self.command_processor.expand_parameter_value(pv_GeoLayerID, self) # Convert the SpatialDataFile parameter value relative path to an absolute path and expand for ${Property} # syntax spatial_data_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_SpatialDataFile, self))) # If the pv_GeoLayerID is a valid %-formatter, assign the pv_GeoLayerID the corresponding value. if pv_GeoLayerID in ['%f', '%F', '%E', '%P', '%p']: pv_GeoLayerID = io_util.expand_formatter(spatial_data_file_absolute, pv_GeoLayerID) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_read_geolayer(spatial_data_file_absolute, pv_GeoLayerID): try: # Create a QGSVectorLayer object with the GeoJSON SpatialDataFile. qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_file(spatial_data_file_absolute) # Create a GeoLayer and add it to the geoprocessor's GeoLayers list. geolayer_obj = GeoLayer(geolayer_id=pv_GeoLayerID, geolayer_qgs_vector_layer=qgs_vector_layer, geolayer_source_path=spatial_data_file_absolute) self.command_processor.add_geolayer(geolayer_obj) # Raise an exception if an unexpected error occurs during the process. except Exception as e: self.warning_count += 1 message = "Unexpected error reading GeoLayer {} from GeoJSON file {}.".format(pv_GeoLayerID, pv_SpatialDataFile) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log(CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format(self.warning_count) logger.error(message) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Write the processor properties to a file. Returns: Nothing. Raises: RuntimeError: if a runtime input error occurs. """ warning_count = 0 logger = logging.getLogger(__name__) # Get data for the command pv_OutputFile = self.get_parameter_value('OutputFile') pv_IncludeProperties = self.get_parameter_value('IncludeProperties') include_properties = [] # Default if pv_IncludeProperties is not None and len(pv_IncludeProperties) > 0: include_properties = string_util.delimited_string_to_list( pv_IncludeProperties) pv_WriteMode = self.get_parameter_value('WriteMode') write_mode = pv_WriteMode if pv_WriteMode is None or pv_WriteMode == "": write_mode = 'Overwrite' # Default pv_FileFormat = self.get_parameter_value('FileFormat') file_format = pv_FileFormat if pv_FileFormat is None or pv_FileFormat == "": file_format = 'NameTypeValue' # Default pv_SortOrder = self.get_parameter_value('SortOrder') sort_order = 0 # no sort if pv_SortOrder is not None: if pv_SortOrder == 'Ascending': sort_order = 1 elif pv_SortOrder == 'Descending': sort_order = -1 # Runtime checks on input pv_OutputFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_OutputFile, self))) if warning_count > 0: message = "There were " + str( warning_count) + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Write the output file try: problems = [] # Empty list of properties io_util.write_property_file(pv_OutputFile_absolute, self.command_processor.properties, include_properties, write_mode, file_format, sort_order, problems) # Record any problems that were found for problem in problems: warning_count += 1 logger.error(problem) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, problem, "See the log file for details.")) except Exception as e: warning_count += 1 traceback.print_exc( file=sys.stdout) # Formatting of error seems to have issue message = 'Unexpected error writing file "' + pv_OutputFile_absolute + '"' logger.error(message, e, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) except: warning_count += 1 traceback.print_exc( file=sys.stdout) # Formatting of error seems to have issue message = 'Unexpected error writing file "' + pv_OutputFile_absolute + '"' logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) if warning_count > 0: message = "There were " + str( warning_count) + " warnings processing the command." logger.warning(message) raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Read the Table from the delimited file. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. pv_InputFile = self.get_parameter_value("InputFile") pv_Delimiter = self.get_parameter_value("Delimiter", default_value=",") pv_TableID = self.get_parameter_value("TableID") pv_HeaderLines = int( self.get_parameter_value("HeaderLines", default_value="0")) pv_NullValues = self.get_parameter_value("NullValues", default_value="''") # Convert the InputFile parameter value relative path to an absolute path and expand for ${Property} syntax input_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_InputFile, self))) # Convert the NullValues parameter values to a list. pv_NullValues = string_util.delimited_string_to_list(pv_NullValues) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_read_table(input_file_absolute, pv_TableID): try: # Create the table from the delimited file. table = self.__read_table_from_delimited_file( input_file_absolute, pv_TableID, pv_Delimiter, pv_HeaderLines, pv_NullValues) # Add the table to the GeoProcessor's Tables list. self.command_processor.add_table(table) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error reading Table {} from delimited file ({}).".format( pv_TableID, input_file_absolute) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Read the tabular data from the Excel workbook/worksheet. Create a Table object, and add to the GeoProcessor's tables list. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. pv_InputFile = self.get_parameter_value("InputFile") pv_Worksheet = self.get_parameter_value("Worksheet") pv_TableID = self.get_parameter_value("TableID") # Convert the InputFile parameter value relative path to an absolute path and expand for ${Property} syntax. file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_InputFile, self))) # If the pv_TableID is a valid %-formatter, assign the pv_GeoLayerID the corresponding value. if pv_TableID in ['%f', '%F', '%E', '%P', '%p']: pv_TableID = io_util.expand_formatter(file_absolute, pv_TableID) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_read_table(file_absolute, pv_Worksheet, pv_TableID): try: # Assign the Worksheet parameter to the name of the first Excel worksheet, if it was not specified. if pv_Worksheet is None: pv_Worksheet = pandas_util.create_excel_workbook_obj( file_absolute).sheet_names[0] # Assign the TableID parameter to the name of the first Excel worksheet, if it was not specified. if pv_TableID is None: pv_TableID = pv_Worksheet # Create a Pandas Data Frame object. df = pandas_util.create_data_frame_from_excel( file_absolute, pv_Worksheet) # Create a Table and add it to the geoprocessor's Tables list. table_obj = Table(pv_TableID, df, file_absolute) self.command_processor.add_table(table_obj) # Raise an exception if an unexpected error occurs during the process. except Exception as e: self.warning_count += 1 message = "Unexpected error reading Table {} from Excel file {}.".format( pv_TableID, pv_InputFile) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Write the Table to a delimited file. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. pv_TableID = self.get_parameter_value("TableID") pv_OutputFile = self.get_parameter_value("OutputFile") pv_Delimiter = self.get_parameter_value("Delimiter", default_value=",") pv_IncludeColumns = self.get_parameter_value("IncludeColumns", default_value="*") pv_ExcludeColumns = self.get_parameter_value("ExcludeColumns", default_value="") pv_WriteHeaderRow = self.get_parameter_value("WriteHeaderRow", default_value="True") pv_WriteIndexColumn = self.get_parameter_value("WriteIndexColumn", default_value="False") pv_SortColumns = self.get_parameter_value("SortColumns") pv_SortOrder = self.get_parameter_value("SortOrder", default_value="") pv_ArrayFormat = self.get_parameter_value( "ArrayFormat", default_value="SquareBrackets") pv_NullValueFormat = self.get_parameter_value("NullValueFormat", default_value="Null") # Convert the IncludeColumns, ExcludeColumns, and SortColumns parameter values to lists. cols_to_include = string_util.delimited_string_to_list( pv_IncludeColumns) cols_to_exclude = string_util.delimited_string_to_list( pv_ExcludeColumns) sort_cols_list = string_util.delimited_string_to_list(pv_SortColumns) # Convert the SortOrder to a dictionary. sort_dictionary = string_util.delimited_string_to_dictionary_one_value( pv_SortOrder, entry_delimiter=",", key_value_delimiter=":", trim=True) # Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax output_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_OutputFile, self))) # Covert the Boolean parameters from string to Boolean values. pv_WriteHeaderRow = string_util.str_to_bool(pv_WriteHeaderRow) pv_WriteIndexColumn = string_util.str_to_bool(pv_WriteIndexColumn) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_write_table(pv_TableID, output_file_absolute, pv_Delimiter, sort_cols_list): try: # Get the Table object table = self.command_processor.get_table(pv_TableID) # Determine if square brackets should be used depending on the user input of the ArrayFormat parameter. use_sq_brackets = False if pv_ArrayFormat.upper() == "SQUAREBRACKETS": use_sq_brackets = True # Determine if the null values should be used depending on the user input of the NullValueFormat # parameter. use_null_value = False if pv_NullValueFormat.upper() == "NULL": use_null_value = True # Write the table to the delimited file. self.__write_table_to_delimited_file( output_file_absolute, table, pv_Delimiter, cols_to_include, cols_to_exclude, pv_WriteHeaderRow, pv_WriteIndexColumn, sort_cols_list, sort_dictionary, use_sq_brackets, use_null_value) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error writing Table {} to delimited file {}.".format( pv_TableID, pv_OutputFile) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Run the program, which can generate output files. Returns: None. Raises: ValueError: if a runtime input error occurs. RuntimeError: if a runtime error occurs. """ warning_count = 0 logger = logging.getLogger(__name__) logger.info('In RunProgram.run_command') # Get data for the command print("command parameters=" + string_util.format_dict(self.command_parameters)) pv_CommandLine = self.get_parameter_value('CommandLine') pv_UseCommandShell = self.get_parameter_value('UseCommandShell') use_command_shell = False # Default if pv_UseCommandShell is not None and pv_UseCommandShell == 'True': use_command_shell = True pv_IncludeParentEnvVars = self.get_parameter_value( 'IncludeParentEnvVars') include_parent_env_vars = True # Default if pv_IncludeParentEnvVars is not None and pv_IncludeParentEnvVars == 'False': include_parent_env_vars = False pv_IncludeEnvVars = self.get_parameter_value('IncludeEnvVars') include_env_vars_dict = None if pv_IncludeEnvVars is not None and pv_IncludeEnvVars != "": # Have specified environment variables to include # - expand the environment variable value using processor properties include_env_vars_dict = string_util.delimited_string_to_dictionary_one_value( pv_IncludeEnvVars, key_value_delimiter="=", trim=True) for key, value in include_env_vars_dict.items(): include_env_vars_dict[ key] = self.command_processor.expand_parameter_value( value, self) # Add environment variables individually by name # - these are used when a list of parameters is difficult to parse # - this is kind of ugly but meets requirements in the short term pv_IncludeEnvVarName1 = self.get_parameter_value('IncludeEnvVarName1') pv_IncludeEnvVarValue1 = self.get_parameter_value( 'IncludeEnvVarValue1') if pv_IncludeEnvVarName1 is not None and pv_IncludeEnvVarName1 != "": if include_env_vars_dict is None: include_env_vars_dict = {} include_env_vars_dict[ pv_IncludeEnvVarName1] = pv_IncludeEnvVarValue1 pv_IncludeEnvVarName2 = self.get_parameter_value('IncludeEnvVarName2') pv_IncludeEnvVarValue2 = self.get_parameter_value( 'IncludeEnvVarValue2') if pv_IncludeEnvVarName2 is not None and pv_IncludeEnvVarName2 != "": if include_env_vars_dict is None: include_env_vars_dict = {} include_env_vars_dict[ pv_IncludeEnvVarName2] = pv_IncludeEnvVarValue2 pv_IncludeEnvVarName3 = self.get_parameter_value('IncludeEnvVarName3') pv_IncludeEnvVarValue3 = self.get_parameter_value( 'IncludeEnvVarValue3') if pv_IncludeEnvVarName3 is not None and pv_IncludeEnvVarName3 != "": if include_env_vars_dict is None: include_env_vars_dict = {} include_env_vars_dict[ pv_IncludeEnvVarName3] = pv_IncludeEnvVarValue3 pv_IncludeEnvVarName4 = self.get_parameter_value('IncludeEnvVarName4') pv_IncludeEnvVarValue4 = self.get_parameter_value( 'IncludeEnvVarValue4') if pv_IncludeEnvVarName4 is not None and pv_IncludeEnvVarName4 != "": if include_env_vars_dict is None: include_env_vars_dict = {} include_env_vars_dict[ pv_IncludeEnvVarName4] = pv_IncludeEnvVarValue4 pv_IncludeEnvVarName5 = self.get_parameter_value('IncludeEnvVarName5') pv_IncludeEnvVarValue5 = self.get_parameter_value( 'IncludeEnvVarValue5') if pv_IncludeEnvVarName5 is not None and pv_IncludeEnvVarName5 != "": if include_env_vars_dict is None: include_env_vars_dict = {} include_env_vars_dict[ pv_IncludeEnvVarName5] = pv_IncludeEnvVarValue5 pv_ExcludeEnvVars = self.get_parameter_value('ExcludeEnvVars') exclude_env_vars_list = None if pv_ExcludeEnvVars is not None and pv_ExcludeEnvVars != "": # Have specified environment variables to exclude exclude_env_vars_list = string_util.delimited_string_to_list( pv_ExcludeEnvVars, trim=True) pv_OutputFiles = self.get_parameter_value('OutputFiles') output_files_list = None if pv_OutputFiles is not None and pv_OutputFiles != "": # Have specified output files to add to command output files output_files_list = string_util.delimited_string_to_list( pv_OutputFiles, trim=True) # Expand each output file ifile = -1 for output_file in output_files_list: ifile = ifile + 1 output_files_list[ifile] = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( output_file, self))) logger.info('Command line before expansion="' + pv_CommandLine + '"') # Runtime checks on input command_line_expanded = self.command_processor.expand_parameter_value( pv_CommandLine, self) if warning_count > 0: message = "There were " + str( warning_count) + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Run the program as a subprocess try: logger.info('Running command line "' + command_line_expanded + '"') # Create the environment dictionary env_dict = self.create_env_dict(include_parent_env_vars, include_env_vars_dict, exclude_env_vars_list) print("env_dict=" + string_util.format_dict(env_dict)) # TODO smalers 2018-12-16 evaluate using shlex.quote() to handle command string # TODO smalers 2018-12-16 handle standard input and output p = subprocess.Popen(command_line_expanded, shell=use_command_shell, env=env_dict) # Wait for the process to terminate since need it to be done before other commands do their work # with the command output. p.wait() return_status = p.poll() if return_status != 0: warning_count += 1 message = 'Nonzero return status running program "' + command_line_expanded + '"' logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) except Exception as e: warning_count += 1 message = 'Unexpected error running program "' + command_line_expanded + '"' logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) except: warning_count += 1 message = 'Unexpected error running program "' + command_line_expanded + '"' logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) # If any output files were indicated, add to the command output if they exist if output_files_list is not None and len(output_files_list) > 0: for output_file in output_files_list: if os.path.isfile(output_file): # Add the log file to output self.command_processor.add_output_file(output_file) if warning_count > 0: message = "There were " + str( warning_count) + " warnings processing the command." logger.warning(message) raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Copy the source file to the destination. Returns: Nothing. Raises: RuntimeError: if a runtime input error occurs. """ warning_count = 0 logger = logging.getLogger(__name__) # Get data for the command pv_SourceFile = self.get_parameter_value('SourceFile') pv_DestinationFile = self.get_parameter_value('DestinationFile') # Runtime checks on input pv_SourceFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_SourceFile, self))) pv_DestinationFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_DestinationFile, self))) if warning_count > 0: message = "There were " + str( warning_count) + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Do the processing try: input_count = 2 if not os.path.exists(pv_SourceFile_absolute): warning_count += 1 message = 'The source file does not exist: "' + pv_SourceFile_absolute + '"' self.command_status.addToLog( CommandPhaseType.RUN, CommandLogRecord( CommandStatusType.FAILURE, message, "Verify that the source exists at the time the command is run." )) logger.warning(message) input_count -= 1 destination_folder = os.path.dirname(pv_DestinationFile_absolute) if not os.path.exists(destination_folder): warning_count += 1 message = 'The destination folder does not exist: "' + destination_folder + '"' self.command_status.addToLog( CommandPhaseType.RUN, CommandLogRecord( CommandStatusType.FAILURE, message, "Verify that the destination folder exists at the time the command is run." )) input_count -= 1 if input_count == 2: # Try to do the copy logger.info('Copying file "' + pv_SourceFile_absolute + '" to "' + pv_DestinationFile_absolute + '"') copyfile(pv_SourceFile_absolute, pv_DestinationFile_absolute) except Exception as e: warning_count += 1 message = 'Unexpected error copying file "' + pv_SourceFile_absolute + '" to "' + \ pv_DestinationFile_absolute + '"' traceback.print_exc(file=sys.stdout) logger.exception(message, e) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) if warning_count > 0: message = "There were " + str( warning_count) + " warnings processing the command." raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Read the layer file from a delimited file, create a GeoLayer object, and add to the GeoProcessor's geolayer list. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. pv_DelimitedFile = self.get_parameter_value("DelimitedFile") pv_Delimiter = self.get_parameter_value("Delimiter", default_value=',') pv_GeometryFormat = self.get_parameter_value("GeometryFormat") pv_XColumn = self.get_parameter_value("XColumn", default_value=None) pv_YColumn = self.get_parameter_value("YColumn", default_value=None) pv_WKTColumn = self.get_parameter_value("WKTColumn", default_value=None) pv_CRS = self.get_parameter_value("CRS") pv_GeoLayerID = self.get_parameter_value("GeoLayerID", default_value='%f') # Convert the DelimitedFile parameter value relative path to an absolute path and expand for ${Property} # syntax delimited_file_abs = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_DelimitedFile, self))) # If the pv_GeoLayerID is a valid %-formatter, assign the pv_GeoLayerID the corresponding value. if pv_GeoLayerID in ['%f', '%F', '%E', '%P', '%p']: pv_GeoLayerID = io_util.expand_formatter(delimited_file_abs, pv_GeoLayerID) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_read_geolayer(delimited_file_abs, pv_Delimiter, pv_GeometryFormat, pv_XColumn, pv_YColumn, pv_WKTColumn, pv_CRS, pv_GeoLayerID): try: if pv_GeometryFormat.upper() == "XY": # Create a QGSVectorLayer object with the delimited file. qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_delimited_file_xy( delimited_file_abs, pv_Delimiter, pv_CRS, pv_XColumn, pv_YColumn) else: # Create a QGSVectorLayer object with the delimited file. qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_delimited_file_wkt( delimited_file_abs, pv_Delimiter, pv_CRS, pv_WKTColumn) # Create a GeoLayer and add it to the geoprocessor's GeoLayers list. geolayer_obj = GeoLayer(pv_GeoLayerID, qgs_vector_layer, delimited_file_abs) self.command_processor.add_geolayer(geolayer_obj) # Raise an exception if an unexpected error occurs during the process. except Exception as e: self.warning_count += 1 message = "Unexpected error reading GeoLayer {} from delimited file {}.".format( pv_GeoLayerID, pv_DelimitedFile) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Read the feature classes within a file geodatabase. For each desired feature class (can be specified by the Subset_Pattern parameter), create a GeoLayer object, and add to the GeoProcessor's geolayer list. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the required and optional parameter values pv_SpatialDataFolder = self.get_parameter_value("SpatialDataFolder") pv_ReadOnlyOneFeatureClass = self.get_parameter_value( "ReadOnlyOneFeatureClass") pv_Subset_Pattern = self.get_parameter_value("Subset_Pattern") pv_GeoLayerID_prefix = self.get_parameter_value("GeoLayerID_prefix") pv_GeoLayerID = self.get_parameter_value("GeoLayerID") pv_FeatureClass = self.get_parameter_value("FeatureClass") # Convert the ReadOnlyOneFeatureClass from a string value to a Boolean value. pv_ReadOnlyOneFeatureClass = string_util.str_to_bool( pv_ReadOnlyOneFeatureClass) # Convert the SpatialDataFolder parameter value relative path to an absolute path sd_folder_abs = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_SpatialDataFolder, self))) # Run the initial checks on the parameter values. Only continue if the checks passed. if self.__should_read_gdb(sd_folder_abs): # If configured to only read one Feature Class into one GeoLayer. if pv_ReadOnlyOneFeatureClass: # Run the check to see if the GeoLayer should be read. if self.__should_read_geolayer(pv_GeoLayerID, True, pv_FeatureClass, sd_folder_abs): try: # Get the full pathname to the feature class # TODO egiles 2018-01-04 Need to research how to properly document feature class source path spatial_data_file_absolute = os.path.join( sd_folder_abs, str(pv_FeatureClass)) # Create a QgsVectorLayer object from the feature class. qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_feature_class( sd_folder_abs, pv_FeatureClass) # Create a GeoLayer and add it to the geoprocessor's GeoLayers list geolayer_obj = GeoLayer(pv_GeoLayerID, qgs_vector_layer, spatial_data_file_absolute) self.command_processor.add_geolayer(geolayer_obj) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error reading feature class ({}) from file geodatabase ({}).".format( pv_FeatureClass, sd_folder_abs) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # If configured to read multiple Feature Classes into multiple GeoLayers. else: # Get a list of all of the feature classes in the file geodatabase. fc_list = self.__return_a_list_of_fc(sd_folder_abs) # Filter the fc_list to only include feature classes that meet the Subset Pattern configuration. If # the Subset Pattern configuration is None, all feature classes will remain in the fc_list. fc_list = string_util.filter_list_of_strings( fc_list, [pv_Subset_Pattern]) # Iterate over the feature classes in the fc_list. for feature_class in fc_list: # Determine the GeoLayerID. if pv_GeoLayerID_prefix: geolayer_id = "{}_{}".format(pv_GeoLayerID_prefix, feature_class) else: geolayer_id = feature_class # Run the secondary checks on the parameter values. Only continue if the checks passed. if self.__should_read_geolayer(geolayer_id, False): try: # Get the full pathname to the feature class # TODO egiles 2018-01-04 Need to research how to properly document feature class source path spatial_data_file_absolute = os.path.join( sd_folder_abs, str(feature_class)) # Create a QgsVectorLayer object from the feature class. qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_feature_class( sd_folder_abs, feature_class) # Create a GeoLayer and add it to the geoprocessor's GeoLayers list geolayer_obj = GeoLayer( geolayer_id, qgs_vector_layer, spatial_data_file_absolute) self.command_processor.add_geolayer(geolayer_obj) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error reading feature class ({}) from file geodatabase ({}).".format( feature_class, sd_folder_abs) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Execute the Sql statement on the DataStore. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. The DatabasePort parameter value will be obtained later in the code. pv_DataStoreID = self.get_parameter_value("DataStoreID") pv_Sql = self.get_parameter_value("Sql") pv_SqlFile = self.get_parameter_value("SqlFile") pv_DataStoreProcedure = self.get_parameter_value("DataStoreProcedure") # Expand for ${Property} syntax. pv_DataStoreID = self.command_processor.expand_parameter_value(pv_DataStoreID, self) pv_Sql = self.command_processor.expand_parameter_value(pv_Sql, self) if pv_SqlFile: pv_SqlFile = io_util.verify_path_for_os(io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_SqlFile, self))) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_run_sql(pv_DataStoreID): try: # Get the DataStore object datastore_obj = self.command_processor.get_datastore(pv_DataStoreID) # If using the Sql method, the sql_statement is the user-provided sql statement. if pv_Sql: sql_statement = pv_Sql # If using the SqlFile method, the sql_statement in read from the provided file. elif pv_SqlFile: # Get the SQL statement from the file. f = open(pv_SqlFile, 'r') sql_statement = f.read().strip() # If using the DataStoreProcedure method, ... . else: sql_statement = None # Execute and commit the SQL statement. datastore_obj.run_sql(sql_statement) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error executing the Sql statement on the {} DataStore.".format(pv_DataStoreID) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log(CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format(self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def ui_action_select_file(self, event_button): """ Open a file selector dialog to select an input or output file (or folder) to be used as a command parameter value. Args: event_button (QPushButton): the instance of the button for which the event is generated. Use to get the parameter name, so as to get other parameter/component data. Returns: None """ logger = logging.getLogger(__name__) # Initialize folder to None and determine below with several checks. folder_start = None # Get properties to configure the file selector. # - use the object name to parse out the parameter name object_name = event_button.objectName() print("object_name=" + str(object_name)) parameter_name = object_name.split(".")[0] # Do the following first because it influences defaults below request_key = parameter_name + "." + "FileSelector.SelectFolder" select_folder = False try: # The following should match ParameterName.FileSelectorTitle select_folder = self.command.parameter_input_metadata[request_key] except KeyError: # Default was specified above... pass request_key = parameter_name + "." + "FileSelector.Title" if select_folder: select_file_title = "Select folder" else: select_file_title = "Select file" try: # The following should match ParameterName.FileSelectorTitle select_file_title = self.command.parameter_input_metadata[request_key] except KeyError: # Default was specified above... pass # Get the existing value in the text field, which will correspond to the parameter name value. # - if specified as absolute path, use it as is # - if a relative path, append to the working directory or if that is not available the user's home folder parameter_QLineEdit = None working_dir = self.command.command_processor.get_property('WorkingDir') user_folder = self.app_session.get_user_folder() try: parameter_QLineEdit = self.input_ui_components[parameter_name] # Get the parameter value parameter_value = parameter_QLineEdit.text() # If the parameter is empty or null if parameter_value is None or parameter_value == "": # Try to set the folder to the working directory first if working_dir is not None: folder_start = working_dir else: # Finally, use the user's home folder folder_start = self.app_session.get_user_folder() else: # The parameter is specified. if os.path.isabs(parameter_value): # The input is an absolute path so use as is folder_start = parameter_value else: # The input is relative to the working directory so append to working directory with # filesystem separator. if working_dir is not None: folder_start = io_util.to_absolute_path(working_dir, parameter_value) else: folder_start = io_util.to_absolute_path(user_folder, parameter_value) except KeyError: # Can't determine the input component so will assume the working directory, if available if working_dir is not None: folder_start = working_dir else: # Finally, use the user's home folder folder_start = user_folder # A browser window will appear to allow the user to browse to the desired file. # The absolute pathname of the command file is added to the cmd_filepath variable. use_qt_dialog = True # For now use Qt build-in dialog but may want to try native dialog filepath_selected = None if use_qt_dialog: parameter_QFileDialog = QtWidgets.QFileDialog(self, select_file_title, folder_start) if select_folder: # A directory is being selected parameter_QFileDialog.setFileMode(QtWidgets.QFileDialog.DirectoryOnly) if parameter_QFileDialog.exec_() == QtWidgets.QFileDialog.Accepted: filepath_selected = parameter_QFileDialog.selectedFiles()[0] if not filepath_selected: # The file selection was canceled return # Convert the selected file path to relative path, using the command file folder as the working directory. if working_dir is not None: filepath_selected = io_util.to_relative_path(working_dir, filepath_selected) # Set the file in the text field. parameter_QLineEdit.setText(filepath_selected)
def run_command(self): """ Run the command. Read the Table from the DataStore Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. pv_DataStoreID = self.get_parameter_value("DataStoreID") pv_DataStoreTable = self.get_parameter_value("DataStoreTable") pv_Sql = self.get_parameter_value("Sql") pv_SqlFile = self.get_parameter_value("SqlFile") pv_Top = self.get_parameter_value("Top") pv_TableID = self.get_parameter_value("TableID") pv_IncludeColumns = self.get_parameter_value("IncludeColumns", default_value="*") pv_ExcludeColumns = self.get_parameter_value("ExcludeColumns", default_value="") # Expand for ${Property} syntax. pv_DataStoreID = self.command_processor.expand_parameter_value( pv_DataStoreID, self) pv_DataStoreTable = self.command_processor.expand_parameter_value( pv_DataStoreTable, self) pv_Sql = self.command_processor.expand_parameter_value(pv_Sql, self) pv_TableID = self.command_processor.expand_parameter_value( pv_TableID, self) # Convert the IncludeColumns and ExcludeColumns parameter values to lists. cols_to_include = string_util.delimited_string_to_list( pv_IncludeColumns) cols_to_exclude = string_util.delimited_string_to_list( pv_ExcludeColumns) # If available, convert the SqlFile parameter value relative path to an absolute path and expand for # ${Property} syntax. if pv_SqlFile: pv_SqlFile = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_SqlFile, self))) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_read_table(pv_SqlFile, pv_TableID, pv_DataStoreID): try: # Get the DataStore object datastore = self.command_processor.get_datastore( pv_DataStoreID) # Set the SQL statement to None until proof that SQL statement exists. sql_statement = None # If using the Sql method, the sql_statement is the user-provided sql statement. if pv_Sql: sql_statement = pv_Sql if '%' in sql_statement: sql_statement = sql_statement.replace('%', '%%') # If using the Sql method, the sql_statement is the user-provided sql statement within a file. if pv_SqlFile: # Get the SQL statement from the file. f = open(pv_SqlFile, 'r') sql_statement = f.read().strip() if '%' in sql_statement: sql_statement = sql_statement.replace('%', '%%') # Create the Table from the DataStore. table = self.__read_table_from_datastore( datastore, pv_DataStoreTable, pv_TableID, pv_Top, sql_statement, cols_to_include, cols_to_exclude) # Add the table to the GeoProcessor's Tables list. self.command_processor.add_table(table) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error reading Table {} from DataStore ({}).".format( pv_TableID, pv_DataStoreID) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Extract the compressed file. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the File and the DeleteFile parameter values. pv_File = self.get_parameter_value("File") pv_DeleteFile = self.get_parameter_value("DeleteFile", default_value="False") # Convert the File parameter value relative path to an absolute path. Expand for ${Property} syntax. file_abs = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_File, self))) # Get the FileType parameter value. default_file_ext = self.__get_default_file_type(file_abs) pv_FileType = self.get_parameter_value("FileType", default_value=default_file_ext) # Get the OutputFolder parameter value. parent_folder = io_util.get_path(file_abs) pv_OutputFolder = self.get_parameter_value("OutputFolder", default_value=parent_folder) # Convert the OutputFolder parameter value relative path to an absolute path. Expand for ${Property} syntax. output_folder_abs = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_OutputFolder, self))) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_extract_file(file_abs, output_folder_abs, pv_FileType): try: # If the file is a .zip file, extract the zip file. if pv_FileType.upper() == "ZIP": zip_util.unzip_all_files(file_abs, output_folder_abs) # If the file is a .tar file, extract the tar file. elif pv_FileType.upper() == "TAR": zip_util.untar_all_files(file_abs, output_folder_abs) # If configured, remove the input compressed file. if string_util.str_to_bool(pv_DeleteFile): os.remove(file_abs) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error extracting the {} file ({}).".format( pv_FileType, pv_File) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Write the Table to an excel file. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values except for the OutputCRS pv_TableID = self.get_parameter_value("TableID") pv_OutputFile = self.get_parameter_value("OutputFile") pv_OutputWorksheet = self.get_parameter_value("OutputWorksheet") pv_ColumnsToInclude = self.get_parameter_value("ColumnsToInclude", default_value="*") pv_ColumnsToExclude = self.get_parameter_value("ColumnsToExclude", default_value="") pv_WriteIndexColumn = self.get_parameter_value("WriteIndexColumn", default_value="True") # Convert the Boolean parameters from string to valid Boolean values. pv_WriteIndexColumn = string_util.str_to_bool(pv_WriteIndexColumn) # Convert the ColumnsToInclude and ColumnsToExclude parameter values to lists. cols_to_include = string_util.delimited_string_to_list( pv_ColumnsToInclude) cols_to_exclude = string_util.delimited_string_to_list( pv_ColumnsToExclude) # Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax output_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_OutputFile, self))) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_write_table(pv_TableID, output_file_absolute): try: # Get the Table object table = self.command_processor.get_table(pv_TableID) # Get a list of all the available column names in the Table. all_cols_names = list(table.df) # Sort the list to create a second list that only includes the attributes that should be removed. cols_to_keep = string_util.filter_list_of_strings( list(table.df), cols_to_include, cols_to_exclude, return_inclusions=True) # For the columns configured to be written, order them in the same order that they were in with the # original table. This step ensures that the output table product has the columns in the same order as # the user would expect. sorted_cols_to_keep = [] for col in all_cols_names: if col in cols_to_keep: sorted_cols_to_keep.append(col) # Write the tables to an Excel file. pandas_util.write_df_to_excel(table.df, output_file_absolute, pv_OutputWorksheet, sorted_cols_to_keep, pv_WriteIndexColumn) # Add the output file to the GeoProcessor's list of output files. self.command_processor.add_output_file(output_file_absolute) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error writing Table {} to Excel workbook file {}.".format( pv_TableID, pv_OutputFile) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)