def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: None. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning = "" # Check that parameter GeoLayerID is a non-empty, non-None string. pv_GeoLayerID = self.get_parameter_value( parameter_name='GeoLayerID', command_parameters=command_parameters) if not validators.validate_string(pv_GeoLayerID, False, False): message = "GeoLayerID parameter has no value." recommendation = "Specify the GeoLayerID parameter to indicate the GeoLayer to copy." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check that optional parameter IfGeoLayerIDExists is either `Replace`, `ReplaceAndWarn`, `Warn`, `Fail`, None. pv_IfGeoLayerIDExists = self.get_parameter_value( parameter_name="IfGeoLayerIDExists", command_parameters=command_parameters) acceptable_values = ["Replace", "ReplaceAndWarn", "Warn", "Fail"] if not validators.validate_string_in_list(pv_IfGeoLayerIDExists, acceptable_values, none_allowed=True, empty_string_allowed=True, ignore_case=True): message = "IfGeoLayerIDExists parameter value ({}) is not recognized.".format( pv_IfGeoLayerIDExists) recommendation = "Specify one of the acceptable values ({}) for the IfGeoLayerIDExists parameter.".format( acceptable_values) warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty triggers an exception below. warning = command_util.validate_command_parameter_names(self, warning) # If any warnings were generated, throw an exception. if len(warning) > 0: self.logger.warning(warning) raise ValueError(warning) else: # Refresh the phase severity self.command_status.refresh_phase_severity( CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def __should_read_geolayer(self, geolayer_id): """ Checks the following: * the ID of the output GeoLayer is unique (not an existing GeoLayer ID) Args: geolayer_id: the ID of the output GeoLayer Returns: run_read: Boolean. If TRUE, the GeoLayer read process should be run. If FALSE, the read process should not be run. """ # Boolean to determine if the read process should be run. Set to true until an error occurs. run_read = True # If the GeoLayerID is the same as an already-registered GeoLayerID, react according to the # pv_IfGeoLayerIDExists value. if self.command_processor.get_geolayer(geolayer_id): # Get the IfGeoLayerIDExists parameter value. pv_IfGeoLayerIDExists = self.get_parameter_value("IfGeoLayerIDExists", default_value="Replace") # Warnings/recommendations if the GeolayerID is the same as a registered GeoLayerID. message = 'The GeoLayerID ({}) value is already in use as a GeoLayer ID.'.format(geolayer_id) recommendation = 'Specify a new GeoLayerID.' # The registered GeoLayer should be replaced with the new GeoLayer (with warnings). if pv_IfGeoLayerIDExists.upper() == "REPLACEANDWARN": self.warning_count += 1 self.logger.warning(message) self.command_status.add_to_log(CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.WARNING, message, recommendation)) # The registered GeoLayer should not be replaced. A warning should be logged. if pv_IfGeoLayerIDExists.upper() == "WARN": run_read = False self.warning_count += 1 self.logger.warning(message) self.command_status.add_to_log(CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.WARNING, message, recommendation)) # The matching IDs should cause a FAILURE. elif pv_IfGeoLayerIDExists.upper() == "FAIL": run_read = False self.warning_count += 1 self.logger.error(message) self.command_status.add_to_log(CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Return the Boolean to determine if the read process should be run. If TRUE, all checks passed. If FALSE, # one or many checks failed. return run_read
def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: None. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning = "" logger = logging.getLogger(__name__) # Unlike most commands, set internal data here because it is needed by initial call to next() # before calls to run_command # Name is required pv_Name = self.get_parameter_value( parameter_name='Name', command_parameters=command_parameters) if not validators.validate_string(pv_Name, False, False): message = "A name for the If block must be specified" recommendation = "Specify the Name." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Condition is required pv_Condition = self.get_parameter_value( parameter_name='Condition', command_parameters=command_parameters) if not validators.validate_string(pv_Condition, False, False): message = "A condition for the If command must be specified" recommendation = "Specify the condition." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty # triggers an exception below. warning = command_util.validate_command_parameter_names(self, warning) # If any warnings were generated, throw an exception if len(warning) > 0: logger.warn(warning) raise ValueError(warning) # Refresh the phase severity self.command_status.refresh_phase_severity( CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: None. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning = "" # Check that parameter URL is a non-empty, non-None string. # - existence of the url will also be checked in run_command(). pv_URL = self.get_parameter_value( parameter_name='URL', command_parameters=command_parameters) if not validators.validate_string(pv_URL, False, False): message = "URL parameter has no value." recommendation = "Specify the URL parameter to indicate the URL of the file to download." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check that parameter OutputFile is a non-empty string (can be None). # - existence of the folder will also be checked in run_command(). pv_OutputFile = self.get_parameter_value( parameter_name='OutputFile', command_parameters=command_parameters) if not validators.validate_string(pv_OutputFile, True, False): message = "OutputFile parameter has no value." recommendation = "Specify the OutputFile parameter to indicate the output file." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty triggers an exception below. warning = command_util.validate_command_parameter_names(self, warning) # If any warnings were generated, throw an exception. if len(warning) > 0: self.logger.warning(warning) raise ValueError(warning) else: # Refresh the phase severity self.command_status.refresh_phase_severity( CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: None. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning = "" # Check that parameter GeoLayerIDs is a non-empty, non-None string. pv_GeoLayerIDs = self.get_parameter_value( parameter_name='GeoLayerIDs', command_parameters=command_parameters) if not validators.validate_string(pv_GeoLayerIDs, False, False): message = "GeoLayerIDs parameter has no value." recommendation = "Specify the GeoLayerIDs parameter to indicate the GeoLayers to merge." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check that parameter OutputGeoLayerID is a non-empty, non-None string. pv_OutputGeoLayerID = self.get_parameter_value( parameter_name='OutputGeoLayerID', command_parameters=command_parameters) if not validators.validate_string(pv_OutputGeoLayerID, False, False): message = "OutputGeoLayerID parameter has no value." recommendation = "Specify the OutputGeoLayerID parameter to indicate the ID of the merged GeoLayer." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty triggers an exception below. warning = command_util.validate_command_parameter_names(self, warning) # If any warnings were generated, throw an exception. if len(warning) > 0: self.logger.warning(warning) raise ValueError(warning) # Refresh the phase severity self.command_status.refresh_phase_severity( CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: Nothing. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning = "" logger = logging.getLogger("gp") # Message is required pv_Message = self.get_parameter_value( parameter_name='Message', command_parameters=command_parameters) if not validators.validate_string(pv_Message, False, False): message = "Message parameter has no value." recommendation = "Specify text for the Message parameter." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) pv_CommandStatus = self.get_parameter_value( parameter_name='CommandStatus', command_parameters=command_parameters) if not validators.validate_string_in_list( pv_CommandStatus, CommandStatusType.get_command_status_types_as_str(), True, True): message = 'The requested command status "' + pv_CommandStatus + '"" is invalid.' recommendation = "Specify a valid command status." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty # triggers an exception below. warning = command_util.validate_command_parameter_names(self, warning) # If any warnings were generated, throw an exception if len(warning) > 0: logger.warning(warning) raise ValueError(warning) # Refresh the phase severity self.command_status.refresh_phase_severity( CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: None. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning = "" # Check that parameter SpatialDataFile is a non-empty, non-None string. # - existence of the file will also be checked in run_command(). pv_SpatialDataFile = self.get_parameter_value(parameter_name='SpatialDataFile', command_parameters=command_parameters) if not validators.validate_string(pv_SpatialDataFile, False, False): message = "SpatialDataFile parameter has no value." recommendation = "Specify the SpatialDataFile parameter to indicate the spatial data layer file." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check that optional parameter IfGeoLayerIDExists is one of the acceptable values or is None. pv_IfGeoLayerIDExists = self.get_parameter_value(parameter_name="IfGeoLayerIDExists", command_parameters=command_parameters) if not validators.validate_string_in_list(pv_IfGeoLayerIDExists, self.__choices_IfGeoLayerIDExists, none_allowed=True, empty_string_allowed=True, ignore_case=True): message = "IfGeoLayerIDExists parameter value ({}) is not recognized.".format(pv_IfGeoLayerIDExists) recommendation = "Specify one of the acceptable values ({}) for the IfGeoLayerIDExists parameter.".format( self.__choices_IfGeoLayerIDExists) warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty triggers an exception below. warning = command_util.validate_command_parameter_names(self, warning) # If any warnings were generated, throw an exception. if len(warning) > 0: self.logger.warning(warning) raise ValueError(warning) else: # Refresh the phase severity self.command_status.refresh_phase_severity(CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: Nothing. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning_message = "" logger = logging.getLogger(__name__) # CommandFile is required pv_CommandFile = self.get_parameter_value(parameter_name='CommandFile', command_parameters=command_parameters) if not validators.validate_string(pv_CommandFile, False, False): message = "The CommandFile must be specified." recommendation = "Specify the command file." warning_message += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # ExpectedStatus is optional, will default to Success at runtime pv_ExpectedStatus = self.get_parameter_value(parameter_name='ExpectedStatus', command_parameters=command_parameters) if not validators.validate_string_in_list(pv_ExpectedStatus, self.__choices_ExpectedStatus, True, True): message = "ExpectedStatus parameter is invalid." recommendation = "Specify the ExpectedStatus parameter as blank or one of " + \ str(self.__choices_ExpectedStatus) warning_message += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty # triggers an exception below. warning_message = command_util.validate_command_parameter_names(self, warning_message) # If any warnings were generated, throw an exception if len(warning_message) > 0: logger.warning(warning_message) raise ValueError(warning_message) # Refresh the phase severity self.command_status.refresh_phase_severity(CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: Nothing. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning_message = "" logger = logging.getLogger(__name__) # SearchFolder is required pv_SearchFolder = self.get_parameter_value(parameter_name='SearchFolder', command_parameters=command_parameters) if not validators.validate_string(pv_SearchFolder, False, False): message = "SearchFolder parameter has no value." warning_message += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, "Specify the search folder.")) print(message) # FilenamePattern is optional, will default to "test-*" at runtime # OutputFile is required pv_OutputFile = self.get_parameter_value(parameter_name='OutputFile', command_parameters=command_parameters) if not validators.validate_string(pv_OutputFile, False, False): message = "OutputFile parameter has no value." warning_message += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, "Specify the output file.")) print(message) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty # triggers an exception below. warning_message = command_util.validate_command_parameter_names(self, warning_message) # If any warnings were generated, throw an exception if len(warning_message) > 0: logger.warning(warning_message) raise ValueError(warning_message) # Refresh the phase severity self.command_status.refresh_phase_severity(CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def __should_write_geolayer(self, geolayer_id, output_file_abs): """ Checks the following: * the ID of the GeoLayer is an existing GeoLayer ID * the output folder is a valid folder Args: geolayer_id: the ID of the GeoLayer to be written output_file_abs: the full pathname to the output file Returns: run_write: Boolean. If TRUE, the writing process should be run. If FALSE, it should not be run. """ # Boolean to determine if the writing process should be run. Set to true until an error occurs. run_write = True # Boolean to determine if the output format parameters have valid bool values. Set to true until proven false. valid_output_bool = True # If the GeoLayer ID is not an existing GeoLayer ID, raise a FAILURE. if not self.command_processor.get_geolayer(geolayer_id): run_write = False self.warning_count += 1 message = 'The GeoLayerID ({}) is not a valid GeoLayer ID.'.format( geolayer_id) recommendation = 'Specify a valid GeoLayerID.' self.logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # If the OutputFolder is not a valid folder, raise a FAILURE. output_folder = os.path.dirname(output_file_abs) if not os.path.isdir(output_folder): run_write = False self.warning_count += 1 message = 'The output folder ({}) of the OutputFile is not a valid folder.'.format( output_folder) recommendation = 'Specify a valid relative pathname for the output file.' self.logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Return the Boolean to determine if the write process should be run. If TRUE, all checks passed. If FALSE, # one or many checks failed. return run_write
def __should_run_webget(self, output_file_abs): """ Checks the following: * the output folder is a valid folder Args: output_file_abs: the full pathname to the output file Returns: run_webget: Boolean. If TRUE, the webget process should be run. If FALSE, it should not be run. """ # Boolean to determine if the webget process should be run. Set to true until an error occurs. run_webget = True # If the OutputFolder is not a valid folder, raise a FAILURE. output_folder = os.path.dirname(output_file_abs) if not os.path.isdir(output_folder): run_webget = False self.warning_count += 1 message = 'The output folder ({}) of the OutputFile is not a valid folder.'.format( output_folder) recommendation = 'Specify a valid relative pathname for the output file.' self.logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Return the Boolean to determine if the webget process should be run. If TRUE, all checks passed. If FALSE, # one or many checks failed. return run_webget
def __should_read_folder(self, spatial_data_folder_abs): """ Checks the following: * the SpatialDataFolder (absolute) is a valid folder Args: spatial_data_folder_abs: the full pathname to the input spatial data folder Returns: run_read: Boolean. If TRUE, the folder read process should be run. If FALSE, it should not be run. """ # Boolean to determine if the read process should be run. Set to true until an error occurs. run_read = True # If the input spatial data folder is not a valid file path, raise a FAILURE. if not os.path.isdir(spatial_data_folder_abs): run_read = False self.warning_count += 1 message = "The SpatialDataFolder ({}) is not a valid folder.".format(spatial_data_folder_abs) recommendation = "Specify a valid folder." self.logger.error(message) self.command_status.add_to_log(CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Return the Boolean to determine if the read process should be run. If TRUE, all checks passed. If FALSE, # one or many checks failed. return run_read
def __should_extract_file(self, file_abs, output_folder_abs, file_type): """ Checks the following: * the File is a valid file * the OutputFolder is a valid folder * the FileType correctly identifies the File's type Args: file_abs (str): the full path to the input compressed File output_folder_abs(str): the full path to the OutputFolder file_type(str): the FileType value depicting the file type of the input File Returns: Boolean. If TRUE, the file should be extracted. If FALSE, at least one check failed and the file should not be extracted. """ # List of Boolean values. The Boolean values correspond to the results of the following tests. If TRUE, the # test confirms that the command should be run. should_run_command = [] # If the File parameter value is not a valid file, raise a FAILURE. should_run_command.append( validators.run_check(self, "IsFilePathValid", "File", file_abs, "FAIL")) # If the OutputFolder parameter value is not a valid folder, raise a FAILURE. should_run_command.append( validators.run_check(self, "IsFolderPathValid", "OutputFolder", output_folder_abs, "FAIL")) # If the File Type is not recognized, raise a FAILURE. if file_type is None: message = "A valid FileType cannot be determined from the file ({}).".format( file_abs) recommendation = "Use the FileType parameter to assign the appropriate file type." should_run_command.append(False) self.logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # If the File Type is not actually recognized by the input File, raise a FAILURE. if file_type.upper() == "ZIP": should_run_command.append( validators.run_check(self, "IsZipFile", "File", file_abs, "FAIL")) elif file_type.upper() == "TAR": should_run_command.append( validators.run_check(self, "IsTarFile", "File", file_abs, "FAIL")) # Return the Boolean to determine if the process should be run. if False in should_run_command: return False else: return True
def run_command(self): """ Run the command. Remove the attribute from the GeoLayer. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. pv_GeoLayerID = self.get_parameter_value("GeoLayerID") pv_AttributeNames = self.get_parameter_value("AttributeNames") # Convert the AttributeNames parameter from a string to a list. attribute_names_list = string_util.delimited_string_to_list( pv_AttributeNames) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_attribute_be_removed(pv_GeoLayerID, attribute_names_list): # Run the process. try: # Get the input GeoLayer. input_geolayer = self.command_processor.get_geolayer( pv_GeoLayerID) # Iterate over the input attributes to remove. for attribute_name in attribute_names_list: # Remove the attribute from the GeoLayer. input_geolayer.remove_attribute(attribute_name) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error removing attribute(s) ({}) from GeoLayer {}.".format( pv_AttributeNames, pv_GeoLayerID) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def __should_write_table2(self, datastore, datastore_table_name, datastore_table_cols_to_receive, writemode): """ Checks the following: * the datastore columns configured to receive data are existing columns within the DataStore table Args: datastore (obj): the DataStore that is receiving the data datastore_table_name (str): the name of the DataStore table that is receiving the data datastore_table_cols_to_receive (list of strings): A list of DataBase table columns that are expected to receive data. Returns: Boolean. If TRUE, the process should be run. If FALSE, it should not be run. """ # List of Boolean values. The Boolean values correspond to the results of the following tests. If TRUE, the # test confirms that the command should be run. should_run_command = [] # A list of Table columns that do not map to the DataStore Table columns. invalid_columns = [] # Do not run this check if the WriteMode is NewTableInsert or ExistingTableOverwrite exception_modes = ["NEWTABLEINSERT", "EXISTINGTABLEOVERWRITE"] if writemode.upper() not in exception_modes: # Get the DataStore Table columns. datastore_table_cols = datastore.return_col_names( datastore_table_name) # Iterate over the DataStore columns that are configured to read data. for datastore_table_col_to_receive in datastore_table_cols_to_receive: # If the configured DataStore column does not exist in the DataStore table, add the configured # DataStore column to the list of invalid columns. if datastore_table_col_to_receive not in datastore_table_cols: invalid_columns.append(datastore_table_col_to_receive) # If there are any invalid configured DataStore columns, raise a FAILURE. if invalid_columns: message = "One or more of the DataStore columns configured to be edited do(es) not exist in the DataStore" \ " table ({}). The invalid columns are: \n({}).".format(datastore_table_name, invalid_columns) recommendation = "Specify valid DataStore columns to edit." self.logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) should_run_command.append(False) # Return the Boolean to determine if the process should be run. if False in should_run_command: self.warning_count += 1 return False else: return True
def run_command(self): """ Run the command. Set the GeoProcessor property to that of the specified GeoLayer property. Returns: Nothing. Raises: RuntimeError if an exception occurs, for example if the property name is not found. """ warning_count = 0 logger = logging.getLogger(__name__) pv_GeoLayerID = self.get_parameter_value("GeoLayerID") pv_GeoLayerPropertyName = self.get_parameter_value('GeoLayerPropertyName') pv_PropertyName = self.get_parameter_value('PropertyName') try: # Get the GeoLayer object geolayer = self.command_processor.get_geolayer(pv_GeoLayerID) if geolayer is None: message = 'Unable to find GeoLayer for GeoLayerID="' + pv_GeoLayerID + '"' warning_count += 1 self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "Check the log file for details.")) else: # First get the property from the GeoLayer property_value = geolayer.get_property(pv_GeoLayerPropertyName) if property_value is not None: self.command_processor.set_property(pv_PropertyName, property_value) except Exception as e: warning_count += 1 message = 'Unexpected error setting property "' + pv_PropertyName + '"' logger.exception(message, e) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "Check the log file for details.")) if warning_count > 0: message = "There were " + str(warning_count) + " warnings processing the command." raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Close an existing DataStore and remove it from the GeoProcessor. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the DataStoreID parameter value and expand for ${Property} syntax. pv_DataStoreID = self.get_parameter_value("DataStoreID") pv_DataStoreID = self.command_processor.expand_parameter_value( pv_DataStoreID, self) # Obtain the StatusMessage parameter value and expand for ${Property} syntax. pv_StatusMessage = self.get_parameter_value("StatusMessage") pv_StatusMessage = self.command_processor.expand_parameter_value( pv_StatusMessage, self) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_close_datastore(pv_DataStoreID): try: # Get the DataStore object datastore_obj = self.command_processor.get_datastore( pv_DataStoreID) # Close the database connection. datastore_obj.close_db_connection() # Update the DataStore's status message. datastore_obj.update_status_message(pv_StatusMessage) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error closing DataStore {}.".format( pv_DataStoreID) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Write the GeoLayer to a spatial data file in KML format. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the parameter values. pv_GeoLayerID = self.get_parameter_value("GeoLayerID") pv_OutputFile = self.get_parameter_value("OutputFile") pv_PlacemarkNameAttribute = self.get_parameter_value("PlacemarkNameAttribute") pv_PlacemarkDescriptionAttribute = self.get_parameter_value("PlacemarkDescriptionAttribute") # Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax output_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_OutputFile, self))) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_write_geolayer(pv_GeoLayerID, output_file_absolute): try: # Get the GeoLayer geolayer = self.command_processor.get_geolayer(pv_GeoLayerID) # Write the GeoLayer to a spatial data file in KML format # "Note that KML by specification uses only a single projection, EPSG:4326. All OGR KML output will be # presented in EPSG:4326. As such OGR will create layers in the correct coordinate system and transform # any geometries." - www.gdal.org/drv_kml.html qgis_util.write_qgsvectorlayer_to_kml(geolayer.qgs_vector_layer, output_file_absolute, "EPSG:4326", pv_PlacemarkNameAttribute, pv_PlacemarkDescriptionAttribute, "clampToGround") # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error writing GeoLayer {} to GeoJSON format.".format(pv_GeoLayerID) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log(CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format(self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def __should_write_table(self, table_id, output_file_abs): """ Checks the following: * the ID of the Table is an existing Table ID * the output folder is a valid folder Args: table_id: the ID of the Table to be written output_file_abs: the full pathname to the output file Returns: run_write: Boolean. If TRUE, the writing process should be run. If FALSE, it should not be run. """ # List of Boolean values. The Boolean values correspond to the results of the following tests. If TRUE, the # test confirms that the command should be run. should_run_command = [] # If the Table ID is not an existing Table ID, raise a FAILURE. should_run_command.append( validators.run_check(self, "IsTableIdExisting", "TableID", table_id, "FAIL")) # Get the full path to the output folder output_folder_abs = io_util.get_path(output_file_abs) # If the output folder is not an existing folder, raise a FAILURE. should_run_command.append( validators.run_check(self, "IsFolderPathValid", "OutputFile", output_folder_abs, "FAIL")) # Continue if the output file is an existing file. if os.path.exists(output_folder_abs): if io_util.get_extension(output_file_abs).upper() == ".XLS": message = 'At the current time, a Table object cannot be appended to or overwrite an existing Excel ' \ 'file in XLS format.' recommendation = "Update the XLS file ({}) to an XLSX file or write the table " \ "to a new XLS file.".format(output_file_abs) self.warning_count += 1 self.logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) should_run_command.append(False) # Return the Boolean to determine if the process should be run. if False in should_run_command: return False else: return True
def run_command(self): """ Run the command. Open a file to receive regression test results. Returns: None. Raises: RuntimeError: if a runtime input error occurs. """ warning_count = 0 logger = logging.getLogger(__name__) # Get data for the command pv_OutputFile = self.get_parameter_value('OutputFile') # Runtime checks on input pv_OutputFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_OutputFile, self))) if warning_count > 0: message = "There were " + str(warning_count) + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Open the regression test results file try: pv_OutputFile_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_OutputFile, self))) self.__open_new_regression_test_report_file(pv_OutputFile_absolute, False) # Do not append logger.info('Opened regression test results report file "' + pv_OutputFile_absolute + '"') except Exception as e: warning_count += 1 message = 'Unexpected error opening file "' + pv_OutputFile_absolute + '"' logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) if warning_count > 0: message = "There were " + str(warning_count) + " warnings processing the command." logger.warning(message) raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: Nothing. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning = "" logger = logging.getLogger(__name__) # LogFile is required pv_LogFile = self.get_parameter_value(parameter_name='LogFile', command_parameters=command_parameters) if not validators.validate_string(pv_LogFile, False, False): message = "The log file must be specified." recommendation = "Specify the log file." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) print(message) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty # triggers an exception below. warning = command_util.validate_command_parameter_names(self, warning) # If any warnings were generated, throw an exception if len(warning) > 0: logger.warning(warning) raise ValueError(warning) # Refresh the phase severity self.command_status.refresh_phase_severity(CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Restart the lof file with the given name. Subsequent logging messages will go to the file until another StartLog() command is run. Returns: Nothing. Raises: RuntimeError if any exception occurs running the command. """ logger = logging.getLogger(__name__) warning_count = 0 pv_LogFile = self.get_parameter_value('LogFile') log_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_LogFile))) try: # Change the GeoProcessor logger to use the specified file # - The initial application log file will be closed. log_util.reset_log_file_handler(log_file_absolute) # Add the log file to output self.command_processor.add_output_file(log_file_absolute) except Exception as e: warning_count += 1 message = 'Unexpected error (re)starting log file "' + log_file_absolute + '"' logger.exception(message, e) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "Check the old log file or command window for details.")) if warning_count > 0: message = "There were " + str(warning_count) + " warnings processing the command." raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Print the message to the log file. Returns: Nothing. Raises: RuntimeError if any exception occurs. """ logger = logging.getLogger(__name__) warning_count = 0 # Message parameter won't be null. pv_Message = self.get_parameter_value('Message') pv_CommandStatus = self.get_parameter_value('CommandStatus') if pv_CommandStatus is None or pv_CommandStatus == "": # Default status as a string pv_commandStatus = str(CommandStatusType.SUCCESS) # Convert the string to the enum command_status_type = CommandStatusType.value_of(pv_CommandStatus, ignore_case=True) message_expanded = self.command_processor.expand_parameter_value( pv_Message) logger.info(message_expanded) # Add a log message for the requested status type # - don't add to the warning count self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(command_status_type, message_expanded, "")) if warning_count > 0: message = "There were " + str( warning_count) + " warnings processing the command." raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Read the feature classes within a file geodatabase. For each desired feature class (can be specified by the Subset_Pattern parameter), create a GeoLayer object, and add to the GeoProcessor's geolayer list. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the required and optional parameter values pv_SpatialDataFolder = self.get_parameter_value("SpatialDataFolder") pv_ReadOnlyOneFeatureClass = self.get_parameter_value( "ReadOnlyOneFeatureClass") pv_Subset_Pattern = self.get_parameter_value("Subset_Pattern") pv_GeoLayerID_prefix = self.get_parameter_value("GeoLayerID_prefix") pv_GeoLayerID = self.get_parameter_value("GeoLayerID") pv_FeatureClass = self.get_parameter_value("FeatureClass") # Convert the ReadOnlyOneFeatureClass from a string value to a Boolean value. pv_ReadOnlyOneFeatureClass = string_util.str_to_bool( pv_ReadOnlyOneFeatureClass) # Convert the SpatialDataFolder parameter value relative path to an absolute path sd_folder_abs = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_SpatialDataFolder, self))) # Run the initial checks on the parameter values. Only continue if the checks passed. if self.__should_read_gdb(sd_folder_abs): # If configured to only read one Feature Class into one GeoLayer. if pv_ReadOnlyOneFeatureClass: # Run the check to see if the GeoLayer should be read. if self.__should_read_geolayer(pv_GeoLayerID, True, pv_FeatureClass, sd_folder_abs): try: # Get the full pathname to the feature class # TODO egiles 2018-01-04 Need to research how to properly document feature class source path spatial_data_file_absolute = os.path.join( sd_folder_abs, str(pv_FeatureClass)) # Create a QgsVectorLayer object from the feature class. qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_feature_class( sd_folder_abs, pv_FeatureClass) # Create a GeoLayer and add it to the geoprocessor's GeoLayers list geolayer_obj = GeoLayer(pv_GeoLayerID, qgs_vector_layer, spatial_data_file_absolute) self.command_processor.add_geolayer(geolayer_obj) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error reading feature class ({}) from file geodatabase ({}).".format( pv_FeatureClass, sd_folder_abs) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # If configured to read multiple Feature Classes into multiple GeoLayers. else: # Get a list of all of the feature classes in the file geodatabase. fc_list = self.__return_a_list_of_fc(sd_folder_abs) # Filter the fc_list to only include feature classes that meet the Subset Pattern configuration. If # the Subset Pattern configuration is None, all feature classes will remain in the fc_list. fc_list = string_util.filter_list_of_strings( fc_list, [pv_Subset_Pattern]) # Iterate over the feature classes in the fc_list. for feature_class in fc_list: # Determine the GeoLayerID. if pv_GeoLayerID_prefix: geolayer_id = "{}_{}".format(pv_GeoLayerID_prefix, feature_class) else: geolayer_id = feature_class # Run the secondary checks on the parameter values. Only continue if the checks passed. if self.__should_read_geolayer(geolayer_id, False): try: # Get the full pathname to the feature class # TODO egiles 2018-01-04 Need to research how to properly document feature class source path spatial_data_file_absolute = os.path.join( sd_folder_abs, str(feature_class)) # Create a QgsVectorLayer object from the feature class. qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_feature_class( sd_folder_abs, feature_class) # Create a GeoLayer and add it to the geoprocessor's GeoLayers list geolayer_obj = GeoLayer( geolayer_id, qgs_vector_layer, spatial_data_file_absolute) self.command_processor.add_geolayer(geolayer_obj) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error reading feature class ({}) from file geodatabase ({}).".format( feature_class, sd_folder_abs) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def __should_attribute_be_removed(self, geolayer_id, attribute_names): """ Checks the following: * The ID of the input GeoLayer is an actual GeoLayer (if not, log an error message and do not continue.) * The attribute names are valid names for the GeoLayer (if not, log an error message and do not continue.) Args: geolayer_id: the ID of the GeoLayer with the attribute to remove attribute_names (list of strings): the names of the attributes to remove from the GeoLayer Returns: remove_attribute: Boolean. If TRUE, the attribute should be removed from the GeoLayer. If FALSE, a check has failed and the attribute should not be removed. """ # Boolean to determine if the attribute should be removed. Set to TRUE until one or many checks fail. remove_attribute = True # If the input GeoLayer does not exist, raise a FAILURE. if not self.command_processor.get_geolayer(geolayer_id): # Boolean to determine if the attribute should be removed. remove_attribute = False self.warning_count += 1 message = 'The input GeoLayer ID ({}) does not exist.'.format( geolayer_id) recommendation = 'Specify a valid GeoLayerID.' self.logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # If the input GeoLayer does exist, continue with the checks. else: # Get the input GeoLayer object. input_geolayer = self.command_processor.get_geolayer(geolayer_id) # Get the existing attribute names of the input GeoLayer. list_of_existing_attributes = input_geolayer.get_attribute_field_names( ) # Create a list of invalid input attribute names. An invalid attribute name is an input attribute name # that is not matching any of the existing attribute names of the GeoLayer. invalid_attrs = (attr_name for attr_name in attribute_names if attr_name not in list_of_existing_attributes) # Iterate over the invalid input attribute names and raise a FAILURE for each. for invalid_attr in invalid_attrs: remove_attribute = False self.warning_count += 1 message = 'The attribute name ({}) is not valid.'.format( invalid_attr) recommendation = 'Specify a valid attribute name. Valid attributes for this layer are' \ ' as follows: {}'.format(list_of_existing_attributes) self.logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Return the Boolean to determine if the attribute should be removed. If TRUE, all checks passed. If FALSE, # one or many checks failed. return remove_attribute
def run_command(self): """ Run the command. Extract the compressed file. Returns: None. Raises: RuntimeError if any warnings occurred during run_command method. """ # Obtain the File and the DeleteFile parameter values. pv_File = self.get_parameter_value("File") pv_DeleteFile = self.get_parameter_value("DeleteFile", default_value="False") # Convert the File parameter value relative path to an absolute path. Expand for ${Property} syntax. file_abs = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value(pv_File, self))) # Get the FileType parameter value. default_file_ext = self.__get_default_file_type(file_abs) pv_FileType = self.get_parameter_value("FileType", default_value=default_file_ext) # Get the OutputFolder parameter value. parent_folder = io_util.get_path(file_abs) pv_OutputFolder = self.get_parameter_value("OutputFolder", default_value=parent_folder) # Convert the OutputFolder parameter value relative path to an absolute path. Expand for ${Property} syntax. output_folder_abs = io_util.verify_path_for_os( io_util.to_absolute_path( self.command_processor.get_property('WorkingDir'), self.command_processor.expand_parameter_value( pv_OutputFolder, self))) # Run the checks on the parameter values. Only continue if the checks passed. if self.__should_extract_file(file_abs, output_folder_abs, pv_FileType): try: # If the file is a .zip file, extract the zip file. if pv_FileType.upper() == "ZIP": zip_util.unzip_all_files(file_abs, output_folder_abs) # If the file is a .tar file, extract the tar file. elif pv_FileType.upper() == "TAR": zip_util.untar_all_files(file_abs, output_folder_abs) # If configured, remove the input compressed file. if string_util.str_to_bool(pv_DeleteFile): os.remove(file_abs) # Raise an exception if an unexpected error occurs during the process except Exception as e: self.warning_count += 1 message = "Unexpected error extracting the {} file ({}).".format( pv_FileType, pv_File) recommendation = "Check the log file for details." self.logger.error(message, exc_info=True) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Determine success of command processing. Raise Runtime Error if any errors occurred if self.warning_count > 0: message = "There were {} warnings proceeding this command.".format( self.warning_count) raise RuntimeError(message) # Set command status type as SUCCESS if there are no errors. else: self.command_status.refresh_phase_severity( CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: None. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning = "" # Check that either the parameter File is a non-empty, non-None string. pv_File = self.get_parameter_value( parameter_name='File', command_parameters=command_parameters) if not validators.validate_string(pv_File, False, False): message = "File parameter has no value." recommendation = "Specify the File parameter to indicate the compressed file to extract." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check that optional parameter FileType is an acceptable value or is None. pv_FileType = self.get_parameter_value( parameter_name="FileType", command_parameters=command_parameters) acceptable_values = ["Zip", "Tar"] if not validators.validate_string_in_list(pv_FileType, acceptable_values, none_allowed=True, empty_string_allowed=False, ignore_case=True): message = "FileType parameter value ({}) is not recognized.".format( pv_FileType) recommendation = "Specify one of the acceptable values ({}) for the" \ " FileType parameter.".format(acceptable_values) warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check that optional DeleteFile parameter value is a valid Boolean value or is None. pv_DeleteFile = self.get_parameter_value( parameter_name="DeleteFile", command_parameters=command_parameters) if not validators.validate_bool( pv_DeleteFile, none_allowed=True, empty_string_allowed=False): message = "DeleteFile parameter value ({}) is not a recognized boolean value.".format( pv_DeleteFile) recommendation = "Specify either 'True' or 'False for the DeleteFile parameter." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty triggers an exception below. warning = command_util.validate_command_parameter_names(self, warning) # If any warnings were generated, throw an exception. if len(warning) > 0: self.logger.warning(warning) raise ValueError(warning) else: # Refresh the phase severity self.command_status.refresh_phase_severity( CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Returns: Nothing. Raises: ValueError: if a runtime input error occurs. """ warning_count = 0 logger = logging.getLogger(__name__) # Get data for the command pv_SearchFolder = self.get_parameter_value('SearchFolder') pv_FilenamePattern = self.get_parameter_value('FilenamePattern') if pv_FilenamePattern is None or pv_FilenamePattern == "": # The pattern that is desired is test_*.gp, using globbing syntax pv_FilenamePattern = "[Tt][Ee][Ss][Tt]-*.gp" pv_OutputFile = self.get_parameter_value('OutputFile') # Runtime checks on input working_dir = self.command_processor.get_property('WorkingDir') logger.info('working_dir: "' + working_dir + '"') search_folder_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(working_dir, self.command_processor.expand_parameter_value(pv_SearchFolder, self))) search_folder_absolute_internal = io_util.verify_path_for_os(search_folder_absolute, always_use_forward_slashes=True) output_file_absolute = io_util.verify_path_for_os( io_util.to_absolute_path(working_dir, self.command_processor.expand_parameter_value(pv_OutputFile, self))) output_file_absolute_internal = io_util.verify_path_for_os(output_file_absolute, always_use_forward_slashes=True) logger.info('search_folder_absolute: "' + search_folder_absolute + '"') logger.info('search_folder_absolute_internal: "' + search_folder_absolute_internal + '"') if not os.path.exists(search_folder_absolute_internal): message = 'The folder to search "' + search_folder_absolute + '" does not exist.' logger.warning(message) warning_count += 1 self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "Verify that the folder exists at the time the command is run.")) if warning_count > 0: message = "There were " + str(warning_count) + " warnings about command parameters." logger.warning(message) raise ValueError(message) # Do the processing try: # Output folder is used when constructing filenames for command files to run output_folder_absolute = os.path.dirname(output_file_absolute_internal) logger.info('output_folder_absolute: "' + output_folder_absolute + '"') logger.info('output_folder_absolute_internal: "' + output_file_absolute_internal + '"') files = [] # List of files to match # Convert globbing-style wildcards to Pythonic regex logger.info('Filename pattern using globbing = "' + pv_FilenamePattern + '"') # filename_pattern_regex = pv_FilenamePattern.replace("*", "[^/]*") # Match any character of any length, making sure that special characters are dealt with filename_pattern_regex = pv_FilenamePattern.replace(".", "\\.") # So .gp is handled literally # The following is used to match between "test-" and ".gp" filename_pattern_regex = filename_pattern_regex.replace("*", ".*") logger.info('Filename pattern using Python regex = "' + filename_pattern_regex + '"') filename_pattern_regex_compiled = re.compile(filename_pattern_regex) CreateRegressionTestCommandFile.__get_matching_filenames_in_tree(files, search_folder_absolute_internal, filename_pattern_regex_compiled, pattern_string=filename_pattern_regex) # Sort the list files = sorted(files) # Open the output file... # TODO smalers 2018-10-20 decide whether to support append mode out = open(output_file_absolute_internal, "w") # Write a standard header to the file so that it is clear when the file was created io_util.print_standard_file_header(out, comment_line_prefix="#") # Include the setup command file if requested # logger.info('Adding commands from setup command file "' + setup_command_file_absolute + '"') # include_setup_file(out, setup_file_absolute, "setup") # Include the matching test cases # Python 2... # nl = os.linesep # newline character for operating system # Python 3 translates \n into the OS-specific end of line so os.linesep introduces extra end of lines nl = "\n" out.write("#" + nl) out.write("# The following " + str(len(files)) + " test cases will be run to compare results with expected results." + nl) out.write("# Individual log files are generally created for each test." + nl) # TODO smalers 2018-01-20 evaluate how to handle output table # - Currently the GeoProcessor does not have table commands table_param = "" # Use absolute path since each developer will regenerate this file. out.write('StartRegressionTestResultsReport(OutputFile="' + output_file_absolute + '.out.txt"' + table_param + ")" + nl) # Find the list of matching files... # logger.debug('output_folder_absolute"' + output_folder_absolute + '"') for a_file in files: logger.info('Adding command file "' + a_file + '"') # The command files to run are relative to the commands file being created. # - use the operating system separator command_file_to_run = io_util.verify_path_for_os( io_util.to_relative_path(output_folder_absolute, a_file)) # Determine if the command file has @expectedStatus in it. If so, define an ExpectedStatus # parameter for the command. logger.info('Relative path to command file is "' + command_file_to_run + '"') out.write('RunCommands(CommandFile="' + command_file_to_run + '"' + CreateRegressionTestCommandFile.__determine_expected_status_parameter(a_file) + ')' + nl) # TODO smalers 2018-12-30 Maybe the output file is relative to the working folder # output_file_relative = io_util.to_relative_path(working_dir, output_file_absolute) # out.write('WriteCommandSummaryToFile(OutputFile="' + output_file_relative + '.summary.html")' + nl) out.write('WriteCommandSummaryToFile(OutputFile="' + output_file_absolute + '.summary.html")' + nl) # TODO smalers 2018-01-28 evaluate whether to support end content # Include the end command file if requested # Message.printStatus ( 2, routine, "Adding commands from end command file \"" + EndCommandFile_full + "\"") # includeCommandFile ( out, EndCommandFile_full, "end" ); out.close() # Add the log file to output self.command_processor.add_output_file(output_file_absolute) except Exception as e: warning_count += 1 message = 'Unexpected error creating regression test command file "' + output_file_absolute + '"' traceback.print_exc(file=sys.stdout) logger.exception(message, e) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "See the log file for details.")) if warning_count > 0: message = "There were " + str(warning_count) + " warnings processing the command." raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self): """ Run the command. Set a GeoLayer property value. Returns: Nothing. Raises: RuntimeError if any exception occurs running the command. """ warning_count = 0 logger = logging.getLogger(__name__) pv_GeoLayerID = self.get_parameter_value("GeoLayerID") pv_PropertyName = self.get_parameter_value('PropertyName') pv_PropertyType = self.get_parameter_value('PropertyType') pv_PropertyValue = self.get_parameter_value('PropertyValue') # Expand the property value string before converting to the requested type pv_PropertyValue_expanded = self.command_processor.expand_parameter_value( pv_PropertyValue) try: # Convert the property value string to the requested type pv_PropertyValue2 = None if pv_PropertyType == 'bool': pv_PropertyValue2 = bool(pv_PropertyValue_expanded) elif pv_PropertyType == 'float': pv_PropertyValue2 = float(pv_PropertyValue_expanded) elif pv_PropertyType == 'int': pv_PropertyValue2 = int(pv_PropertyValue_expanded) elif pv_PropertyType == 'str': pv_PropertyValue2 = str(pv_PropertyValue_expanded) # Now set the object as a property, will be the requested type if pv_PropertyValue2 is not None: self.command_processor.set_property(pv_PropertyName, pv_PropertyValue2) # Get the GeoLayer object geolayer = self.command_processor.get_geolayer(pv_GeoLayerID) if geolayer is None: message = 'Unable to find GeoLayer for GeoLayerID="' + pv_GeoLayerID + '"' warning_count += 1 logger.error(message) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "Check the log file for details.")) else: geolayer.set_property(pv_PropertyName, pv_PropertyValue2) except Exception as e: warning_count += 1 message = 'Unexpected error setting GeoLayer property "' + pv_PropertyName + '"' traceback.print_exc(file=sys.stdout) logger.exception(message, e) self.command_status.add_to_log( CommandPhaseType.RUN, CommandLogRecord(CommandStatusType.FAILURE, message, "Check the log file for details.")) if warning_count > 0: message = "There were " + str( warning_count) + " warnings processing the command." raise RuntimeError(message) self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters): """ Check the command parameters for validity. Args: command_parameters: the dictionary of command parameters to check (key:string_value) Returns: Nothing. Raises: ValueError if any parameters are invalid or do not have a valid value. The command status messages for initialization are populated with validation messages. """ warning = "" logger = logging.getLogger(__name__) # GeoLayerID is required # - non-empty, non-None string. # - existence of the GeoLayer will also be checked in run_command(). pv_GeoLayerID = self.get_parameter_value( parameter_name='GeoLayerID', command_parameters=command_parameters) if not validators.validate_string(pv_GeoLayerID, False, False): message = "GeoLayerID parameter has no value." recommendation = "Specify the GeoLayerID parameter to indicate the GeoLayer to process." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # PropertyName is required pv_PropertyName = self.get_parameter_value( parameter_name='PropertyName', command_parameters=command_parameters) if not validators.validate_string(pv_PropertyName, False, False): message = "PropertyName parameter has no value." recommendation = "Specify a property name." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # PropertyType is required pv_PropertyType = self.get_parameter_value( parameter_name='PropertyType', command_parameters=command_parameters) property_types = ["bool", "float", "int", "long", "str"] if not validators.validate_string_in_list( pv_PropertyType, property_types, False, False): message = 'The requested property type "' + pv_PropertyType + '"" is invalid.' recommendation = "Specify a valid property type: " + str( property_types) warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # PropertyValue is required pv_PropertyValue = self.get_parameter_value( parameter_name='PropertyValue', command_parameters=command_parameters) if not validators.validate_string(pv_PropertyValue, False, False): # TODO smalers 2017-12-28 add other parameters similar to TSTool to set special values message = "PropertyValue parameter is not specified." recommendation = "Specify a property value." warning += "\n" + message self.command_status.add_to_log( CommandPhaseType.INITIALIZATION, CommandLogRecord(CommandStatusType.FAILURE, message, recommendation)) # Check for unrecognized parameters. # This returns a message that can be appended to the warning, which if non-empty # triggers an exception below. warning = command_util.validate_command_parameter_names(self, warning) # If any warnings were generated, throw an exception if len(warning) > 0: logger.warning(warning) raise ValueError(warning) # Refresh the phase severity self.command_status.refresh_phase_severity( CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)