def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns: None.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning = ""
# Check that parameter GeoLayerID is a non-empty, non-None string.
pv_GeoLayerID = self.get_parameter_value(
parameter_name='GeoLayerID', command_parameters=command_parameters)
if not validators.validate_string(pv_GeoLayerID, False, False):
message = "GeoLayerID parameter has no value."
recommendation = "Specify the GeoLayerID parameter to indicate the GeoLayer to copy."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check that optional parameter IfGeoLayerIDExists is either `Replace`, `ReplaceAndWarn`, `Warn`, `Fail`, None.
pv_IfGeoLayerIDExists = self.get_parameter_value(
parameter_name="IfGeoLayerIDExists",
command_parameters=command_parameters)
acceptable_values = ["Replace", "ReplaceAndWarn", "Warn", "Fail"]
if not validators.validate_string_in_list(pv_IfGeoLayerIDExists,
acceptable_values,
none_allowed=True,
empty_string_allowed=True,
ignore_case=True):
message = "IfGeoLayerIDExists parameter value ({}) is not recognized.".format(
pv_IfGeoLayerIDExists)
recommendation = "Specify one of the acceptable values ({}) for the IfGeoLayerIDExists parameter.".format(
acceptable_values)
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty triggers an exception below.
warning = command_util.validate_command_parameter_names(self, warning)
# If any warnings were generated, throw an exception.
if len(warning) > 0:
self.logger.warning(warning)
raise ValueError(warning)
else:
# Refresh the phase severity
self.command_status.refresh_phase_severity(
CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def __should_read_geolayer(self, geolayer_id):
"""
Checks the following:
* the ID of the output GeoLayer is unique (not an existing GeoLayer ID)
Args:
geolayer_id: the ID of the output GeoLayer
Returns:
run_read: Boolean. If TRUE, the GeoLayer read process should be run. If FALSE, the read process should
not be run.
"""
# Boolean to determine if the read process should be run. Set to true until an error occurs.
run_read = True
# If the GeoLayerID is the same as an already-registered GeoLayerID, react according to the
# pv_IfGeoLayerIDExists value.
if self.command_processor.get_geolayer(geolayer_id):
# Get the IfGeoLayerIDExists parameter value.
pv_IfGeoLayerIDExists = self.get_parameter_value("IfGeoLayerIDExists", default_value="Replace")
# Warnings/recommendations if the GeolayerID is the same as a registered GeoLayerID.
message = 'The GeoLayerID ({}) value is already in use as a GeoLayer ID.'.format(geolayer_id)
recommendation = 'Specify a new GeoLayerID.'
# The registered GeoLayer should be replaced with the new GeoLayer (with warnings).
if pv_IfGeoLayerIDExists.upper() == "REPLACEANDWARN":
self.warning_count += 1
self.logger.warning(message)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.WARNING,
message, recommendation))
# The registered GeoLayer should not be replaced. A warning should be logged.
if pv_IfGeoLayerIDExists.upper() == "WARN":
run_read = False
self.warning_count += 1
self.logger.warning(message)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.WARNING,
message, recommendation))
# The matching IDs should cause a FAILURE.
elif pv_IfGeoLayerIDExists.upper() == "FAIL":
run_read = False
self.warning_count += 1
self.logger.error(message)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE,
message, recommendation))
# Return the Boolean to determine if the read process should be run. If TRUE, all checks passed. If FALSE,
# one or many checks failed.
return run_read
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns:
None.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning = ""
logger = logging.getLogger(__name__)
# Unlike most commands, set internal data here because it is needed by initial call to next()
# before calls to run_command
# Name is required
pv_Name = self.get_parameter_value(
parameter_name='Name', command_parameters=command_parameters)
if not validators.validate_string(pv_Name, False, False):
message = "A name for the If block must be specified"
recommendation = "Specify the Name."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Condition is required
pv_Condition = self.get_parameter_value(
parameter_name='Condition', command_parameters=command_parameters)
if not validators.validate_string(pv_Condition, False, False):
message = "A condition for the If command must be specified"
recommendation = "Specify the condition."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty
# triggers an exception below.
warning = command_util.validate_command_parameter_names(self, warning)
# If any warnings were generated, throw an exception
if len(warning) > 0:
logger.warn(warning)
raise ValueError(warning)
# Refresh the phase severity
self.command_status.refresh_phase_severity(
CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns: None.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning = ""
# Check that parameter URL is a non-empty, non-None string.
# - existence of the url will also be checked in run_command().
pv_URL = self.get_parameter_value(
parameter_name='URL', command_parameters=command_parameters)
if not validators.validate_string(pv_URL, False, False):
message = "URL parameter has no value."
recommendation = "Specify the URL parameter to indicate the URL of the file to download."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check that parameter OutputFile is a non-empty string (can be None).
# - existence of the folder will also be checked in run_command().
pv_OutputFile = self.get_parameter_value(
parameter_name='OutputFile', command_parameters=command_parameters)
if not validators.validate_string(pv_OutputFile, True, False):
message = "OutputFile parameter has no value."
recommendation = "Specify the OutputFile parameter to indicate the output file."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty triggers an exception below.
warning = command_util.validate_command_parameter_names(self, warning)
# If any warnings were generated, throw an exception.
if len(warning) > 0:
self.logger.warning(warning)
raise ValueError(warning)
else:
# Refresh the phase severity
self.command_status.refresh_phase_severity(
CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns: None.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning = ""
# Check that parameter GeoLayerIDs is a non-empty, non-None string.
pv_GeoLayerIDs = self.get_parameter_value(
parameter_name='GeoLayerIDs',
command_parameters=command_parameters)
if not validators.validate_string(pv_GeoLayerIDs, False, False):
message = "GeoLayerIDs parameter has no value."
recommendation = "Specify the GeoLayerIDs parameter to indicate the GeoLayers to merge."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check that parameter OutputGeoLayerID is a non-empty, non-None string.
pv_OutputGeoLayerID = self.get_parameter_value(
parameter_name='OutputGeoLayerID',
command_parameters=command_parameters)
if not validators.validate_string(pv_OutputGeoLayerID, False, False):
message = "OutputGeoLayerID parameter has no value."
recommendation = "Specify the OutputGeoLayerID parameter to indicate the ID of the merged GeoLayer."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty triggers an exception below.
warning = command_util.validate_command_parameter_names(self, warning)
# If any warnings were generated, throw an exception.
if len(warning) > 0:
self.logger.warning(warning)
raise ValueError(warning)
# Refresh the phase severity
self.command_status.refresh_phase_severity(
CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns:
Nothing.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning = ""
logger = logging.getLogger("gp")
# Message is required
pv_Message = self.get_parameter_value(
parameter_name='Message', command_parameters=command_parameters)
if not validators.validate_string(pv_Message, False, False):
message = "Message parameter has no value."
recommendation = "Specify text for the Message parameter."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
pv_CommandStatus = self.get_parameter_value(
parameter_name='CommandStatus',
command_parameters=command_parameters)
if not validators.validate_string_in_list(
pv_CommandStatus,
CommandStatusType.get_command_status_types_as_str(), True,
True):
message = 'The requested command status "' + pv_CommandStatus + '"" is invalid.'
recommendation = "Specify a valid command status."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty
# triggers an exception below.
warning = command_util.validate_command_parameter_names(self, warning)
# If any warnings were generated, throw an exception
if len(warning) > 0:
logger.warning(warning)
raise ValueError(warning)
# Refresh the phase severity
self.command_status.refresh_phase_severity(
CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns: None.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning = ""
# Check that parameter SpatialDataFile is a non-empty, non-None string.
# - existence of the file will also be checked in run_command().
pv_SpatialDataFile = self.get_parameter_value(parameter_name='SpatialDataFile',
command_parameters=command_parameters)
if not validators.validate_string(pv_SpatialDataFile, False, False):
message = "SpatialDataFile parameter has no value."
recommendation = "Specify the SpatialDataFile parameter to indicate the spatial data layer file."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
# Check that optional parameter IfGeoLayerIDExists is one of the acceptable values or is None.
pv_IfGeoLayerIDExists = self.get_parameter_value(parameter_name="IfGeoLayerIDExists",
command_parameters=command_parameters)
if not validators.validate_string_in_list(pv_IfGeoLayerIDExists, self.__choices_IfGeoLayerIDExists,
none_allowed=True, empty_string_allowed=True, ignore_case=True):
message = "IfGeoLayerIDExists parameter value ({}) is not recognized.".format(pv_IfGeoLayerIDExists)
recommendation = "Specify one of the acceptable values ({}) for the IfGeoLayerIDExists parameter.".format(
self.__choices_IfGeoLayerIDExists)
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty triggers an exception below.
warning = command_util.validate_command_parameter_names(self, warning)
# If any warnings were generated, throw an exception.
if len(warning) > 0:
self.logger.warning(warning)
raise ValueError(warning)
else:
# Refresh the phase severity
self.command_status.refresh_phase_severity(CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns:
Nothing.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning_message = ""
logger = logging.getLogger(__name__)
# CommandFile is required
pv_CommandFile = self.get_parameter_value(parameter_name='CommandFile', command_parameters=command_parameters)
if not validators.validate_string(pv_CommandFile, False, False):
message = "The CommandFile must be specified."
recommendation = "Specify the command file."
warning_message += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
# ExpectedStatus is optional, will default to Success at runtime
pv_ExpectedStatus = self.get_parameter_value(parameter_name='ExpectedStatus',
command_parameters=command_parameters)
if not validators.validate_string_in_list(pv_ExpectedStatus,
self.__choices_ExpectedStatus, True, True):
message = "ExpectedStatus parameter is invalid."
recommendation = "Specify the ExpectedStatus parameter as blank or one of " + \
str(self.__choices_ExpectedStatus)
warning_message += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty
# triggers an exception below.
warning_message = command_util.validate_command_parameter_names(self, warning_message)
# If any warnings were generated, throw an exception
if len(warning_message) > 0:
logger.warning(warning_message)
raise ValueError(warning_message)
# Refresh the phase severity
self.command_status.refresh_phase_severity(CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns:
Nothing.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning_message = ""
logger = logging.getLogger(__name__)
# SearchFolder is required
pv_SearchFolder = self.get_parameter_value(parameter_name='SearchFolder', command_parameters=command_parameters)
if not validators.validate_string(pv_SearchFolder, False, False):
message = "SearchFolder parameter has no value."
warning_message += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message, "Specify the search folder."))
print(message)
# FilenamePattern is optional, will default to "test-*" at runtime
# OutputFile is required
pv_OutputFile = self.get_parameter_value(parameter_name='OutputFile', command_parameters=command_parameters)
if not validators.validate_string(pv_OutputFile, False, False):
message = "OutputFile parameter has no value."
warning_message += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message, "Specify the output file."))
print(message)
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty
# triggers an exception below.
warning_message = command_util.validate_command_parameter_names(self, warning_message)
# If any warnings were generated, throw an exception
if len(warning_message) > 0:
logger.warning(warning_message)
raise ValueError(warning_message)
# Refresh the phase severity
self.command_status.refresh_phase_severity(CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def __should_write_geolayer(self, geolayer_id, output_file_abs):
"""
Checks the following:
* the ID of the GeoLayer is an existing GeoLayer ID
* the output folder is a valid folder
Args:
geolayer_id: the ID of the GeoLayer to be written
output_file_abs: the full pathname to the output file
Returns:
run_write: Boolean. If TRUE, the writing process should be run. If FALSE, it should not be run.
"""
# Boolean to determine if the writing process should be run. Set to true until an error occurs.
run_write = True
# Boolean to determine if the output format parameters have valid bool values. Set to true until proven false.
valid_output_bool = True
# If the GeoLayer ID is not an existing GeoLayer ID, raise a FAILURE.
if not self.command_processor.get_geolayer(geolayer_id):
run_write = False
self.warning_count += 1
message = 'The GeoLayerID ({}) is not a valid GeoLayer ID.'.format(
geolayer_id)
recommendation = 'Specify a valid GeoLayerID.'
self.logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# If the OutputFolder is not a valid folder, raise a FAILURE.
output_folder = os.path.dirname(output_file_abs)
if not os.path.isdir(output_folder):
run_write = False
self.warning_count += 1
message = 'The output folder ({}) of the OutputFile is not a valid folder.'.format(
output_folder)
recommendation = 'Specify a valid relative pathname for the output file.'
self.logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Return the Boolean to determine if the write process should be run. If TRUE, all checks passed. If FALSE,
# one or many checks failed.
return run_write
def __should_run_webget(self, output_file_abs):
"""
Checks the following:
* the output folder is a valid folder
Args:
output_file_abs: the full pathname to the output file
Returns:
run_webget: Boolean. If TRUE, the webget process should be run. If FALSE, it should not be run.
"""
# Boolean to determine if the webget process should be run. Set to true until an error occurs.
run_webget = True
# If the OutputFolder is not a valid folder, raise a FAILURE.
output_folder = os.path.dirname(output_file_abs)
if not os.path.isdir(output_folder):
run_webget = False
self.warning_count += 1
message = 'The output folder ({}) of the OutputFile is not a valid folder.'.format(
output_folder)
recommendation = 'Specify a valid relative pathname for the output file.'
self.logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Return the Boolean to determine if the webget process should be run. If TRUE, all checks passed. If FALSE,
# one or many checks failed.
return run_webget
def __should_read_folder(self, spatial_data_folder_abs):
"""
Checks the following:
* the SpatialDataFolder (absolute) is a valid folder
Args:
spatial_data_folder_abs: the full pathname to the input spatial data folder
Returns:
run_read: Boolean. If TRUE, the folder read process should be run. If FALSE, it should not be run.
"""
# Boolean to determine if the read process should be run. Set to true until an error occurs.
run_read = True
# If the input spatial data folder is not a valid file path, raise a FAILURE.
if not os.path.isdir(spatial_data_folder_abs):
run_read = False
self.warning_count += 1
message = "The SpatialDataFolder ({}) is not a valid folder.".format(spatial_data_folder_abs)
recommendation = "Specify a valid folder."
self.logger.error(message)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
# Return the Boolean to determine if the read process should be run. If TRUE, all checks passed. If FALSE,
# one or many checks failed.
return run_read
def __should_extract_file(self, file_abs, output_folder_abs, file_type):
"""
Checks the following:
* the File is a valid file
* the OutputFolder is a valid folder
* the FileType correctly identifies the File's type
Args:
file_abs (str): the full path to the input compressed File
output_folder_abs(str): the full path to the OutputFolder
file_type(str): the FileType value depicting the file type of the input File
Returns:
Boolean. If TRUE, the file should be extracted. If FALSE, at least one check failed and the file should
not be extracted.
"""
# List of Boolean values. The Boolean values correspond to the results of the following tests. If TRUE, the
# test confirms that the command should be run.
should_run_command = []
# If the File parameter value is not a valid file, raise a FAILURE.
should_run_command.append(
validators.run_check(self, "IsFilePathValid", "File", file_abs,
"FAIL"))
# If the OutputFolder parameter value is not a valid folder, raise a FAILURE.
should_run_command.append(
validators.run_check(self, "IsFolderPathValid", "OutputFolder",
output_folder_abs, "FAIL"))
# If the File Type is not recognized, raise a FAILURE.
if file_type is None:
message = "A valid FileType cannot be determined from the file ({}).".format(
file_abs)
recommendation = "Use the FileType parameter to assign the appropriate file type."
should_run_command.append(False)
self.logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# If the File Type is not actually recognized by the input File, raise a FAILURE.
if file_type.upper() == "ZIP":
should_run_command.append(
validators.run_check(self, "IsZipFile", "File", file_abs,
"FAIL"))
elif file_type.upper() == "TAR":
should_run_command.append(
validators.run_check(self, "IsTarFile", "File", file_abs,
"FAIL"))
# Return the Boolean to determine if the process should be run.
if False in should_run_command:
return False
else:
return True
def run_command(self):
"""
Run the command. Remove the attribute from the GeoLayer.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_AttributeNames = self.get_parameter_value("AttributeNames")
# Convert the AttributeNames parameter from a string to a list.
attribute_names_list = string_util.delimited_string_to_list(
pv_AttributeNames)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_attribute_be_removed(pv_GeoLayerID,
attribute_names_list):
# Run the process.
try:
# Get the input GeoLayer.
input_geolayer = self.command_processor.get_geolayer(
pv_GeoLayerID)
# Iterate over the input attributes to remove.
for attribute_name in attribute_names_list:
# Remove the attribute from the GeoLayer.
input_geolayer.remove_attribute(attribute_name)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error removing attribute(s) ({}) from GeoLayer {}.".format(
pv_AttributeNames, pv_GeoLayerID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def __should_write_table2(self, datastore, datastore_table_name,
datastore_table_cols_to_receive, writemode):
"""
Checks the following:
* the datastore columns configured to receive data are existing columns within the DataStore table
Args:
datastore (obj): the DataStore that is receiving the data
datastore_table_name (str): the name of the DataStore table that is receiving the data
datastore_table_cols_to_receive (list of strings): A list of DataBase table columns that are expected
to receive data.
Returns:
Boolean. If TRUE, the process should be run. If FALSE, it should not be run.
"""
# List of Boolean values. The Boolean values correspond to the results of the following tests. If TRUE, the
# test confirms that the command should be run.
should_run_command = []
# A list of Table columns that do not map to the DataStore Table columns.
invalid_columns = []
# Do not run this check if the WriteMode is NewTableInsert or ExistingTableOverwrite
exception_modes = ["NEWTABLEINSERT", "EXISTINGTABLEOVERWRITE"]
if writemode.upper() not in exception_modes:
# Get the DataStore Table columns.
datastore_table_cols = datastore.return_col_names(
datastore_table_name)
# Iterate over the DataStore columns that are configured to read data.
for datastore_table_col_to_receive in datastore_table_cols_to_receive:
# If the configured DataStore column does not exist in the DataStore table, add the configured
# DataStore column to the list of invalid columns.
if datastore_table_col_to_receive not in datastore_table_cols:
invalid_columns.append(datastore_table_col_to_receive)
# If there are any invalid configured DataStore columns, raise a FAILURE.
if invalid_columns:
message = "One or more of the DataStore columns configured to be edited do(es) not exist in the DataStore" \
" table ({}). The invalid columns are: \n({}).".format(datastore_table_name, invalid_columns)
recommendation = "Specify valid DataStore columns to edit."
self.logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
should_run_command.append(False)
# Return the Boolean to determine if the process should be run.
if False in should_run_command:
self.warning_count += 1
return False
else:
return True
def run_command(self):
"""
Run the command. Set the GeoProcessor property to that of the specified GeoLayer property.
Returns:
Nothing.
Raises:
RuntimeError if an exception occurs, for example if the property name is not found.
"""
warning_count = 0
logger = logging.getLogger(__name__)
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_GeoLayerPropertyName = self.get_parameter_value('GeoLayerPropertyName')
pv_PropertyName = self.get_parameter_value('PropertyName')
try:
# Get the GeoLayer object
geolayer = self.command_processor.get_geolayer(pv_GeoLayerID)
if geolayer is None:
message = 'Unable to find GeoLayer for GeoLayerID="' + pv_GeoLayerID + '"'
warning_count += 1
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message, "Check the log file for details."))
else:
# First get the property from the GeoLayer
property_value = geolayer.get_property(pv_GeoLayerPropertyName)
if property_value is not None:
self.command_processor.set_property(pv_PropertyName, property_value)
except Exception as e:
warning_count += 1
message = 'Unexpected error setting property "' + pv_PropertyName + '"'
logger.exception(message, e)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"Check the log file for details."))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Close an existing DataStore and remove it from the GeoProcessor.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the DataStoreID parameter value and expand for ${Property} syntax.
pv_DataStoreID = self.get_parameter_value("DataStoreID")
pv_DataStoreID = self.command_processor.expand_parameter_value(
pv_DataStoreID, self)
# Obtain the StatusMessage parameter value and expand for ${Property} syntax.
pv_StatusMessage = self.get_parameter_value("StatusMessage")
pv_StatusMessage = self.command_processor.expand_parameter_value(
pv_StatusMessage, self)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_close_datastore(pv_DataStoreID):
try:
# Get the DataStore object
datastore_obj = self.command_processor.get_datastore(
pv_DataStoreID)
# Close the database connection.
datastore_obj.close_db_connection()
# Update the DataStore's status message.
datastore_obj.update_status_message(pv_StatusMessage)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error closing DataStore {}.".format(
pv_DataStoreID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Write the GeoLayer to a spatial data file in KML format.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_OutputFile = self.get_parameter_value("OutputFile")
pv_PlacemarkNameAttribute = self.get_parameter_value("PlacemarkNameAttribute")
pv_PlacemarkDescriptionAttribute = self.get_parameter_value("PlacemarkDescriptionAttribute")
# Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_OutputFile, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_write_geolayer(pv_GeoLayerID, output_file_absolute):
try:
# Get the GeoLayer
geolayer = self.command_processor.get_geolayer(pv_GeoLayerID)
# Write the GeoLayer to a spatial data file in KML format
# "Note that KML by specification uses only a single projection, EPSG:4326. All OGR KML output will be
# presented in EPSG:4326. As such OGR will create layers in the correct coordinate system and transform
# any geometries." - www.gdal.org/drv_kml.html
qgis_util.write_qgsvectorlayer_to_kml(geolayer.qgs_vector_layer,
output_file_absolute,
"EPSG:4326",
pv_PlacemarkNameAttribute,
pv_PlacemarkDescriptionAttribute,
"clampToGround")
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error writing GeoLayer {} to GeoJSON format.".format(pv_GeoLayerID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def __should_write_table(self, table_id, output_file_abs):
"""
Checks the following:
* the ID of the Table is an existing Table ID
* the output folder is a valid folder
Args:
table_id: the ID of the Table to be written
output_file_abs: the full pathname to the output file
Returns:
run_write: Boolean. If TRUE, the writing process should be run. If FALSE, it should not be run.
"""
# List of Boolean values. The Boolean values correspond to the results of the following tests. If TRUE, the
# test confirms that the command should be run.
should_run_command = []
# If the Table ID is not an existing Table ID, raise a FAILURE.
should_run_command.append(
validators.run_check(self, "IsTableIdExisting", "TableID",
table_id, "FAIL"))
# Get the full path to the output folder
output_folder_abs = io_util.get_path(output_file_abs)
# If the output folder is not an existing folder, raise a FAILURE.
should_run_command.append(
validators.run_check(self, "IsFolderPathValid", "OutputFile",
output_folder_abs, "FAIL"))
# Continue if the output file is an existing file.
if os.path.exists(output_folder_abs):
if io_util.get_extension(output_file_abs).upper() == ".XLS":
message = 'At the current time, a Table object cannot be appended to or overwrite an existing Excel ' \
'file in XLS format.'
recommendation = "Update the XLS file ({}) to an XLSX file or write the table " \
"to a new XLS file.".format(output_file_abs)
self.warning_count += 1
self.logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
should_run_command.append(False)
# Return the Boolean to determine if the process should be run.
if False in should_run_command:
return False
else:
return True
def run_command(self):
"""
Run the command. Open a file to receive regression test results.
Returns:
None.
Raises:
RuntimeError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_OutputFile = self.get_parameter_value('OutputFile')
# Runtime checks on input
pv_OutputFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_OutputFile, self)))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Open the regression test results file
try:
pv_OutputFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_OutputFile, self)))
self.__open_new_regression_test_report_file(pv_OutputFile_absolute, False) # Do not append
logger.info('Opened regression test results report file "' + pv_OutputFile_absolute + '"')
except Exception as e:
warning_count += 1
message = 'Unexpected error opening file "' + pv_OutputFile_absolute + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings processing the command."
logger.warning(message)
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns:
Nothing.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning = ""
logger = logging.getLogger(__name__)
# LogFile is required
pv_LogFile = self.get_parameter_value(parameter_name='LogFile', command_parameters=command_parameters)
if not validators.validate_string(pv_LogFile, False, False):
message = "The log file must be specified."
recommendation = "Specify the log file."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
print(message)
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty
# triggers an exception below.
warning = command_util.validate_command_parameter_names(self, warning)
# If any warnings were generated, throw an exception
if len(warning) > 0:
logger.warning(warning)
raise ValueError(warning)
# Refresh the phase severity
self.command_status.refresh_phase_severity(CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Restart the lof file with the given name.
Subsequent logging messages will go to the file until another StartLog() command is run.
Returns:
Nothing.
Raises:
RuntimeError if any exception occurs running the command.
"""
logger = logging.getLogger(__name__)
warning_count = 0
pv_LogFile = self.get_parameter_value('LogFile')
log_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_LogFile)))
try:
# Change the GeoProcessor logger to use the specified file
# - The initial application log file will be closed.
log_util.reset_log_file_handler(log_file_absolute)
# Add the log file to output
self.command_processor.add_output_file(log_file_absolute)
except Exception as e:
warning_count += 1
message = 'Unexpected error (re)starting log file "' + log_file_absolute + '"'
logger.exception(message, e)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"Check the old log file or command window for details."))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Print the message to the log file.
Returns:
Nothing.
Raises:
RuntimeError if any exception occurs.
"""
logger = logging.getLogger(__name__)
warning_count = 0
# Message parameter won't be null.
pv_Message = self.get_parameter_value('Message')
pv_CommandStatus = self.get_parameter_value('CommandStatus')
if pv_CommandStatus is None or pv_CommandStatus == "":
# Default status as a string
pv_commandStatus = str(CommandStatusType.SUCCESS)
# Convert the string to the enum
command_status_type = CommandStatusType.value_of(pv_CommandStatus,
ignore_case=True)
message_expanded = self.command_processor.expand_parameter_value(
pv_Message)
logger.info(message_expanded)
# Add a log message for the requested status type
# - don't add to the warning count
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(command_status_type, message_expanded, ""))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Read the feature classes within a file geodatabase. For each desired feature class (can be
specified by the Subset_Pattern parameter), create a GeoLayer object, and add to the GeoProcessor's geolayer
list.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the required and optional parameter values
pv_SpatialDataFolder = self.get_parameter_value("SpatialDataFolder")
pv_ReadOnlyOneFeatureClass = self.get_parameter_value(
"ReadOnlyOneFeatureClass")
pv_Subset_Pattern = self.get_parameter_value("Subset_Pattern")
pv_GeoLayerID_prefix = self.get_parameter_value("GeoLayerID_prefix")
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_FeatureClass = self.get_parameter_value("FeatureClass")
# Convert the ReadOnlyOneFeatureClass from a string value to a Boolean value.
pv_ReadOnlyOneFeatureClass = string_util.str_to_bool(
pv_ReadOnlyOneFeatureClass)
# Convert the SpatialDataFolder parameter value relative path to an absolute path
sd_folder_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_SpatialDataFolder, self)))
# Run the initial checks on the parameter values. Only continue if the checks passed.
if self.__should_read_gdb(sd_folder_abs):
# If configured to only read one Feature Class into one GeoLayer.
if pv_ReadOnlyOneFeatureClass:
# Run the check to see if the GeoLayer should be read.
if self.__should_read_geolayer(pv_GeoLayerID, True,
pv_FeatureClass, sd_folder_abs):
try:
# Get the full pathname to the feature class
# TODO egiles 2018-01-04 Need to research how to properly document feature class source path
spatial_data_file_absolute = os.path.join(
sd_folder_abs, str(pv_FeatureClass))
# Create a QgsVectorLayer object from the feature class.
qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_feature_class(
sd_folder_abs, pv_FeatureClass)
# Create a GeoLayer and add it to the geoprocessor's GeoLayers list
geolayer_obj = GeoLayer(pv_GeoLayerID,
qgs_vector_layer,
spatial_data_file_absolute)
self.command_processor.add_geolayer(geolayer_obj)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading feature class ({}) from file geodatabase ({}).".format(
pv_FeatureClass, sd_folder_abs)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE,
message, recommendation))
# If configured to read multiple Feature Classes into multiple GeoLayers.
else:
# Get a list of all of the feature classes in the file geodatabase.
fc_list = self.__return_a_list_of_fc(sd_folder_abs)
# Filter the fc_list to only include feature classes that meet the Subset Pattern configuration. If
# the Subset Pattern configuration is None, all feature classes will remain in the fc_list.
fc_list = string_util.filter_list_of_strings(
fc_list, [pv_Subset_Pattern])
# Iterate over the feature classes in the fc_list.
for feature_class in fc_list:
# Determine the GeoLayerID.
if pv_GeoLayerID_prefix:
geolayer_id = "{}_{}".format(pv_GeoLayerID_prefix,
feature_class)
else:
geolayer_id = feature_class
# Run the secondary checks on the parameter values. Only continue if the checks passed.
if self.__should_read_geolayer(geolayer_id, False):
try:
# Get the full pathname to the feature class
# TODO egiles 2018-01-04 Need to research how to properly document feature class source path
spatial_data_file_absolute = os.path.join(
sd_folder_abs, str(feature_class))
# Create a QgsVectorLayer object from the feature class.
qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_feature_class(
sd_folder_abs, feature_class)
# Create a GeoLayer and add it to the geoprocessor's GeoLayers list
geolayer_obj = GeoLayer(
geolayer_id, qgs_vector_layer,
spatial_data_file_absolute)
self.command_processor.add_geolayer(geolayer_obj)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading feature class ({}) from file geodatabase ({}).".format(
feature_class, sd_folder_abs)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE,
message, recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def __should_attribute_be_removed(self, geolayer_id, attribute_names):
"""
Checks the following:
* The ID of the input GeoLayer is an actual GeoLayer (if not, log an error message and do not continue.)
* The attribute names are valid names for the GeoLayer (if not, log an error message and do not continue.)
Args:
geolayer_id: the ID of the GeoLayer with the attribute to remove
attribute_names (list of strings): the names of the attributes to remove from the GeoLayer
Returns:
remove_attribute: Boolean. If TRUE, the attribute should be removed from the GeoLayer. If FALSE, a check
has failed and the attribute should not be removed.
"""
# Boolean to determine if the attribute should be removed. Set to TRUE until one or many checks fail.
remove_attribute = True
# If the input GeoLayer does not exist, raise a FAILURE.
if not self.command_processor.get_geolayer(geolayer_id):
# Boolean to determine if the attribute should be removed.
remove_attribute = False
self.warning_count += 1
message = 'The input GeoLayer ID ({}) does not exist.'.format(
geolayer_id)
recommendation = 'Specify a valid GeoLayerID.'
self.logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# If the input GeoLayer does exist, continue with the checks.
else:
# Get the input GeoLayer object.
input_geolayer = self.command_processor.get_geolayer(geolayer_id)
# Get the existing attribute names of the input GeoLayer.
list_of_existing_attributes = input_geolayer.get_attribute_field_names(
)
# Create a list of invalid input attribute names. An invalid attribute name is an input attribute name
# that is not matching any of the existing attribute names of the GeoLayer.
invalid_attrs = (attr_name for attr_name in attribute_names
if attr_name not in list_of_existing_attributes)
# Iterate over the invalid input attribute names and raise a FAILURE for each.
for invalid_attr in invalid_attrs:
remove_attribute = False
self.warning_count += 1
message = 'The attribute name ({}) is not valid.'.format(
invalid_attr)
recommendation = 'Specify a valid attribute name. Valid attributes for this layer are' \
' as follows: {}'.format(list_of_existing_attributes)
self.logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Return the Boolean to determine if the attribute should be removed. If TRUE, all checks passed. If FALSE,
# one or many checks failed.
return remove_attribute
def run_command(self):
"""
Run the command. Extract the compressed file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the File and the DeleteFile parameter values.
pv_File = self.get_parameter_value("File")
pv_DeleteFile = self.get_parameter_value("DeleteFile",
default_value="False")
# Convert the File parameter value relative path to an absolute path. Expand for ${Property} syntax.
file_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_File, self)))
# Get the FileType parameter value.
default_file_ext = self.__get_default_file_type(file_abs)
pv_FileType = self.get_parameter_value("FileType",
default_value=default_file_ext)
# Get the OutputFolder parameter value.
parent_folder = io_util.get_path(file_abs)
pv_OutputFolder = self.get_parameter_value("OutputFolder",
default_value=parent_folder)
# Convert the OutputFolder parameter value relative path to an absolute path. Expand for ${Property} syntax.
output_folder_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFolder, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_extract_file(file_abs, output_folder_abs,
pv_FileType):
try:
# If the file is a .zip file, extract the zip file.
if pv_FileType.upper() == "ZIP":
zip_util.unzip_all_files(file_abs, output_folder_abs)
# If the file is a .tar file, extract the tar file.
elif pv_FileType.upper() == "TAR":
zip_util.untar_all_files(file_abs, output_folder_abs)
# If configured, remove the input compressed file.
if string_util.str_to_bool(pv_DeleteFile):
os.remove(file_abs)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error extracting the {} file ({}).".format(
pv_FileType, pv_File)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns: None.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning = ""
# Check that either the parameter File is a non-empty, non-None string.
pv_File = self.get_parameter_value(
parameter_name='File', command_parameters=command_parameters)
if not validators.validate_string(pv_File, False, False):
message = "File parameter has no value."
recommendation = "Specify the File parameter to indicate the compressed file to extract."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check that optional parameter FileType is an acceptable value or is None.
pv_FileType = self.get_parameter_value(
parameter_name="FileType", command_parameters=command_parameters)
acceptable_values = ["Zip", "Tar"]
if not validators.validate_string_in_list(pv_FileType,
acceptable_values,
none_allowed=True,
empty_string_allowed=False,
ignore_case=True):
message = "FileType parameter value ({}) is not recognized.".format(
pv_FileType)
recommendation = "Specify one of the acceptable values ({}) for the" \
" FileType parameter.".format(acceptable_values)
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check that optional DeleteFile parameter value is a valid Boolean value or is None.
pv_DeleteFile = self.get_parameter_value(
parameter_name="DeleteFile", command_parameters=command_parameters)
if not validators.validate_bool(
pv_DeleteFile, none_allowed=True, empty_string_allowed=False):
message = "DeleteFile parameter value ({}) is not a recognized boolean value.".format(
pv_DeleteFile)
recommendation = "Specify either 'True' or 'False for the DeleteFile parameter."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty triggers an exception below.
warning = command_util.validate_command_parameter_names(self, warning)
# If any warnings were generated, throw an exception.
if len(warning) > 0:
self.logger.warning(warning)
raise ValueError(warning)
else:
# Refresh the phase severity
self.command_status.refresh_phase_severity(
CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command.
Returns:
Nothing.
Raises:
ValueError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_SearchFolder = self.get_parameter_value('SearchFolder')
pv_FilenamePattern = self.get_parameter_value('FilenamePattern')
if pv_FilenamePattern is None or pv_FilenamePattern == "":
# The pattern that is desired is test_*.gp, using globbing syntax
pv_FilenamePattern = "[Tt][Ee][Ss][Tt]-*.gp"
pv_OutputFile = self.get_parameter_value('OutputFile')
# Runtime checks on input
working_dir = self.command_processor.get_property('WorkingDir')
logger.info('working_dir: "' + working_dir + '"')
search_folder_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(working_dir, self.command_processor.expand_parameter_value(pv_SearchFolder, self)))
search_folder_absolute_internal = io_util.verify_path_for_os(search_folder_absolute,
always_use_forward_slashes=True)
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(working_dir, self.command_processor.expand_parameter_value(pv_OutputFile, self)))
output_file_absolute_internal = io_util.verify_path_for_os(output_file_absolute,
always_use_forward_slashes=True)
logger.info('search_folder_absolute: "' + search_folder_absolute + '"')
logger.info('search_folder_absolute_internal: "' + search_folder_absolute_internal + '"')
if not os.path.exists(search_folder_absolute_internal):
message = 'The folder to search "' + search_folder_absolute + '" does not exist.'
logger.warning(message)
warning_count += 1
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE,
message, "Verify that the folder exists at the time the command is run."))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Do the processing
try:
# Output folder is used when constructing filenames for command files to run
output_folder_absolute = os.path.dirname(output_file_absolute_internal)
logger.info('output_folder_absolute: "' + output_folder_absolute + '"')
logger.info('output_folder_absolute_internal: "' + output_file_absolute_internal + '"')
files = [] # List of files to match
# Convert globbing-style wildcards to Pythonic regex
logger.info('Filename pattern using globbing = "' + pv_FilenamePattern + '"')
# filename_pattern_regex = pv_FilenamePattern.replace("*", "[^/]*")
# Match any character of any length, making sure that special characters are dealt with
filename_pattern_regex = pv_FilenamePattern.replace(".", "\\.") # So .gp is handled literally
# The following is used to match between "test-" and ".gp"
filename_pattern_regex = filename_pattern_regex.replace("*", ".*")
logger.info('Filename pattern using Python regex = "' + filename_pattern_regex + '"')
filename_pattern_regex_compiled = re.compile(filename_pattern_regex)
CreateRegressionTestCommandFile.__get_matching_filenames_in_tree(files, search_folder_absolute_internal,
filename_pattern_regex_compiled,
pattern_string=filename_pattern_regex)
# Sort the list
files = sorted(files)
# Open the output file...
# TODO smalers 2018-10-20 decide whether to support append mode
out = open(output_file_absolute_internal, "w")
# Write a standard header to the file so that it is clear when the file was created
io_util.print_standard_file_header(out, comment_line_prefix="#")
# Include the setup command file if requested
# logger.info('Adding commands from setup command file "' + setup_command_file_absolute + '"')
# include_setup_file(out, setup_file_absolute, "setup")
# Include the matching test cases
# Python 2...
# nl = os.linesep # newline character for operating system
# Python 3 translates \n into the OS-specific end of line so os.linesep introduces extra end of lines
nl = "\n"
out.write("#" + nl)
out.write("# The following " + str(len(files)) +
" test cases will be run to compare results with expected results." + nl)
out.write("# Individual log files are generally created for each test." + nl)
# TODO smalers 2018-01-20 evaluate how to handle output table
# - Currently the GeoProcessor does not have table commands
table_param = ""
# Use absolute path since each developer will regenerate this file.
out.write('StartRegressionTestResultsReport(OutputFile="' +
output_file_absolute + '.out.txt"' + table_param + ")" + nl)
# Find the list of matching files...
# logger.debug('output_folder_absolute"' + output_folder_absolute + '"')
for a_file in files:
logger.info('Adding command file "' + a_file + '"')
# The command files to run are relative to the commands file being created.
# - use the operating system separator
command_file_to_run = io_util.verify_path_for_os(
io_util.to_relative_path(output_folder_absolute, a_file))
# Determine if the command file has @expectedStatus in it. If so, define an ExpectedStatus
# parameter for the command.
logger.info('Relative path to command file is "' + command_file_to_run + '"')
out.write('RunCommands(CommandFile="' + command_file_to_run + '"' +
CreateRegressionTestCommandFile.__determine_expected_status_parameter(a_file) + ')' + nl)
# TODO smalers 2018-12-30 Maybe the output file is relative to the working folder
# output_file_relative = io_util.to_relative_path(working_dir, output_file_absolute)
# out.write('WriteCommandSummaryToFile(OutputFile="' + output_file_relative + '.summary.html")' + nl)
out.write('WriteCommandSummaryToFile(OutputFile="' + output_file_absolute + '.summary.html")' + nl)
# TODO smalers 2018-01-28 evaluate whether to support end content
# Include the end command file if requested
# Message.printStatus ( 2, routine, "Adding commands from end command file \"" + EndCommandFile_full + "\"")
# includeCommandFile ( out, EndCommandFile_full, "end" );
out.close()
# Add the log file to output
self.command_processor.add_output_file(output_file_absolute)
except Exception as e:
warning_count += 1
message = 'Unexpected error creating regression test command file "' + output_file_absolute + '"'
traceback.print_exc(file=sys.stdout)
logger.exception(message, e)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Set a GeoLayer property value.
Returns:
Nothing.
Raises:
RuntimeError if any exception occurs running the command.
"""
warning_count = 0
logger = logging.getLogger(__name__)
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_PropertyName = self.get_parameter_value('PropertyName')
pv_PropertyType = self.get_parameter_value('PropertyType')
pv_PropertyValue = self.get_parameter_value('PropertyValue')
# Expand the property value string before converting to the requested type
pv_PropertyValue_expanded = self.command_processor.expand_parameter_value(
pv_PropertyValue)
try:
# Convert the property value string to the requested type
pv_PropertyValue2 = None
if pv_PropertyType == 'bool':
pv_PropertyValue2 = bool(pv_PropertyValue_expanded)
elif pv_PropertyType == 'float':
pv_PropertyValue2 = float(pv_PropertyValue_expanded)
elif pv_PropertyType == 'int':
pv_PropertyValue2 = int(pv_PropertyValue_expanded)
elif pv_PropertyType == 'str':
pv_PropertyValue2 = str(pv_PropertyValue_expanded)
# Now set the object as a property, will be the requested type
if pv_PropertyValue2 is not None:
self.command_processor.set_property(pv_PropertyName,
pv_PropertyValue2)
# Get the GeoLayer object
geolayer = self.command_processor.get_geolayer(pv_GeoLayerID)
if geolayer is None:
message = 'Unable to find GeoLayer for GeoLayerID="' + pv_GeoLayerID + '"'
warning_count += 1
logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"Check the log file for details."))
else:
geolayer.set_property(pv_PropertyName, pv_PropertyValue2)
except Exception as e:
warning_count += 1
message = 'Unexpected error setting GeoLayer property "' + pv_PropertyName + '"'
traceback.print_exc(file=sys.stdout)
logger.exception(message, e)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"Check the log file for details."))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns:
Nothing.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning = ""
logger = logging.getLogger(__name__)
# GeoLayerID is required
# - non-empty, non-None string.
# - existence of the GeoLayer will also be checked in run_command().
pv_GeoLayerID = self.get_parameter_value(
parameter_name='GeoLayerID', command_parameters=command_parameters)
if not validators.validate_string(pv_GeoLayerID, False, False):
message = "GeoLayerID parameter has no value."
recommendation = "Specify the GeoLayerID parameter to indicate the GeoLayer to process."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# PropertyName is required
pv_PropertyName = self.get_parameter_value(
parameter_name='PropertyName',
command_parameters=command_parameters)
if not validators.validate_string(pv_PropertyName, False, False):
message = "PropertyName parameter has no value."
recommendation = "Specify a property name."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# PropertyType is required
pv_PropertyType = self.get_parameter_value(
parameter_name='PropertyType',
command_parameters=command_parameters)
property_types = ["bool", "float", "int", "long", "str"]
if not validators.validate_string_in_list(
pv_PropertyType, property_types, False, False):
message = 'The requested property type "' + pv_PropertyType + '"" is invalid.'
recommendation = "Specify a valid property type: " + str(
property_types)
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# PropertyValue is required
pv_PropertyValue = self.get_parameter_value(
parameter_name='PropertyValue',
command_parameters=command_parameters)
if not validators.validate_string(pv_PropertyValue, False, False):
# TODO smalers 2017-12-28 add other parameters similar to TSTool to set special values
message = "PropertyValue parameter is not specified."
recommendation = "Specify a property value."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty
# triggers an exception below.
warning = command_util.validate_command_parameter_names(self, warning)
# If any warnings were generated, throw an exception
if len(warning) > 0:
logger.warning(warning)
raise ValueError(warning)
# Refresh the phase severity
self.command_status.refresh_phase_severity(
CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
