def run_command(self):
"""
Run the command. Read the feature classes within a file geodatabase. For each desired feature class (can be
specified by the Subset_Pattern parameter), create a GeoLayer object, and add to the GeoProcessor's geolayer
list.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the required and optional parameter values
pv_SpatialDataFolder = self.get_parameter_value("SpatialDataFolder")
pv_ReadOnlyOneFeatureClass = self.get_parameter_value(
"ReadOnlyOneFeatureClass")
pv_Subset_Pattern = self.get_parameter_value("Subset_Pattern")
pv_GeoLayerID_prefix = self.get_parameter_value("GeoLayerID_prefix")
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_FeatureClass = self.get_parameter_value("FeatureClass")
# Convert the ReadOnlyOneFeatureClass from a string value to a Boolean value.
pv_ReadOnlyOneFeatureClass = string_util.str_to_bool(
pv_ReadOnlyOneFeatureClass)
# Convert the SpatialDataFolder parameter value relative path to an absolute path
sd_folder_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_SpatialDataFolder, self)))
# Run the initial checks on the parameter values. Only continue if the checks passed.
if self.__should_read_gdb(sd_folder_abs):
# If configured to only read one Feature Class into one GeoLayer.
if pv_ReadOnlyOneFeatureClass:
# Run the check to see if the GeoLayer should be read.
if self.__should_read_geolayer(pv_GeoLayerID, True,
pv_FeatureClass, sd_folder_abs):
try:
# Get the full pathname to the feature class
# TODO egiles 2018-01-04 Need to research how to properly document feature class source path
spatial_data_file_absolute = os.path.join(
sd_folder_abs, str(pv_FeatureClass))
# Create a QgsVectorLayer object from the feature class.
qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_feature_class(
sd_folder_abs, pv_FeatureClass)
# Create a GeoLayer and add it to the geoprocessor's GeoLayers list
geolayer_obj = GeoLayer(pv_GeoLayerID,
qgs_vector_layer,
spatial_data_file_absolute)
self.command_processor.add_geolayer(geolayer_obj)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading feature class ({}) from file geodatabase ({}).".format(
pv_FeatureClass, sd_folder_abs)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE,
message, recommendation))
# If configured to read multiple Feature Classes into multiple GeoLayers.
else:
# Get a list of all of the feature classes in the file geodatabase.
fc_list = self.__return_a_list_of_fc(sd_folder_abs)
# Filter the fc_list to only include feature classes that meet the Subset Pattern configuration. If
# the Subset Pattern configuration is None, all feature classes will remain in the fc_list.
fc_list = string_util.filter_list_of_strings(
fc_list, [pv_Subset_Pattern])
# Iterate over the feature classes in the fc_list.
for feature_class in fc_list:
# Determine the GeoLayerID.
if pv_GeoLayerID_prefix:
geolayer_id = "{}_{}".format(pv_GeoLayerID_prefix,
feature_class)
else:
geolayer_id = feature_class
# Run the secondary checks on the parameter values. Only continue if the checks passed.
if self.__should_read_geolayer(geolayer_id, False):
try:
# Get the full pathname to the feature class
# TODO egiles 2018-01-04 Need to research how to properly document feature class source path
spatial_data_file_absolute = os.path.join(
sd_folder_abs, str(feature_class))
# Create a QgsVectorLayer object from the feature class.
qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_feature_class(
sd_folder_abs, feature_class)
# Create a GeoLayer and add it to the geoprocessor's GeoLayers list
geolayer_obj = GeoLayer(
geolayer_id, qgs_vector_layer,
spatial_data_file_absolute)
self.command_processor.add_geolayer(geolayer_obj)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading feature class ({}) from file geodatabase ({}).".format(
feature_class, sd_folder_abs)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE,
message, recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Extract the compressed file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the File and the DeleteFile parameter values.
pv_File = self.get_parameter_value("File")
pv_DeleteFile = self.get_parameter_value("DeleteFile",
default_value="False")
# Convert the File parameter value relative path to an absolute path. Expand for ${Property} syntax.
file_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_File, self)))
# Get the FileType parameter value.
default_file_ext = self.__get_default_file_type(file_abs)
pv_FileType = self.get_parameter_value("FileType",
default_value=default_file_ext)
# Get the OutputFolder parameter value.
parent_folder = io_util.get_path(file_abs)
pv_OutputFolder = self.get_parameter_value("OutputFolder",
default_value=parent_folder)
# Convert the OutputFolder parameter value relative path to an absolute path. Expand for ${Property} syntax.
output_folder_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFolder, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_extract_file(file_abs, output_folder_abs,
pv_FileType):
try:
# If the file is a .zip file, extract the zip file.
if pv_FileType.upper() == "ZIP":
zip_util.unzip_all_files(file_abs, output_folder_abs)
# If the file is a .tar file, extract the tar file.
elif pv_FileType.upper() == "TAR":
zip_util.untar_all_files(file_abs, output_folder_abs)
# If configured, remove the input compressed file.
if string_util.str_to_bool(pv_DeleteFile):
os.remove(file_abs)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error extracting the {} file ({}).".format(
pv_FileType, pv_File)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Write the Table to a delimited file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_TableID = self.get_parameter_value("TableID")
pv_OutputFile = self.get_parameter_value("OutputFile")
pv_Delimiter = self.get_parameter_value("Delimiter", default_value=",")
pv_IncludeColumns = self.get_parameter_value("IncludeColumns",
default_value="*")
pv_ExcludeColumns = self.get_parameter_value("ExcludeColumns",
default_value="")
pv_WriteHeaderRow = self.get_parameter_value("WriteHeaderRow",
default_value="True")
pv_WriteIndexColumn = self.get_parameter_value("WriteIndexColumn",
default_value="False")
pv_SortColumns = self.get_parameter_value("SortColumns")
pv_SortOrder = self.get_parameter_value("SortOrder", default_value="")
pv_ArrayFormat = self.get_parameter_value(
"ArrayFormat", default_value="SquareBrackets")
pv_NullValueFormat = self.get_parameter_value("NullValueFormat",
default_value="Null")
# Convert the IncludeColumns, ExcludeColumns, and SortColumns parameter values to lists.
cols_to_include = string_util.delimited_string_to_list(
pv_IncludeColumns)
cols_to_exclude = string_util.delimited_string_to_list(
pv_ExcludeColumns)
sort_cols_list = string_util.delimited_string_to_list(pv_SortColumns)
# Convert the SortOrder to a dictionary.
sort_dictionary = string_util.delimited_string_to_dictionary_one_value(
pv_SortOrder,
entry_delimiter=",",
key_value_delimiter=":",
trim=True)
# Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
# Covert the Boolean parameters from string to Boolean values.
pv_WriteHeaderRow = string_util.str_to_bool(pv_WriteHeaderRow)
pv_WriteIndexColumn = string_util.str_to_bool(pv_WriteIndexColumn)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_write_table(pv_TableID, output_file_absolute,
pv_Delimiter, sort_cols_list):
try:
# Get the Table object
table = self.command_processor.get_table(pv_TableID)
# Determine if square brackets should be used depending on the user input of the ArrayFormat parameter.
use_sq_brackets = False
if pv_ArrayFormat.upper() == "SQUAREBRACKETS":
use_sq_brackets = True
# Determine if the null values should be used depending on the user input of the NullValueFormat
# parameter.
use_null_value = False
if pv_NullValueFormat.upper() == "NULL":
use_null_value = True
# Write the table to the delimited file.
self.__write_table_to_delimited_file(
output_file_absolute, table, pv_Delimiter, cols_to_include,
cols_to_exclude, pv_WriteHeaderRow, pv_WriteIndexColumn,
sort_cols_list, sort_dictionary, use_sq_brackets,
use_null_value)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error writing Table {} to delimited file {}.".format(
pv_TableID, pv_OutputFile)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def check_command_parameters(self, command_parameters):
"""
Check the command parameters for validity.
Args:
command_parameters: the dictionary of command parameters to check (key:string_value)
Returns: None.
Raises:
ValueError if any parameters are invalid or do not have a valid value.
The command status messages for initialization are populated with validation messages.
"""
warning = ""
# Check that parameter SpatialDataFolder is a non-empty, non-None string.
# - existence of the folder will also be checked in run_command().
pv_SpatialDataFolder = self.get_parameter_value(
parameter_name='SpatialDataFolder',
command_parameters=command_parameters)
if not validators.validate_string(pv_SpatialDataFolder, False, False):
message = "SpatialDataFolder parameter has no value."
recommendation = "Specify text for the SpatialDataFolder parameter to indicate the file geodatabase."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check that optional parameter IfGeoLayerIDExists is either `Replace`, `ReplaceAndWarn`, `Warn`, `Fail`, None.
pv_IfGeoLayerIDExists = self.get_parameter_value(
parameter_name="IfGeoLayerIDExists",
command_parameters=command_parameters)
acceptable_values = ["Replace", "ReplaceAndWarn", "Warn", "Fail"]
if not validators.validate_string_in_list(pv_IfGeoLayerIDExists,
acceptable_values,
none_allowed=True,
empty_string_allowed=True,
ignore_case=True):
message = "IfGeoLayerIDExists parameter value ({}) is not recognized.".format(
pv_IfGeoLayerIDExists)
recommendation = "Specify one of the acceptable values ({}) for the IfGeoLayerIDExists parameter.".format(
acceptable_values)
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check that the optional parameter ReadOnlyOneFeatureClass is a valid Boolean.
pv_ReadOnlyOneFeatureClass = self.get_parameter_value(
parameter_name="ReadOnlyOneFeatureClass",
command_parameters=command_parameters)
if not validators.validate_bool(pv_ReadOnlyOneFeatureClass, True,
False):
message = "ReadOnlyOneFeatureClass is not a valid boolean value."
recommendation = "Specify a valid boolean value for the ReadOnlyOneFeatureClass parameter."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Continue with checks if the ReadOnlyOneFeatureClass is a valid TRUE Boolean.
elif string_util.str_to_bool(pv_ReadOnlyOneFeatureClass):
# Check that parameter GeoLayerID is a non-empty, non-None string.
pv_GeoLayerID = self.get_parameter_value(
parameter_name='GeoLayerID',
command_parameters=command_parameters)
if not validators.validate_string(pv_GeoLayerID, False, False):
message = "GeoLayerID parameter has no value."
recommendation = "Specify the GeoLayerID parameter."
warning += "\n" + message
self.command_status.add_to_log(
CommandPhaseType.INITIALIZATION,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Check for unrecognized parameters.
# This returns a message that can be appended to the warning, which if non-empty triggers an exception below.
warning = command_util.validate_command_parameter_names(self, warning)
# If any warnings were generated, throw an exception.
if len(warning) > 0:
self.logger.warning(warning)
raise ValueError(warning)
else:
# Refresh the phase severity
self.command_status.refresh_phase_severity(
CommandPhaseType.INITIALIZATION, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Initializes the If.
Returns:
None.
"""
logger = logging.getLogger(__name__)
debug = True # Use for troubleshooting
warning_count = 0 # General count for issues
pv_Name = self.get_parameter_value("Name")
pv_Condition = self.get_parameter_value("Condition")
condition_upper = None
if pv_Condition is not None:
condition_upper = pv_Condition.upper()
pv_CompareAsStrings = self.get_parameter_value("CompareAsStrings")
compare_as_strings = False
if pv_CompareAsStrings is not None and pv_CompareAsStrings.upper == "TRUE":
compare_as_strings = True
# TODO smalers 2018-02-18 need to add other special conditions such as empty properties, GeoLayer exists, etc.
# - see TSTool code
try:
condition_eval = False
self.__condition_eval = condition_eval
if pv_Condition is not None and pv_Condition != "":
# Condition is specified, rather than special check.
# First determine the condition operator.
# TODO SAM 2013-12-07 Figure out if there is a more elegant way to do this
# Currently only Value1 Operator Value2 is allowed. Brute force split by finding the operator
logger.info('Evaluating Condition="' + pv_Condition + '"')
pos1 = -1
pos2 = -1
value1 = ""
value2 = ""
op = "??"
if pv_Condition.find("<=") > 0:
pos = pv_Condition.find("<=")
op = "<="
pos1 = pos
pos2 = pos + 2
elif pv_Condition.find("<") > 0:
pos = pv_Condition.find("<")
op = "<"
pos1 = pos
pos2 = pos + 1
elif pv_Condition.find(">=") > 0:
pos = pv_Condition.find(">=")
op = ">="
pos1 = pos
pos2 = pos + 2
elif pv_Condition.find(">") > 0:
pos = pv_Condition.find(">")
op = ">"
pos1 = pos
pos2 = pos + 1
elif pv_Condition.find("==") > 0:
pos = pv_Condition.find("==")
op = "=="
pos1 = pos
pos2 = pos + 2
elif pv_Condition.find("!=") > 0:
pos = pv_Condition.find("!=")
op = "!="
pos1 = pos
pos2 = pos + 2
elif condition_upper.find("!CONTAINS") > 0:
# Put this before the next "CONTAINS" operator
pos = condition_upper.find("!CONTAINS")
op = "!CONTAINS"
pos1 = pos
pos2 = pos + 9
compare_as_strings = True # "!contains" is only used on strings
elif condition_upper.find("CONTAINS") > 0:
pos = condition_upper.find("CONTAINS")
op = "CONTAINS"
pos1 = pos
pos2 = pos + 8
compare_as_strings = True # "contains" is only used on strings
elif pv_Condition.find("=") > 0:
message = "Bad use of = in condition."
recommendation = "Use == to check for equality."
logger.warning(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
else:
message = 'Unknown condition operator for "' + pv_Condition + '"'
recommendation =\
"Make sure condition operator is supported - refer to command editor and documentation."
logger.warning(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
logger.info('operator="' + op + '" pos1=' + str(pos1) +
" pos2=" + str(pos2))
# Now evaluate the left and right sides of the condition
arg1 = pv_Condition[0:pos1].strip()
if debug:
logger.info(
'Left side of condition before property expansion check: "'
+ str(arg1) + '"')
if arg1.find("${") >= 0:
value1 = self.command_processor.expand_parameter_value(
arg1, self)
if debug:
logger.info(
"Left side of condition after property expansion: "
+ value1)
else:
value1 = arg1
if debug:
logger.info("Left side (no expansion needed): " +
value1)
arg2 = pv_Condition[pos2:].strip()
if debug:
logger.info(
'Right side of condition before property expansion check: "'
+ str(arg2) + '"')
if arg2.find("${") >= 0:
value2 = self.command_processor.expand_parameter_value(
arg2, self)
if debug:
logger.info("Right side after property expansion: " +
value2)
else:
value2 = arg2
if debug:
logger.info("Right side (no expansion needed): " +
value2)
# If the arguments are quoted, then all of the following will be false
is_value1_int = string_util.is_int(value1)
is_value2_int = string_util.is_int(value2)
is_value1_float = string_util.is_float(value1)
is_value2_float = string_util.is_float(value2)
is_value1_bool = string_util.is_bool(value1)
is_value2_bool = string_util.is_bool(value2)
# Strip surrounding double quotes for comparisons below - do after above checks for type
value1 = value1.replace('"', "")
value2 = value2.replace('"', "")
if not compare_as_strings and is_value1_int and is_value2_int:
# Do an integer comparison
ivalue1 = int(value1)
ivalue2 = int(value2)
if op == "<=":
if ivalue1 <= ivalue2:
condition_eval = True
elif op == "<":
if ivalue1 < ivalue2:
condition_eval = True
elif op == ">=":
if ivalue1 >= ivalue2:
condition_eval = True
elif op == ">":
if ivalue1 > ivalue2:
condition_eval = True
elif op == "==":
if ivalue1 == ivalue2:
condition_eval = True
elif op == "!=":
if ivalue1 != ivalue2:
condition_eval = True
elif not compare_as_strings and is_value1_float and is_value2_float:
# Compare floats
fvalue1 = float(value1)
fvalue2 = float(value2)
if op == "<=":
if fvalue1 <= fvalue2:
condition_eval = True
elif op == "<":
if fvalue1 < fvalue2:
condition_eval = True
elif op == ">=":
if fvalue1 >= fvalue2:
condition_eval = True
elif op == ">":
if fvalue1 > fvalue2:
condition_eval = True
elif op == "==":
if fvalue1 == fvalue2:
condition_eval = True
elif op == "!=":
if fvalue1 != fvalue2:
condition_eval = True
elif not compare_as_strings and is_value1_bool and is_value2_bool:
# Do a boolean comparison
# - bool("") is False, every other string is True!
bvalue1 = string_util.str_to_bool(value1)
bvalue2 = string_util.str_to_bool(value2)
if debug:
logger.info('Evaluating boolean condition "' +
str(bvalue1) + " " + op + " " +
str(bvalue2))
if op == "<=":
if not bvalue1:
# false <= false or true (does not matter what right side is)
condition_eval = True
elif op == "<":
if not bvalue1 and bvalue2:
# false < true
condition_eval = True
elif op == ">=":
if bvalue1:
# true >= false or true (does not matter what right side is)
condition_eval = True
elif op == ">":
if bvalue1 and not bvalue2:
# true > false
condition_eval = True
elif op == "==":
if bvalue1 == bvalue2:
condition_eval = True
elif op == "!=":
if bvalue1 != bvalue2:
condition_eval = True
elif compare_as_strings or\
(not is_value1_int and not is_value2_int and
not is_value1_float and not is_value2_float and not is_value1_bool and not is_value2_bool):
# Always compare the string values or the input is not other types so assume strings
if op == "CONTAINS":
if value1.find(value2) >= 0:
condition_eval = True
elif op == "!CONTAINS":
if value1.find(value2) < 0:
condition_eval = True
else:
# Do a comparison of the strings to figure out lexicographically order
if value1 < value2:
comp = -1
elif value1 == value2:
comp = 0
else:
comp = 1
if op == "<=":
if comp <= 0:
condition_eval = True
elif op == "<":
if comp < 0:
condition_eval = True
elif op == ">=":
if comp >= 0:
condition_eval = True
elif op == ">":
if comp > 0:
condition_eval = True
elif op == "==":
if comp == 0:
condition_eval = True
elif op == "!=":
if comp != 0:
condition_eval = True
else:
message = 'Left and right have different type - cannot evaluate condition "' + pv_Condition + '"'
recommendation = "Make sure data types on each side of operator are the same - " + \
"refer to command editor and documentation."
logger.warning(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
if debug:
logger.info("Condition evaluated to: " +
str(condition_eval))
# For now leave the following two messages in to reinforce to user the evaluation
# - may only output when in debug mode in the future
if pv_Condition.find("${") >= 0:
# Show the original
message = pv_Condition + " (showing ${Property} notation) evaluates to " + str(
condition_eval)
recommendation = "Informational message."
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.SUCCESS, message,
recommendation))
# Always also show the expanded
message = str(value1) + " " + op + " " + str(
value2) + " evaluates to " + str(condition_eval)
recommendation = "Informational message."
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.SUCCESS, message,
recommendation))
self.__condition_eval = condition_eval
except Exception as e:
warning_count += 1
traceback.print_exc(
file=sys.stdout) # Formatting of error seems to have issue
message = 'Unexpected error in If, Name="' + pv_Name + '"'
logger.error(message, e, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"Check the log file for details."))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
# Obtain the parameter values.
pv_Folder = self.get_parameter_value("Folder")
pv_URL = self.get_parameter_value("URL")
pv_ListFiles = self.get_parameter_value("ListFiles",
default_value="True")
pv_ListFolders = self.get_parameter_value("ListFolders",
default_value="True")
pv_ListProperty = self.get_parameter_value("ListProperty")
pv_IncludePatterns = self.get_parameter_value("IncludePatterns",
default_value="*")
pv_ExcludePatterns = self.get_parameter_value("ExcludePatterns",
default_value="''")
# Convert the IncludeAttributes and ExcludeAttributes to lists.
to_include = string_util.delimited_string_to_list(pv_IncludePatterns)
to_exclude = string_util.delimited_string_to_list(pv_ExcludePatterns)
# Convert the pv_ListFiles and pv_ListFolders to Boolean values.
list_files_bool = string_util.str_to_bool(pv_ListFiles)
list_dirs_bool = string_util.str_to_bool(pv_ListFolders)
# Set the absolute paths for the Folder and the URL to None until proven to exist within the command process.
folder_abs = None
url_abs = None
# If the input is a local folder.
if pv_Folder:
# Convert the Folder parameter value relative path to an absolute path and expand for ${Property} syntax.
folder_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_Folder, self)))
# If the input is a url.
if pv_URL:
# Convert the URL parameter to expand for ${Property} syntax.
url_abs = self.command_processor.expand_parameter_value(
pv_URL, self)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_list_files(folder_abs, url_abs, list_files_bool,
list_dirs_bool, pv_ListProperty):
try:
# If the input is a local folder.
if pv_Folder:
# Get a list of the files in the folder.
files = [
f for f in os.listdir(folder_abs)
if os.path.isfile(os.path.join(folder_abs, f))
]
# Get a list of directories in the folder.
dirs = [
d for d in os.listdir(folder_abs)
if os.path.isdir(os.path.join(folder_abs, d))
]
# If configured to list files and folders, continue.
if list_files_bool and list_dirs_bool:
# Filter the list of files and folders with regards to the IncludePatterns and ExcludePatterns.
output_filtered = string_util.filter_list_of_strings(
files + dirs, to_include, to_exclude)
# If configured to list files, continue.
elif list_files_bool:
# Filter the list of files with regards to the IIncludePatterns and ExcludePatterns.
output_filtered = string_util.filter_list_of_strings(
files, to_include, to_exclude)
# If configured to list folders, continue.
elif list_dirs_bool:
# Filter the list of folders with regards to the IIncludePatterns and ExcludePatterns.
output_filtered = string_util.filter_list_of_strings(
dirs, to_include, to_exclude)
else:
output_filtered = []
# Add the filtered list to the desired ListProperty. Sort the list alphabetically
self.command_processor.set_property(
pv_ListProperty, sorted(output_filtered,
key=str.lower))
# If the input is a url.
if pv_URL:
# If configured to list files and folders, continue.
if list_files_bool and list_dirs_bool:
output_filtered = self.scrape_html(
url_abs, True, True, to_include, to_exclude)
# If configured to list files, continue.
elif list_files_bool:
output_filtered = self.scrape_html(
url_abs, True, False, to_include, to_exclude)
# If configured to list folders, continue.
elif list_dirs_bool:
output_filtered = self.scrape_html(
url_abs, False, True, to_include, to_exclude)
else:
output_filtered = None
# Add the filtered list to the desired ListProperty.
self.command_processor.set_property(
pv_ListProperty, sorted(output_filtered,
key=str.lower))
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error running ListFiles command."
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command.
Remove the specified file from the operating system.
Returns:
Nothing.
Raises:
RuntimeError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_SourceFile = self.get_parameter_value('SourceFile')
pv_IfSourceFileNotFound = self.get_parameter_value(
'IfSourceFileNotFound')
if pv_IfSourceFileNotFound is None or pv_IfSourceFileNotFound == "":
pv_IfSourceFileNotFound = 'Warn' # Default
pv_RemoveIfFolder = self.get_parameter_value('RemoveIfFolder',
default_value="False")
pv_RemoveIfFolder = string_util.str_to_bool(pv_RemoveIfFolder)
# Runtime checks on input
pv_SourceFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_SourceFile, self)))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Remove the file
try:
input_count = 1
if not os.path.exists(pv_SourceFile_absolute):
message = 'The source file does not exist: "' + pv_SourceFile_absolute + '"'
if pv_IfSourceFileNotFound == 'Warn':
warning_count += 1
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(
CommandStatusType.WARNING, message,
"Verify that the source exists at the time the command is run."
))
logger.warning(message)
elif pv_IfSourceFileNotFound == 'Fail':
warning_count += 1
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(
CommandStatusType.FAILURE, message,
"Verify that the source exists at the time the command is run."
))
logger.warning(message)
input_count -= 1
if input_count == 1:
try:
logger.info('Removing file "' + pv_SourceFile_absolute +
'"')
os.remove(pv_SourceFile_absolute)
except:
if pv_RemoveIfFolder:
shutil.rmtree(pv_SourceFile_absolute)
except Exception as e:
warning_count += 1
message = 'Unexpected error removing file "' + pv_SourceFile_absolute + '"'
traceback.print_exc(file=sys.stdout)
logger.exception(message, e)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Set a property on the GeoProcessor.
Returns:
Nothing.
Raises:
RuntimeError if there is any exception running the command.
"""
warning_count = 0
logger = logging.getLogger(__name__)
pv_PropertyName = self.get_parameter_value('PropertyName')
pv_PropertyType = self.get_parameter_value('PropertyType')
pv_PropertyValue = self.get_parameter_value('PropertyValue')
pv_PropertyValues = self.get_parameter_value('PropertyValues')
# Expand the property value string before converting to the requested type
pv_PropertyValue_expanded = self.command_processor.expand_parameter_value(
pv_PropertyValue)
pv_PropertyValues_expanded = self.command_processor.expand_parameter_value(
pv_PropertyValues)
do_list = False # Single property is the default
if pv_PropertyValues is not None and pv_PropertyValues != "":
# Doing a list
do_list = True
try:
parameter_value_as_list = []
parameter_value_to_parse = ""
if do_list:
parameter_value_to_parse = pv_PropertyValues_expanded
else:
# Single property - add to a list as if a single-value list
parameter_value_to_parse = pv_PropertyValue_expanded
# logger.info('Parsing parameter "' + str(parameter_value_to_parse) + "'")
# Parse the list into a string list
# - Remove leading [ and trailing ] so only have simple string list
parameter_value_to_parse = parameter_value_to_parse.strip(
) # First strip whitespace
if parameter_value_to_parse.startswith("["):
parameter_value_to_parse = parameter_value_to_parse[1:]
if parameter_value_to_parse.endswith("]"):
parameter_value_to_parse = parameter_value_to_parse[
0:len(parameter_value_to_parse) - 1]
parameter_value_as_string_list = string_util.delimited_string_to_list(
parameter_value_to_parse, trim=True)
# logger.info('Parsed parameter "' + str(parameter_value_as_string_list) + "'")
# Loop through the list
parameter_value2 = None
for parameter_value in parameter_value_as_string_list:
# Convert the property value string to the requested type
parameter_value2 = None
if pv_PropertyType == 'bool':
# Use the following because conversion of strings to booleans is tricky, too many unexpected True
parameter_value2 = string_util.str_to_bool(parameter_value)
elif pv_PropertyType == 'float':
parameter_value2 = float(parameter_value)
elif pv_PropertyType == 'int':
parameter_value2 = int(parameter_value)
elif pv_PropertyType == 'str':
parameter_value2 = str(parameter_value)
# Now set the object as a property, will be the requested type
if do_list:
# The property is a list
# - for now avoid adding None - later For command should process only valid values
if parameter_value2 is not None:
parameter_value_as_list.append(parameter_value2)
if do_list:
# Doing a list so set the property value to the list
# - list could be empty - is this an issue?
self.command_processor.set_property(pv_PropertyName,
parameter_value_as_list)
# logger.info('Setting parameter "' + str(pv_PropertyName) + '"="' + str(parameter_value_as_list) + '"')
else:
# The property value is a single object and will have been processed in the loop's only iteration
if parameter_value2 is not None:
self.command_processor.set_property(
pv_PropertyName, parameter_value2)
# logger.info('Setting parameter "' + str(pv_PropertyName) + '"="' + str(parameter_value2) + '"')
except Exception as e:
warning_count += 1
message = 'Unexpected error setting property "' + str(
pv_PropertyName) + '"'
logger.error(message, e, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"Check the log file for details."))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Write the Table to an excel file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values except for the OutputCRS
pv_TableID = self.get_parameter_value("TableID")
pv_OutputFile = self.get_parameter_value("OutputFile")
pv_OutputWorksheet = self.get_parameter_value("OutputWorksheet")
pv_ColumnsToInclude = self.get_parameter_value("ColumnsToInclude",
default_value="*")
pv_ColumnsToExclude = self.get_parameter_value("ColumnsToExclude",
default_value="")
pv_WriteIndexColumn = self.get_parameter_value("WriteIndexColumn",
default_value="True")
# Convert the Boolean parameters from string to valid Boolean values.
pv_WriteIndexColumn = string_util.str_to_bool(pv_WriteIndexColumn)
# Convert the ColumnsToInclude and ColumnsToExclude parameter values to lists.
cols_to_include = string_util.delimited_string_to_list(
pv_ColumnsToInclude)
cols_to_exclude = string_util.delimited_string_to_list(
pv_ColumnsToExclude)
# Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_write_table(pv_TableID, output_file_absolute):
try:
# Get the Table object
table = self.command_processor.get_table(pv_TableID)
# Get a list of all the available column names in the Table.
all_cols_names = list(table.df)
# Sort the list to create a second list that only includes the attributes that should be removed.
cols_to_keep = string_util.filter_list_of_strings(
list(table.df),
cols_to_include,
cols_to_exclude,
return_inclusions=True)
# For the columns configured to be written, order them in the same order that they were in with the
# original table. This step ensures that the output table product has the columns in the same order as
# the user would expect.
sorted_cols_to_keep = []
for col in all_cols_names:
if col in cols_to_keep:
sorted_cols_to_keep.append(col)
# Write the tables to an Excel file.
pandas_util.write_df_to_excel(table.df, output_file_absolute,
pv_OutputWorksheet,
sorted_cols_to_keep,
pv_WriteIndexColumn)
# Add the output file to the GeoProcessor's list of output files.
self.command_processor.add_output_file(output_file_absolute)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error writing Table {} to Excel workbook file {}.".format(
pv_TableID, pv_OutputFile)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)