def __get_table_cols_to_exclude(include_col_patterns, exclude_col_patterns,
table):
"""
The command allows for users to select a subset of the Table columns to write to the DataStore database. This
function returns a list of Table columns NOT configured to write data by the user inputs.
Args:
include_col_patterns (str): A comma-separated list of the glob-style patterns filtering which table columns
to write.
exclude_col_patterns (str): A comma-separated list of the glob-style patterns filtering which table columns
to NOT write. This will override IncludeColumns.
table (obj): the Table that is being written to the DataStore
Return:
A list of Table column names configured to write data.
"""
# Convert the IncludeColumns and the ExcludeColumns parameters from strings to lists
table_cols_to_include_patterns = string_util.delimited_string_to_list(
include_col_patterns)
table_cols_to_exclude_patterns = string_util.delimited_string_to_list(
exclude_col_patterns)
# Get a list of all of the columns in the Table
all_table_cols = table.get_column_names()
# Get a list of the columns in the Table that are NOT configured to be pushed to the DataStore
table_cols_to_exclude = string_util.filter_list_of_strings(
all_table_cols,
table_cols_to_include_patterns,
table_cols_to_exclude_patterns,
return_inclusions=False)
# Return a list of Table column names NOT configured to write data.
return table_cols_to_exclude
def run_command(self):
"""
Run the command. Remove the attribute from the GeoLayer.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_AttributeNames = self.get_parameter_value("AttributeNames")
# Convert the AttributeNames parameter from a string to a list.
attribute_names_list = string_util.delimited_string_to_list(
pv_AttributeNames)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_attribute_be_removed(pv_GeoLayerID,
attribute_names_list):
# Run the process.
try:
# Get the input GeoLayer.
input_geolayer = self.command_processor.get_geolayer(
pv_GeoLayerID)
# Iterate over the input attributes to remove.
for attribute_name in attribute_names_list:
# Remove the attribute from the GeoLayer.
input_geolayer.remove_attribute(attribute_name)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error removing attribute(s) ({}) from GeoLayer {}.".format(
pv_AttributeNames, pv_GeoLayerID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
# Get the command parameter values.
pv_GeoLayerIDs = self.get_parameter_value("GeoLayerIDs")
pv_OutputGeoLayerID = self.get_parameter_value("OutputGeoLayerID")
pv_AttributeMap = self.get_parameter_value("AttributeMap",
default_value="")
# Expand for ${Property} syntax.
pv_GeoLayerIDs = self.command_processor.expand_parameter_value(
pv_GeoLayerIDs, self)
# Convert the AttributeMap parameter from string to a list of mapping entries.
attribute_map_entry_list = string_util.delimited_string_to_list(
pv_AttributeMap, delimiter=',')
# The attribute map dictionary contains the attributes of the output merged GeoLayer and the corresponding
# attributes of the input GeoLayers.
# key (str): an attribute of the output, merged GeoLayer
# value (list): a list of attributes from the input GeoLayers that should be mapped to the output attribute
attribute_map_dic = {}
# Iterate over each attribute mapping entry.
for attribute_map_entry in attribute_map_entry_list:
# Get a list of the keys (merged attributes) currently in the attribute map dictionary.
curr_merged_attributes = list(attribute_map_dic.keys())
# If the attribute map entry has the correct format, continue.
if ':' in attribute_map_entry:
# Get the output merged attribute name from the entry.
merged_attr = attribute_map_entry.split(':')[1].strip()
# Get the input attribute name from the entry.
input_attr = attribute_map_entry.split(':')[0].strip()
# If the attribute map entry does has the correct format, the merged_attr and the input_attr variables
# are set to empty strings.
else:
merged_attr = ''
input_attr = ''
# If the merged attribute name is already registered in the attribute mapping dictionary.
if merged_attr in curr_merged_attributes:
# Add the input attribute to the list of input attributes within the dictionary (associated with
# the corresponding merged_attribute).
curr_input_attrs = attribute_map_dic[merged_attr]
curr_input_attrs.append(input_attr)
attribute_map_dic[merged_attr] = curr_input_attrs
# If the merged attribute is not already registered in the attribute mapping dictionary, add the input
# attribute (as a one-item list) to the dictionary (associated with the corresponding merged_attribute).
else:
attribute_map_dic[merged_attr] = [input_attr]
# Convert the GeoLayerIDs parameter from string to list format.
# If configured, list all of the registered GeoLayer IDs.
if pv_GeoLayerIDs == "*":
list_of_geolayer_ids = []
# Iterate over each GeoLayer registered within the GeoProcessor. Add each GeoLayer's ID to the list.
for geolayer_obj in self.command_processor.geolayers:
list_of_geolayer_ids.append(geolayer_obj.id)
# If specific GeoLayer IDs are listed, convert the string into list format.
else:
list_of_geolayer_ids = string_util.delimited_string_to_list(
pv_GeoLayerIDs)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_merge(list_of_geolayer_ids, pv_OutputGeoLayerID):
try:
# A list to hold the GeoLayer IDs of the copied GeoLayers. Copied GeoLayers are only required for this
# command. They will be removed from the GeoProcessor (to save processing space and speed) after the
# processing has been completed. This list will be used to remove the copied GeoLayers.
copied_geolayer_ids = []
# A list to hold the full pathname of the copied GeoLayers (written to disk). The
# qgis:mergevectorlayers requires that the QGSVectorLayer objects are not in memory. This list will be
# used as an input to the qgis:mergevectorlayers algorithm.
copied_geolayer_sourcepath = []
first_geolayer = self.command_processor.get_geolayer(
list_of_geolayer_ids[0])
first_crs = first_geolayer.get_crs()
# Iterate over the GeoLayers to be merged.
for geolayer_id in list_of_geolayer_ids:
# Get the appropriate GeoLayer based on the GeoLayer ID.
geolayer = self.command_processor.get_geolayer(geolayer_id)
# Get an attribute dictionary mapping the GeoLayer attributes that are to be renamed.
# Key: Existing attribute name. Value: New attribute name.
attribute_dictionary = self.__create_attribute_dictionary(
geolayer, attribute_map_dic)
# Make a copy of the GeoLayer and add it to the GeoProcessor. Renaming of attributes will occur on
# a copy of the GeoLayer so that the original GeoLayer's attribute values are not affected.
copied_geolayer = geolayer.deepcopy(
"{}_copyForMerge".format(geolayer_id))
self.command_processor.add_geolayer(copied_geolayer)
# Add the copied GeoLayer ID to the master list.
copied_geolayer_ids.append(copied_geolayer.id)
# Iterate over the GeoLayer's attribute dictionary.
for existing_attr_name, new_attr_name in attribute_dictionary.items(
):
# If the attribute should be renamed, then rename the attribute in the copied GeoLayer.
if not (existing_attr_name == new_attr_name):
copied_geolayer.rename_attribute(
existing_attr_name, new_attr_name)
# Write copied GeoLayer (memory) to the temporary directory (written to disk).
output_file_absolute = os.path.join(
self.command_processor.get_property('TempDir'),
copied_geolayer.id)
on_disk_geolayer = copied_geolayer.write_to_disk(
output_file_absolute)
# Overwrite the copied (memory) GeoLayer in the geoprocessor with the on-disk GeoLayer.
self.command_processor.add_geolayer(on_disk_geolayer)
# Add the source path of the copied on-disk GeoLayer to the master list.
copied_geolayer_sourcepath.append(
on_disk_geolayer.source_path)
# Merge all of the copied GeoLayers (the GeoLayers with the new attribute names).
# Using QGIS algorithm but can also use saga:mergelayers algorithm.
# saga:mergelayers documentation at http://www.saga-gis.org/saga_tool_doc/2.3.0/shapes_tools_2.html
alg_parameters = {
"LAYERS": copied_geolayer_sourcepath,
"CRS": first_crs,
"OUTPUT": "memory:"
}
merged_output = self.command_processor.qgis_processor.runAlgorithm(
"qgis:mergevectorlayers", alg_parameters)
# Create a new GeoLayer and add it to the GeoProcessor's geolayers list.
# in QGIS3, merged_output["OUTPUT"] returns the returns the QGS vector layer object
# see ClipGeoLayer.py for information about value in QGIS2 environment
self.command_processor.add_geolayer(
GeoLayer(pv_OutputGeoLayerID, merged_output["OUTPUT"],
"MEMORY"))
# Release the copied GeoLayers from the GeoProcessor.
for copied_geolayer_id in copied_geolayer_ids:
# Get the copied GeoLayer based on the GeoLayer ID.
copied_geolayer = self.command_processor.get_geolayer(
copied_geolayer_id)
# Remove the copied GeoLayer from the GeoProcessor
self.command_processor.free_geolayer(copied_geolayer)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error merging the following GeoLayers {}.".format(
pv_GeoLayerIDs)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_check(self,
condition,
parameter_name,
parameter_value,
fail_response,
other_values=None):
"""
The run_check utility function is used to store all of the checks done within the command classes. There are
many types of checks that are performed on the command parameter values before a command can be run. Initially,
the GeoProcessor was designed in a way that the checks for each command class (including messages and
recommendations) were within each command class. This new design allows for all of the messages and recommendations
(quite clunky and ugly code) to be pulled away from the command class and placed in this utility function.
A benefit to this design is that popular checks that are called on many commands (for example: is the CRS_code, the
coordinate reference code, valid) are only written out once. Then the same check can be called from however many
command classes necessary. If the message and recommendation strings are changed for a given check, those messages
only have to be changed once here in this utility command rather than in multiple command classes.
Each check has a name called the condition. The checks are alphabetized below by their condition statement. In
the developer documentation there is explanation for each available check. This way, when there are additional
parameters required (entered by the other_values parameter), the developer knows exactly what the check requires.
Before utilizing a check in the command class, it is highly recommended that the developer documentation for that
check if read.
Each check condition statement is written in a way that answers YES (or TRUE) if the check passes. This makes it
easy for checks to be written and standardized by multiple developers.
Args:
self: the class object of the command being checked
condition: the condition statement (or name) of the check that is to be run
parameter_name: the command parameter being checked (the name, not the value)
parameter_value: the command parameter value being checked (the value, not the name)
fail_response: the action that occurs if the check fails. The available options are as follows:
(1) FAIL: a FAIL message is logged and the function returns FALSE for run_the_command Boolean
(2) WARN: a WARN message is logged and the function returns TRUE for run_the_command Boolean
(3) WARNBUTDONOTRUN: a WARN message is logged and the function returns FALSE for run_the_command Boolean
other_values: an optional argument that allows the checks to take in more than one parameter_value for the check
refer to the developer documentation for each individual check to determine if the other_values argument is
used for that check.
Returns:
run_the_command: Boolean. If True, the check has determined that it is ok for the command to run. If False, the
check has determined that it is not ok for the command to run.
"""
# Boolean to determine if the check failed. Set to FALSE until the check FAILS.
check_failed = False
# Check if the attributes in a list exist in a GeoLayer based off of its attribute name.
if condition.upper() == "DOATTRIBUTESEXIST":
geolayer_id = other_values[0]
# Get the GeoLayer.
input_geolayer = self.command_processor.get_geolayer(geolayer_id)
# Get the existing attribute names of the input GeoLayer.
list_of_existing_attributes = input_geolayer.get_attribute_field_names(
)
# Create a list of invalid input attribute names. An invalid attribute name is an input attribute name
# that is not matching any of the existing attribute names of the GeoLayer.
invalid_attrs = []
for attr in parameter_value:
if attr not in list_of_existing_attributes:
invalid_attrs.append(attr)
# The message is dependent on the invalid_attrs varaible. Assign message AFTER invalid_attrs variable has been
# created.
message = "The following attributes ({}) of the {} parameter do" \
" not exist within the GeoLayer ({}).".format(invalid_attrs, parameter_name, geolayer_id)
recommendation = "Specify valid attribute names."
# If there are invalid attributes, the check failed.
if invalid_attrs:
check_failed = True
# Check if the parameter value (absolute file path) has a valid and existing folder.
elif condition.upper() == "DOESFILEPATHHAVEAVALIDFOLDER":
message = 'The folder of the {} ({}) is not a valid folder.'.format(
parameter_name, parameter_value)
recommendation = "Specify a valid folder for the {} parameter.".format(
parameter_name)
output_folder = os.path.dirname(parameter_value)
if not os.path.isdir(output_folder):
check_failed = True
# Check if the GeoLayer of the parameter value (GeoLayer ID) has the correct geometry type.
elif condition.upper() == "DOESGEOLAYERIDHAVECORRECTGEOMETRY":
desired_geom_type_list = [item.upper() for item in other_values[0]]
message = 'The {} ({}) does not have geometry in the correct ' \
'format ({}).'.format(parameter_name, parameter_value, desired_geom_type_list)
recommendation = 'Specify a GeoLayerID of a GeoLayer with geometry in' \
' correct format ({}).'.format(desired_geom_type_list)
if not self.command_processor.get_geolayer(
parameter_value).get_geometry().upper(
) in desired_geom_type_list:
check_failed = True
# Check if the GeoLayer of the parameter value (GeoLayer ID) has a different CRS than another GeoLayer (referenced
# by its GeoLayer ID)
elif condition.upper() == "DOGEOLAYERIDSHAVEMATCHINGCRS":
second_parameter_name = other_values[0]
second_parameter_value = other_values[1]
message = 'The {} ({}) and the {} ({}) do not have the same coordinate reference' \
' system.'.format(parameter_name, parameter_value, second_parameter_name, second_parameter_value)
recommendation = 'Specify GeoLayers that have the same coordinate reference system.'
input_crs = self.command_processor.get_geolayer(
parameter_value).get_crs()
second_crs = self.command_processor.get_geolayer(
second_parameter_value).get_crs()
if not input_crs == second_crs:
check_failed = True
# Check if the parameter value (crs code)is a valid CRS code usable in the QGIS environment.
elif condition.upper() == "ISCRSCODEVALID":
message = 'The {} ({}) is not a valid CRS code.'.format(
parameter_name, parameter_value)
recommendation = 'Specify a valid CRS code (EPSG codes are an approved format).'
if qgis_util.get_qgscoordinatereferencesystem_obj(
parameter_value) is None:
check_failed = True
# Check if the parameter value (DataStoreID) is an existing DataStoreID.
elif condition.upper() == "ISDATASTOREIDEXISTING":
message = 'The {} ({}) is not a valid DataStore ID.'.format(
parameter_name, parameter_value)
recommendation = 'Specify a valid DataStore ID.'
if not self.command_processor.get_datastore(parameter_value):
check_failed = True
# Check if the parameter value (DataStore ID) is a unique DataStoreID.
elif condition.upper() == "ISDATASTOREIDUNIQUE":
message = 'The {} ({}) value is already in use as a DataStore ID.'.format(
parameter_name, parameter_value)
recommendation = 'Specify a new {}.'.format(parameter_name)
if self.command_processor.get_geolayer(parameter_value):
check_failed = True
pv_IfDataStoreIDExists = self.get_parameter_value(
"IfDataStoreIDExists", default_value="Replace")
if pv_IfDataStoreIDExists.upper() == "REPLACEANDWARN":
fail_response = "WARN"
elif pv_IfDataStoreIDExists.upper() == "WARN":
fail_response = "WARNBUTDONOTRUN"
elif pv_IfDataStoreIDExists.upper() == "FAIL":
fail_response = "FAIL"
elif pv_IfDataStoreIDExists.upper() == "OPEN":
check_failed = False
elif pv_IfDataStoreIDExists.upper() == "REPLACE":
check_failed = False
# Check if the parameter value (Table Name) is unique within the DataStore.
elif condition.upper() == "ISDATASTORETABLEUNIQUE":
data_store_id = other_values[0]
message = "The {} ({}) value is already an existing table in the {} DataStore.".format(
parameter_name, parameter_value, data_store_id)
recommendation = "Specify a unique {} value.".format(parameter_name)
data_store_obj = self.command_processor.get_datastore(data_store_id)
list_of_tables = data_store_obj.return_table_names()
if parameter_value in list_of_tables:
check_failed = True
# Check if the parameter value (column name) is a valid column name of a delimited file.
elif condition.upper() == "ISDELIMITEDFILECOLUMNNAMEVALID":
delimited_file_abs = other_values[0]
delimiter = other_values[1]
message = "The {} ({}) is not a valid column name in the delimited file ({}).".format(
parameter_name, parameter_value, delimited_file_abs)
recommendation = "Specify an existing and valid {}.".format(
parameter_name)
if parameter_value not in io_util.get_col_names_from_delimited_file(
delimited_file_abs, delimiter):
check_failed = True
# Check if the paramter value (sheet name) is a valid sheet name of an excel file.
elif condition.upper() == "ISEXCELSHEETNAMEVALID":
excel_file_abs = other_values[0]
message = "The {} ({}) is not a valid excel worksheet name in the excel file ({}).".format(
parameter_name, parameter_value, excel_file_abs)
recommendation = "Specify an existing and valid {}.".format(
parameter_name)
excel_workbook_obj = pandas_util.create_excel_workbook_obj(
excel_file_abs)
excel_worksheet_list = excel_workbook_obj.sheet_names
if parameter_value not in excel_worksheet_list:
check_failed = True
# Check if the parameter value (feature class) is within a file geodatabase.
elif condition.upper() == "ISFEATURECLASSINFGDB":
file_gdb_path_abs = other_values[0]
message = "The {} ({}) is not a valid feature class in the file geodatabase ({}).".format(
parameter_name, parameter_value, file_gdb_path_abs)
recommendation = "Specify an existing and valid {}.".format(
parameter_name)
ogr.UseExceptions()
driver = ogr.GetDriverByName("OpenFileGDB")
gdb = driver.Open(file_gdb_path_abs)
feature_class_list = []
for feature_class_idx in range(gdb.GetLayerCount()):
feature_class = gdb.GetLayerByIndex(feature_class_idx)
feature_class_list.append(feature_class.GetName())
if parameter_value not in feature_class_list:
check_failed = True
# Check if the parameter value (absolute file path) is a valid and existing file.
elif condition.upper() == "ISFILEPATHVALID":
message = "The {} ({}) is not a valid file.".format(
parameter_name, parameter_value)
recommendation = "Specify a valid file for the {} parameter.".format(
parameter_name)
if not os.path.isfile(parameter_value):
check_failed = True
# Check if the parameter value (absolute folder path) is a valid file geodatabase.
elif condition.upper() == "ISFOLDERAFGDB":
message = "The {} ({}) is not a valid file geodatabase.".format(
parameter_name, parameter_value)
recommendation = "Specify a valid file geodatabase for the {} parameter.".format(
parameter_name)
ogr.UseExceptions()
driver = ogr.GetDriverByName("OpenFileGDB")
if driver.Open(parameter_value) is None:
check_failed = True
# Check if the parameter value (absolute folder path) is a valid and existing folder.
elif condition.upper() == "ISFOLDERPATHVALID":
message = "The {} ({}) is not a valid folder.".format(
parameter_name, parameter_value)
recommendation = "Specify a valid folder for the {} parameter.".format(
parameter_name)
if not os.path.isdir(parameter_value):
check_failed = True
# Check if the parameter value (GeoLayerID) is an existing GeoLayerID.
elif condition.upper() == "ISGEOLAYERIDEXISTING":
message = 'The {} ({}) is not a valid GeoLayer ID.'.format(
parameter_name, parameter_value)
recommendation = 'Specify a valid GeoLayer ID.'
if not self.command_processor.get_geolayer(parameter_value):
check_failed = True
# Check if the parameter value (GeoLayer ID) is a unique GeoLayerID.
elif condition.upper() == "ISGEOLAYERIDUNIQUE":
message = 'The {} ({}) value is already in use as a GeoLayer ID.'.format(
parameter_name, parameter_value)
recommendation = 'Specify a new {}.'.format(parameter_name)
if self.command_processor.get_geolayer(parameter_value):
check_failed = True
pv_IfGeoLayerIDExists = self.get_parameter_value(
"IfGeoLayerIDExists", default_value="Replace")
if pv_IfGeoLayerIDExists.upper() == "REPLACEANDWARN":
fail_response = "WARN"
elif pv_IfGeoLayerIDExists.upper() == "WARN":
fail_response = "WARNBUTDONOTRUN"
elif pv_IfGeoLayerIDExists.upper() == "FAIL":
fail_response = "FAIL"
else:
check_failed = False
# Check if the parameter value (integer) is between or at two values/numbers.
elif condition.upper() == "ISINTBETWEENRANGE":
int_min = other_values[0]
int_max = other_values[1]
message = 'The {} ({}) must be at or between {} & {}'.format(
parameter_name, parameter_value, int_min, int_max)
recommendation = 'Specify a valid {} value.'.format(parameter_name)
if not validate_int_in_range(parameter_value, int_min, int_max, False,
False):
check_failed = True
# Check if the length of a list is correct.
elif condition.upper() == "ISLISTLENGTHCORRECT":
delimiter = other_values[0]
correct_length = other_values[1]
message = 'The {} ({}) must have {} number of items.'.format(
parameter_name, parameter_value, correct_length)
recommendation = 'Specify a list of {} items for the {} parameter.'.format(
correct_length, parameter_name)
# Convert the string into a list.
list_of_strings = string_util.delimited_string_to_list(
parameter_value, delimiter)
if len(list_of_strings) != correct_length:
check_failed = True
# Check if the property name is a unique property name
elif condition.upper() == "ISPROPERTYUNIQUE":
message = 'The {} ({}) value is already in use.'.format(
parameter_name, parameter_value)
recommendation = 'Specify a new {}.'.format(parameter_name)
if self.command_processor.get_property(parameter_value):
check_failed = True
pv_IfPropertyExists = self.get_parameter_value(
"IfPropertyExists", default_value="Replace")
if pv_IfPropertyExists.upper() == "REPLACEANDWARN":
fail_response = "WARN"
elif pv_IfPropertyExists.upper() == "WARN":
fail_response = "WARNBUTDONOTRUN"
elif pv_IfPropertyExists.upper() == "FAIL":
fail_response = "FAIL"
else:
check_failed = False
# Check if the input string is a valid QGSExpression.
elif condition.upper() == "ISQGSEXPRESSIONVALID":
message = "{} ({}) is not a valid QgsExpression.".format(
parameter_name, parameter_value)
recommendation = "Specify a valid QgsExpression for {}.".format(
parameter_name)
if qgis_util.get_qgsexpression_obj(parameter_value) is None:
check_failed = True
# Check if the input string is the correct length.
elif condition.upper() == "ISSTRINGLENGTHCORRECT":
correct_length = other_values[0]
message = 'The {} ({}) must have exactly {} character(s).'.format(
parameter_name, parameter_value, correct_length)
recommendation = 'Specify a string with {} characters for the {} parameter.'.format(
correct_length, parameter_name)
# Convert the string into a list.
if len(parameter_value) != correct_length:
check_failed = True
# Check if the parameter value (Table ID) is an existing Table ID.
elif condition.upper() == "ISTABLEIDEXISTING":
message = 'The {} ({}) is not a valid Table ID.'.format(
parameter_name, parameter_value)
recommendation = 'Specify a valid Table ID.'
if not self.command_processor.get_table(parameter_value):
check_failed = True
# Check if the parameter value (Table ID) is a unique TableID.
elif condition.upper() == "ISTABLEIDUNIQUE":
message = 'The {} ({}) value is already in use as a Table ID.'.format(
parameter_name, parameter_value)
recommendation = 'Specify a new {}.'.format(parameter_name)
if self.command_processor.get_table(parameter_value):
check_failed = True
pv_IfTableIDExists = self.get_parameter_value(
"IfTableIDExists", default_value="Replace")
if pv_IfTableIDExists.upper() == "REPLACEANDWARN":
fail_response = "WARN"
elif pv_IfTableIDExists.upper() == "WARN":
fail_response = "WARNBUTDONOTRUN"
elif pv_IfTableIDExists.upper() == "FAIL":
fail_response = "FAIL"
else:
check_failed = False
# Check if the parameter value (Table Name) is a table within the DataStore.
elif condition.upper() == "ISTABLEINDATASTORE":
data_store_id = other_values[0]
message = "{} ({}) is not an existing table in the {} DataStore.".format(
parameter_name, parameter_value, data_store_id)
recommendation = "Specify a valid {} value.".format(parameter_name)
data_store_obj = self.command_processor.get_datastore(data_store_id)
list_of_tables = data_store_obj.return_table_names()
if parameter_value not in list_of_tables:
check_failed = True
# Check if the file is a valid tar file.
elif condition.upper() == "ISTARFILE":
message = "{} ({}) is not a valid TAR file.".format(
parameter_name, parameter_value)
recommendation = "Specify a valid TAR file for {}.".format(
parameter_name)
if not zip_util.is_tar_file(parameter_value):
check_failed = True
# Check if the input string is a valid URL.
elif condition.upper() == "ISURLVALID":
message = "{} ({}) is not a valid URL.".format(parameter_name,
parameter_value)
recommendation = "Specify a valid URL for {}.".format(parameter_name)
try:
urlopen(parameter_value)
except:
check_failed = True
# Check if the file is a valid zip file.
elif condition.upper() == "ISZIPFILE":
message = "{} ({}) is not a valid ZIP file.".format(
parameter_name, parameter_value)
recommendation = "Specify a valid ZIP file for {}.".format(
parameter_name)
if not zip_util.is_zip_file(parameter_value):
check_failed = True
else:
message = "Check {} is not a valid check in the validators library.".format(
condition)
recommendation = "Contact the maintainers of the GeoProcessor software."
check_failed = True
fail_response = "FAIL"
# If the check failed, increase the warning count of the command instance by one.
if check_failed:
self.warning_count += 1
# If configured, log a FAILURE message about the failed check. Set the run_the_command boolean to False.
if fail_response.upper() == "FAIL":
self.logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
run_the_command = False
# If configured, log a WARNING message about the failed check. Set the run_the_command boolean to True.
elif fail_response.upper() == "WARN":
self.logger.warning(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.WARNING, message,
recommendation))
run_the_command = True
# If configured, log a WARNING message about the failed check. Set the run_the_command boolean to False.
elif fail_response.upper() == "WARNBUTDONOTRUN":
self.logger.warning(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.WARNING, message,
recommendation))
run_the_command = False
# If the check passed, set the run_the_command boolean to True.
else:
run_the_command = True
# Return the run_the_command boolean.
return run_the_command
def run_command(self):
"""
Run the command. Write the Table to a delimited file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_TableID = self.get_parameter_value("TableID")
pv_OutputFile = self.get_parameter_value("OutputFile")
pv_Delimiter = self.get_parameter_value("Delimiter", default_value=",")
pv_IncludeColumns = self.get_parameter_value("IncludeColumns",
default_value="*")
pv_ExcludeColumns = self.get_parameter_value("ExcludeColumns",
default_value="")
pv_WriteHeaderRow = self.get_parameter_value("WriteHeaderRow",
default_value="True")
pv_WriteIndexColumn = self.get_parameter_value("WriteIndexColumn",
default_value="False")
pv_SortColumns = self.get_parameter_value("SortColumns")
pv_SortOrder = self.get_parameter_value("SortOrder", default_value="")
pv_ArrayFormat = self.get_parameter_value(
"ArrayFormat", default_value="SquareBrackets")
pv_NullValueFormat = self.get_parameter_value("NullValueFormat",
default_value="Null")
# Convert the IncludeColumns, ExcludeColumns, and SortColumns parameter values to lists.
cols_to_include = string_util.delimited_string_to_list(
pv_IncludeColumns)
cols_to_exclude = string_util.delimited_string_to_list(
pv_ExcludeColumns)
sort_cols_list = string_util.delimited_string_to_list(pv_SortColumns)
# Convert the SortOrder to a dictionary.
sort_dictionary = string_util.delimited_string_to_dictionary_one_value(
pv_SortOrder,
entry_delimiter=",",
key_value_delimiter=":",
trim=True)
# Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
# Covert the Boolean parameters from string to Boolean values.
pv_WriteHeaderRow = string_util.str_to_bool(pv_WriteHeaderRow)
pv_WriteIndexColumn = string_util.str_to_bool(pv_WriteIndexColumn)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_write_table(pv_TableID, output_file_absolute,
pv_Delimiter, sort_cols_list):
try:
# Get the Table object
table = self.command_processor.get_table(pv_TableID)
# Determine if square brackets should be used depending on the user input of the ArrayFormat parameter.
use_sq_brackets = False
if pv_ArrayFormat.upper() == "SQUAREBRACKETS":
use_sq_brackets = True
# Determine if the null values should be used depending on the user input of the NullValueFormat
# parameter.
use_null_value = False
if pv_NullValueFormat.upper() == "NULL":
use_null_value = True
# Write the table to the delimited file.
self.__write_table_to_delimited_file(
output_file_absolute, table, pv_Delimiter, cols_to_include,
cols_to_exclude, pv_WriteHeaderRow, pv_WriteIndexColumn,
sort_cols_list, sort_dictionary, use_sq_brackets,
use_null_value)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error writing Table {} to delimited file {}.".format(
pv_TableID, pv_OutputFile)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Read the Table from the DataStore
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_DataStoreID = self.get_parameter_value("DataStoreID")
pv_DataStoreTable = self.get_parameter_value("DataStoreTable")
pv_Sql = self.get_parameter_value("Sql")
pv_SqlFile = self.get_parameter_value("SqlFile")
pv_Top = self.get_parameter_value("Top")
pv_TableID = self.get_parameter_value("TableID")
pv_IncludeColumns = self.get_parameter_value("IncludeColumns",
default_value="*")
pv_ExcludeColumns = self.get_parameter_value("ExcludeColumns",
default_value="")
# Expand for ${Property} syntax.
pv_DataStoreID = self.command_processor.expand_parameter_value(
pv_DataStoreID, self)
pv_DataStoreTable = self.command_processor.expand_parameter_value(
pv_DataStoreTable, self)
pv_Sql = self.command_processor.expand_parameter_value(pv_Sql, self)
pv_TableID = self.command_processor.expand_parameter_value(
pv_TableID, self)
# Convert the IncludeColumns and ExcludeColumns parameter values to lists.
cols_to_include = string_util.delimited_string_to_list(
pv_IncludeColumns)
cols_to_exclude = string_util.delimited_string_to_list(
pv_ExcludeColumns)
# If available, convert the SqlFile parameter value relative path to an absolute path and expand for
# ${Property} syntax.
if pv_SqlFile:
pv_SqlFile = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_SqlFile, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_read_table(pv_SqlFile, pv_TableID, pv_DataStoreID):
try:
# Get the DataStore object
datastore = self.command_processor.get_datastore(
pv_DataStoreID)
# Set the SQL statement to None until proof that SQL statement exists.
sql_statement = None
# If using the Sql method, the sql_statement is the user-provided sql statement.
if pv_Sql:
sql_statement = pv_Sql
if '%' in sql_statement:
sql_statement = sql_statement.replace('%', '%%')
# If using the Sql method, the sql_statement is the user-provided sql statement within a file.
if pv_SqlFile:
# Get the SQL statement from the file.
f = open(pv_SqlFile, 'r')
sql_statement = f.read().strip()
if '%' in sql_statement:
sql_statement = sql_statement.replace('%', '%%')
# Create the Table from the DataStore.
table = self.__read_table_from_datastore(
datastore, pv_DataStoreTable, pv_TableID, pv_Top,
sql_statement, cols_to_include, cols_to_exclude)
# Add the table to the GeoProcessor's Tables list.
self.command_processor.add_table(table)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading Table {} from DataStore ({}).".format(
pv_TableID, pv_DataStoreID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Create the GeoLayer with the input geometries. Add GeoLayer to the GeoProcessor's geolayers
list.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_NewGeoLayerID = self.get_parameter_value("NewGeoLayerID")
pv_GeometryFormat = self.get_parameter_value("GeometryFormat").upper()
pv_GeometryData = self.get_parameter_value("GeometryData")
pv_CRS = self.get_parameter_value("CRS")
if self.__should_geolayer_be_created(pv_NewGeoLayerID, pv_CRS,
pv_GeometryFormat,
pv_GeometryData):
try:
# If the geometry format is bounding box, continue.
if pv_GeometryFormat == "BOUNDINGBOX":
# Convert the geometry input from a string to a list of strings.
# Items are in the following order:
# 1. Left (West) bound coordinate
# 2. Bottom (South) bound coordinate
# 3. Right (East) bound coordinate
# 4. Top (North) bound coordinate
NSWE_extents = string_util.delimited_string_to_list(
pv_GeometryData)
NW = "{} {}".format(NSWE_extents[0], NSWE_extents[3])
NE = "{} {}".format(NSWE_extents[2], NSWE_extents[3])
SE = "{} {}".format(NSWE_extents[2], NSWE_extents[1])
SW = "{} {}".format(NSWE_extents[0], NSWE_extents[1])
wkt_conversion = "POLYGON(({}, {}, {}, {}))".format(
NW, NE, SE, SW)
# Create the QgsVectorLayer. BoundingBox will always create a POLYGON layer.
layer = qgis_util.create_qgsvectorlayer(
"Polygon", pv_CRS, "layer")
# Create the QgsGeometry object for the bounding box geometry.
qgs_geometry = qgis_util.create_qgsgeometry(
"WKT", wkt_conversion)
# If the geometry format is Well-Known Text, continue.
elif pv_GeometryFormat == "WKT":
# Get the equivalent QGS geometry type to the input WKT geometry.
# Ex: MultiLineString is converted to LineString.
qgsvectorlayer_geom_type = qgis_util.get_geometrytype_qgis_from_wkt(
pv_GeometryData)
# Create the QgsVectorLayer. The geometry type will be determined from the WKT specifications.
layer = qgis_util.create_qgsvectorlayer(
qgsvectorlayer_geom_type, pv_CRS, "layer")
# Create the QgsGeometry object for the Well-Known Text geometry.
qgs_geometry = qgis_util.create_qgsgeometry(
"WKT", pv_GeometryData)
# If the geometry format is Well-Known Binary, continue.
elif pv_GeometryFormat == "WKB":
# Create the QgsGeometry object for the Well-Known Binary geometry.
qgs_geometry = qgis_util.create_qgsgeometry(
"WKB", pv_GeometryData)
# Get the equivalent Well-Known Text for the geometry.
qgs_geometry_as_wkt = qgs_geometry.exportToWkt()
# Get the equivalent QGS geometry type to the input WKT geometry.
# Ex: MultiLineString is converted to LineString.
qgsvectorlayer_geom_type = qgis_util.get_geometrytype_qgis_from_wkt(
qgs_geometry_as_wkt)
# Create the QgsVectorLayer. The geometry type will be determined from the WKB specifications.
layer = qgis_util.create_qgsvectorlayer(
qgsvectorlayer_geom_type, pv_CRS, "layer")
# Add the feature (with the appropriate geometry) to the Qgs Vector Layer.
qgis_util.add_feature_to_qgsvectorlayer(layer, qgs_geometry)
# Create a new GeoLayer with the QgsVectorLayer and add it to the GeoProcesor's geolayers list.
new_geolayer = GeoLayer(pv_NewGeoLayerID, layer, "MEMORY")
self.command_processor.add_geolayer(new_geolayer)
# Raise an exception if an unexpected error occurs during the process.
except Exception as e:
self.warning_count += 1
message = "Unexpected error creating GeoLayer ({}).".format(
pv_NewGeoLayerID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Read the Table from the delimited file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_InputFile = self.get_parameter_value("InputFile")
pv_Delimiter = self.get_parameter_value("Delimiter", default_value=",")
pv_TableID = self.get_parameter_value("TableID")
pv_HeaderLines = int(
self.get_parameter_value("HeaderLines", default_value="0"))
pv_NullValues = self.get_parameter_value("NullValues",
default_value="''")
# Convert the InputFile parameter value relative path to an absolute path and expand for ${Property} syntax
input_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_InputFile, self)))
# Convert the NullValues parameter values to a list.
pv_NullValues = string_util.delimited_string_to_list(pv_NullValues)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_read_table(input_file_absolute, pv_TableID):
try:
# Create the table from the delimited file.
table = self.__read_table_from_delimited_file(
input_file_absolute, pv_TableID, pv_Delimiter,
pv_HeaderLines, pv_NullValues)
# Add the table to the GeoProcessor's Tables list.
self.command_processor.add_table(table)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading Table {} from delimited file ({}).".format(
pv_TableID, input_file_absolute)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Run the program, which can generate output files.
Returns:
None.
Raises:
ValueError: if a runtime input error occurs.
RuntimeError: if a runtime error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
logger.info('In RunProgram.run_command')
# Get data for the command
print("command parameters=" +
string_util.format_dict(self.command_parameters))
pv_CommandLine = self.get_parameter_value('CommandLine')
pv_UseCommandShell = self.get_parameter_value('UseCommandShell')
use_command_shell = False # Default
if pv_UseCommandShell is not None and pv_UseCommandShell == 'True':
use_command_shell = True
pv_IncludeParentEnvVars = self.get_parameter_value(
'IncludeParentEnvVars')
include_parent_env_vars = True # Default
if pv_IncludeParentEnvVars is not None and pv_IncludeParentEnvVars == 'False':
include_parent_env_vars = False
pv_IncludeEnvVars = self.get_parameter_value('IncludeEnvVars')
include_env_vars_dict = None
if pv_IncludeEnvVars is not None and pv_IncludeEnvVars != "":
# Have specified environment variables to include
# - expand the environment variable value using processor properties
include_env_vars_dict = string_util.delimited_string_to_dictionary_one_value(
pv_IncludeEnvVars, key_value_delimiter="=", trim=True)
for key, value in include_env_vars_dict.items():
include_env_vars_dict[
key] = self.command_processor.expand_parameter_value(
value, self)
# Add environment variables individually by name
# - these are used when a list of parameters is difficult to parse
# - this is kind of ugly but meets requirements in the short term
pv_IncludeEnvVarName1 = self.get_parameter_value('IncludeEnvVarName1')
pv_IncludeEnvVarValue1 = self.get_parameter_value(
'IncludeEnvVarValue1')
if pv_IncludeEnvVarName1 is not None and pv_IncludeEnvVarName1 != "":
if include_env_vars_dict is None:
include_env_vars_dict = {}
include_env_vars_dict[
pv_IncludeEnvVarName1] = pv_IncludeEnvVarValue1
pv_IncludeEnvVarName2 = self.get_parameter_value('IncludeEnvVarName2')
pv_IncludeEnvVarValue2 = self.get_parameter_value(
'IncludeEnvVarValue2')
if pv_IncludeEnvVarName2 is not None and pv_IncludeEnvVarName2 != "":
if include_env_vars_dict is None:
include_env_vars_dict = {}
include_env_vars_dict[
pv_IncludeEnvVarName2] = pv_IncludeEnvVarValue2
pv_IncludeEnvVarName3 = self.get_parameter_value('IncludeEnvVarName3')
pv_IncludeEnvVarValue3 = self.get_parameter_value(
'IncludeEnvVarValue3')
if pv_IncludeEnvVarName3 is not None and pv_IncludeEnvVarName3 != "":
if include_env_vars_dict is None:
include_env_vars_dict = {}
include_env_vars_dict[
pv_IncludeEnvVarName3] = pv_IncludeEnvVarValue3
pv_IncludeEnvVarName4 = self.get_parameter_value('IncludeEnvVarName4')
pv_IncludeEnvVarValue4 = self.get_parameter_value(
'IncludeEnvVarValue4')
if pv_IncludeEnvVarName4 is not None and pv_IncludeEnvVarName4 != "":
if include_env_vars_dict is None:
include_env_vars_dict = {}
include_env_vars_dict[
pv_IncludeEnvVarName4] = pv_IncludeEnvVarValue4
pv_IncludeEnvVarName5 = self.get_parameter_value('IncludeEnvVarName5')
pv_IncludeEnvVarValue5 = self.get_parameter_value(
'IncludeEnvVarValue5')
if pv_IncludeEnvVarName5 is not None and pv_IncludeEnvVarName5 != "":
if include_env_vars_dict is None:
include_env_vars_dict = {}
include_env_vars_dict[
pv_IncludeEnvVarName5] = pv_IncludeEnvVarValue5
pv_ExcludeEnvVars = self.get_parameter_value('ExcludeEnvVars')
exclude_env_vars_list = None
if pv_ExcludeEnvVars is not None and pv_ExcludeEnvVars != "":
# Have specified environment variables to exclude
exclude_env_vars_list = string_util.delimited_string_to_list(
pv_ExcludeEnvVars, trim=True)
pv_OutputFiles = self.get_parameter_value('OutputFiles')
output_files_list = None
if pv_OutputFiles is not None and pv_OutputFiles != "":
# Have specified output files to add to command output files
output_files_list = string_util.delimited_string_to_list(
pv_OutputFiles, trim=True)
# Expand each output file
ifile = -1
for output_file in output_files_list:
ifile = ifile + 1
output_files_list[ifile] = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
output_file, self)))
logger.info('Command line before expansion="' + pv_CommandLine + '"')
# Runtime checks on input
command_line_expanded = self.command_processor.expand_parameter_value(
pv_CommandLine, self)
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Run the program as a subprocess
try:
logger.info('Running command line "' + command_line_expanded + '"')
# Create the environment dictionary
env_dict = self.create_env_dict(include_parent_env_vars,
include_env_vars_dict,
exclude_env_vars_list)
print("env_dict=" + string_util.format_dict(env_dict))
# TODO smalers 2018-12-16 evaluate using shlex.quote() to handle command string
# TODO smalers 2018-12-16 handle standard input and output
p = subprocess.Popen(command_line_expanded,
shell=use_command_shell,
env=env_dict)
# Wait for the process to terminate since need it to be done before other commands do their work
# with the command output.
p.wait()
return_status = p.poll()
if return_status != 0:
warning_count += 1
message = 'Nonzero return status running program "' + command_line_expanded + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
except Exception as e:
warning_count += 1
message = 'Unexpected error running program "' + command_line_expanded + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
except:
warning_count += 1
message = 'Unexpected error running program "' + command_line_expanded + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
# If any output files were indicated, add to the command output if they exist
if output_files_list is not None and len(output_files_list) > 0:
for output_file in output_files_list:
if os.path.isfile(output_file):
# Add the log file to output
self.command_processor.add_output_file(output_file)
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
logger.warning(message)
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Write the processor properties to a file.
Returns:
Nothing.
Raises:
RuntimeError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_OutputFile = self.get_parameter_value('OutputFile')
pv_IncludeProperties = self.get_parameter_value('IncludeProperties')
include_properties = [] # Default
if pv_IncludeProperties is not None and len(pv_IncludeProperties) > 0:
include_properties = string_util.delimited_string_to_list(
pv_IncludeProperties)
pv_WriteMode = self.get_parameter_value('WriteMode')
write_mode = pv_WriteMode
if pv_WriteMode is None or pv_WriteMode == "":
write_mode = 'Overwrite' # Default
pv_FileFormat = self.get_parameter_value('FileFormat')
file_format = pv_FileFormat
if pv_FileFormat is None or pv_FileFormat == "":
file_format = 'NameTypeValue' # Default
pv_SortOrder = self.get_parameter_value('SortOrder')
sort_order = 0 # no sort
if pv_SortOrder is not None:
if pv_SortOrder == 'Ascending':
sort_order = 1
elif pv_SortOrder == 'Descending':
sort_order = -1
# Runtime checks on input
pv_OutputFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Write the output file
try:
problems = [] # Empty list of properties
io_util.write_property_file(pv_OutputFile_absolute,
self.command_processor.properties,
include_properties, write_mode,
file_format, sort_order, problems)
# Record any problems that were found
for problem in problems:
warning_count += 1
logger.error(problem)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, problem,
"See the log file for details."))
except Exception as e:
warning_count += 1
traceback.print_exc(
file=sys.stdout) # Formatting of error seems to have issue
message = 'Unexpected error writing file "' + pv_OutputFile_absolute + '"'
logger.error(message, e, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
except:
warning_count += 1
traceback.print_exc(
file=sys.stdout) # Formatting of error seems to have issue
message = 'Unexpected error writing file "' + pv_OutputFile_absolute + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
logger.warning(message)
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
# Obtain the parameter values.
pv_Folder = self.get_parameter_value("Folder")
pv_URL = self.get_parameter_value("URL")
pv_ListFiles = self.get_parameter_value("ListFiles",
default_value="True")
pv_ListFolders = self.get_parameter_value("ListFolders",
default_value="True")
pv_ListProperty = self.get_parameter_value("ListProperty")
pv_IncludePatterns = self.get_parameter_value("IncludePatterns",
default_value="*")
pv_ExcludePatterns = self.get_parameter_value("ExcludePatterns",
default_value="''")
# Convert the IncludeAttributes and ExcludeAttributes to lists.
to_include = string_util.delimited_string_to_list(pv_IncludePatterns)
to_exclude = string_util.delimited_string_to_list(pv_ExcludePatterns)
# Convert the pv_ListFiles and pv_ListFolders to Boolean values.
list_files_bool = string_util.str_to_bool(pv_ListFiles)
list_dirs_bool = string_util.str_to_bool(pv_ListFolders)
# Set the absolute paths for the Folder and the URL to None until proven to exist within the command process.
folder_abs = None
url_abs = None
# If the input is a local folder.
if pv_Folder:
# Convert the Folder parameter value relative path to an absolute path and expand for ${Property} syntax.
folder_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_Folder, self)))
# If the input is a url.
if pv_URL:
# Convert the URL parameter to expand for ${Property} syntax.
url_abs = self.command_processor.expand_parameter_value(
pv_URL, self)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_list_files(folder_abs, url_abs, list_files_bool,
list_dirs_bool, pv_ListProperty):
try:
# If the input is a local folder.
if pv_Folder:
# Get a list of the files in the folder.
files = [
f for f in os.listdir(folder_abs)
if os.path.isfile(os.path.join(folder_abs, f))
]
# Get a list of directories in the folder.
dirs = [
d for d in os.listdir(folder_abs)
if os.path.isdir(os.path.join(folder_abs, d))
]
# If configured to list files and folders, continue.
if list_files_bool and list_dirs_bool:
# Filter the list of files and folders with regards to the IncludePatterns and ExcludePatterns.
output_filtered = string_util.filter_list_of_strings(
files + dirs, to_include, to_exclude)
# If configured to list files, continue.
elif list_files_bool:
# Filter the list of files with regards to the IIncludePatterns and ExcludePatterns.
output_filtered = string_util.filter_list_of_strings(
files, to_include, to_exclude)
# If configured to list folders, continue.
elif list_dirs_bool:
# Filter the list of folders with regards to the IIncludePatterns and ExcludePatterns.
output_filtered = string_util.filter_list_of_strings(
dirs, to_include, to_exclude)
else:
output_filtered = []
# Add the filtered list to the desired ListProperty. Sort the list alphabetically
self.command_processor.set_property(
pv_ListProperty, sorted(output_filtered,
key=str.lower))
# If the input is a url.
if pv_URL:
# If configured to list files and folders, continue.
if list_files_bool and list_dirs_bool:
output_filtered = self.scrape_html(
url_abs, True, True, to_include, to_exclude)
# If configured to list files, continue.
elif list_files_bool:
output_filtered = self.scrape_html(
url_abs, True, False, to_include, to_exclude)
# If configured to list folders, continue.
elif list_dirs_bool:
output_filtered = self.scrape_html(
url_abs, False, True, to_include, to_exclude)
else:
output_filtered = None
# Add the filtered list to the desired ListProperty.
self.command_processor.set_property(
pv_ListProperty, sorted(output_filtered,
key=str.lower))
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error running ListFiles command."
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Remove the GeoLayer object from the GeoProcessor. Delete the GeoLayer instance.
Returns:
None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_GeoLayerIDs = self.get_parameter_value("GeoLayerIDs")
# Convert the GeoLayerIDs parameter from string to list format.
# If configured, list all of the registered GeoLayer IDs.
if pv_GeoLayerIDs == "*":
list_of_geolayer_ids = []
# Iterate over each GeoLayer registered within the GeoProcessor. Add each GeoLayer's ID to the list.
for geolayer_obj in self.command_processor.geolayers:
list_of_geolayer_ids.append(geolayer_obj.id)
# If specific GeoLayer IDs are listed, convert the string into list format.
else:
list_of_geolayer_ids = string_util.delimited_string_to_list(pv_GeoLayerIDs)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_geolayer_be_deleted(list_of_geolayer_ids):
try:
# Iterate over the GeoLayer IDS.
for geolayer_id in list_of_geolayer_ids:
# Get GeoLayer to remove.
geolayer = self.command_processor.get_geolayer(geolayer_id)
# Remove the GeoLayer from the GeoProcessor's geolayers list.
index = self.command_processor.geolayers.index(geolayer)
del self.command_processor.geolayers[index]
# Delete the Qgs Vector Layer object.
del geolayer.qgs_vector_layer
# Delete the GeoLayer.
del geolayer
# Raise an exception if an unexpected error occurs during the process.
except Exception as e:
self.warning_count += 1
message = "Unexpected error removing GeoLayer ({}).".format(pv_GeoLayerIDs)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Set a property on the GeoProcessor.
Returns:
Nothing.
Raises:
RuntimeError if there is any exception running the command.
"""
warning_count = 0
logger = logging.getLogger(__name__)
pv_PropertyName = self.get_parameter_value('PropertyName')
pv_PropertyType = self.get_parameter_value('PropertyType')
pv_PropertyValue = self.get_parameter_value('PropertyValue')
pv_PropertyValues = self.get_parameter_value('PropertyValues')
# Expand the property value string before converting to the requested type
pv_PropertyValue_expanded = self.command_processor.expand_parameter_value(
pv_PropertyValue)
pv_PropertyValues_expanded = self.command_processor.expand_parameter_value(
pv_PropertyValues)
do_list = False # Single property is the default
if pv_PropertyValues is not None and pv_PropertyValues != "":
# Doing a list
do_list = True
try:
parameter_value_as_list = []
parameter_value_to_parse = ""
if do_list:
parameter_value_to_parse = pv_PropertyValues_expanded
else:
# Single property - add to a list as if a single-value list
parameter_value_to_parse = pv_PropertyValue_expanded
# logger.info('Parsing parameter "' + str(parameter_value_to_parse) + "'")
# Parse the list into a string list
# - Remove leading [ and trailing ] so only have simple string list
parameter_value_to_parse = parameter_value_to_parse.strip(
) # First strip whitespace
if parameter_value_to_parse.startswith("["):
parameter_value_to_parse = parameter_value_to_parse[1:]
if parameter_value_to_parse.endswith("]"):
parameter_value_to_parse = parameter_value_to_parse[
0:len(parameter_value_to_parse) - 1]
parameter_value_as_string_list = string_util.delimited_string_to_list(
parameter_value_to_parse, trim=True)
# logger.info('Parsed parameter "' + str(parameter_value_as_string_list) + "'")
# Loop through the list
parameter_value2 = None
for parameter_value in parameter_value_as_string_list:
# Convert the property value string to the requested type
parameter_value2 = None
if pv_PropertyType == 'bool':
# Use the following because conversion of strings to booleans is tricky, too many unexpected True
parameter_value2 = string_util.str_to_bool(parameter_value)
elif pv_PropertyType == 'float':
parameter_value2 = float(parameter_value)
elif pv_PropertyType == 'int':
parameter_value2 = int(parameter_value)
elif pv_PropertyType == 'str':
parameter_value2 = str(parameter_value)
# Now set the object as a property, will be the requested type
if do_list:
# The property is a list
# - for now avoid adding None - later For command should process only valid values
if parameter_value2 is not None:
parameter_value_as_list.append(parameter_value2)
if do_list:
# Doing a list so set the property value to the list
# - list could be empty - is this an issue?
self.command_processor.set_property(pv_PropertyName,
parameter_value_as_list)
# logger.info('Setting parameter "' + str(pv_PropertyName) + '"="' + str(parameter_value_as_list) + '"')
else:
# The property value is a single object and will have been processed in the loop's only iteration
if parameter_value2 is not None:
self.command_processor.set_property(
pv_PropertyName, parameter_value2)
# logger.info('Setting parameter "' + str(pv_PropertyName) + '"="' + str(parameter_value2) + '"')
except Exception as e:
warning_count += 1
message = 'Unexpected error setting property "' + str(
pv_PropertyName) + '"'
logger.error(message, e, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"Check the log file for details."))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Write the Table to an excel file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values except for the OutputCRS
pv_TableID = self.get_parameter_value("TableID")
pv_OutputFile = self.get_parameter_value("OutputFile")
pv_OutputWorksheet = self.get_parameter_value("OutputWorksheet")
pv_ColumnsToInclude = self.get_parameter_value("ColumnsToInclude",
default_value="*")
pv_ColumnsToExclude = self.get_parameter_value("ColumnsToExclude",
default_value="")
pv_WriteIndexColumn = self.get_parameter_value("WriteIndexColumn",
default_value="True")
# Convert the Boolean parameters from string to valid Boolean values.
pv_WriteIndexColumn = string_util.str_to_bool(pv_WriteIndexColumn)
# Convert the ColumnsToInclude and ColumnsToExclude parameter values to lists.
cols_to_include = string_util.delimited_string_to_list(
pv_ColumnsToInclude)
cols_to_exclude = string_util.delimited_string_to_list(
pv_ColumnsToExclude)
# Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_write_table(pv_TableID, output_file_absolute):
try:
# Get the Table object
table = self.command_processor.get_table(pv_TableID)
# Get a list of all the available column names in the Table.
all_cols_names = list(table.df)
# Sort the list to create a second list that only includes the attributes that should be removed.
cols_to_keep = string_util.filter_list_of_strings(
list(table.df),
cols_to_include,
cols_to_exclude,
return_inclusions=True)
# For the columns configured to be written, order them in the same order that they were in with the
# original table. This step ensures that the output table product has the columns in the same order as
# the user would expect.
sorted_cols_to_keep = []
for col in all_cols_names:
if col in cols_to_keep:
sorted_cols_to_keep.append(col)
# Write the tables to an Excel file.
pandas_util.write_df_to_excel(table.df, output_file_absolute,
pv_OutputWorksheet,
sorted_cols_to_keep,
pv_WriteIndexColumn)
# Add the output file to the GeoProcessor's list of output files.
self.command_processor.add_output_file(output_file_absolute)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error writing Table {} to Excel workbook file {}.".format(
pv_TableID, pv_OutputFile)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
