def run_command(self):
"""
Run the command. Open a file to receive regression test results.
Returns:
None.
Raises:
RuntimeError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_OutputFile = self.get_parameter_value('OutputFile')
# Runtime checks on input
pv_OutputFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_OutputFile, self)))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Open the regression test results file
try:
pv_OutputFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_OutputFile, self)))
self.__open_new_regression_test_report_file(pv_OutputFile_absolute, False) # Do not append
logger.info('Opened regression test results report file "' + pv_OutputFile_absolute + '"')
except Exception as e:
warning_count += 1
message = 'Unexpected error opening file "' + pv_OutputFile_absolute + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings processing the command."
logger.warning(message)
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_batch(command_file, runtime_properties):
"""
Run in batch mode by processing the specific command file.
Args:
command_file (str): The name of the command file to run, absolute path or relative to the current folder.
runtime_properties (dict): A dictionary of properties for the processor.
Returns:
None.
"""
logger = logging.getLogger(__name__)
print('Running command file: ' + command_file)
working_dir = os.getcwd()
# Convert the command file to an absolute path if not already.
logger.info("Working directory=" + working_dir)
logger.info("Command file=" + command_file)
command_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(working_dir, command_file))
logger.info("Command file (absolute)=" + command_file_absolute)
# from geoprocessor.core.CommandFileRunner import CommandFileRunner
CommandFileRunner = importlib.import_module(
'geoprocessor.core.CommandFileRunner')
class_ = getattr(CommandFileRunner, 'CommandFileRunner')
runner = class_()
# Read the command file
try:
runner.read_command_file(command_file_absolute)
except IOError as e:
# Would be nice to use FileNotFoundError but that is Python 3.3+
message = 'Error: Command file "' + command_file_absolute + '" was not found.'
print(message)
logger.error(message, exc_info=True)
return
except Exception as e2:
message = 'Error reading command file "' + command_file_absolute + '".'
logger.error(message, exc_info=True)
print(message)
return
# Run the command file
try:
# Pass the runtime properties to supplement default properties and those created in the command file
runner.run_commands(env_properties=runtime_properties)
logger.info("At end of gp.run_batch")
except Exception as e:
message = 'Error running command file.'
print(message)
logger.error(message, exc_info=True)
return
finally:
StartRegressionTestResultsReport.close_regression_test_report_file()
logger.info("GeoProcessor properties after running:")
for property_name, property_value in runner.get_processor(
).properties.items():
logger.info(property_name + " = " + str(property_value))
print("See log file for more information.")
def run_command(self):
"""
Run the command. Write the GeoLayer to a spatial data file in KML format.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_OutputFile = self.get_parameter_value("OutputFile")
pv_PlacemarkNameAttribute = self.get_parameter_value("PlacemarkNameAttribute")
pv_PlacemarkDescriptionAttribute = self.get_parameter_value("PlacemarkDescriptionAttribute")
# Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_OutputFile, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_write_geolayer(pv_GeoLayerID, output_file_absolute):
try:
# Get the GeoLayer
geolayer = self.command_processor.get_geolayer(pv_GeoLayerID)
# Write the GeoLayer to a spatial data file in KML format
# "Note that KML by specification uses only a single projection, EPSG:4326. All OGR KML output will be
# presented in EPSG:4326. As such OGR will create layers in the correct coordinate system and transform
# any geometries." - www.gdal.org/drv_kml.html
qgis_util.write_qgsvectorlayer_to_kml(geolayer.qgs_vector_layer,
output_file_absolute,
"EPSG:4326",
pv_PlacemarkNameAttribute,
pv_PlacemarkDescriptionAttribute,
"clampToGround")
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error writing GeoLayer {} to GeoJSON format.".format(pv_GeoLayerID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def do_run(cls, line):
"""
Run the command file from the command line using syntax: run command-file
"""
if line is None: # Put this here to get rid of PyCharm warning about line not being used
pass
logger = logging.getLogger(__name__)
command_file = line
if command_file is None:
print('Error: no command file specified for "run" command.')
print('Running command file: ' + command_file)
working_dir = os.getcwd()
# Convert the command file to an absolute path if not already.
command_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(working_dir, command_file))
# Make sure the file exists so ugly exception is not printed
if not os.path.isfile(command_file_absolute):
print("File does not exist, cannot run: " + command_file_absolute)
return
# from geoprocessor.core.CommandFileRunner import CommandFileRunner
CommandFileRunner = importlib.import_module(
'geoprocessor.core.CommandFileRunner')
class_ = getattr(CommandFileRunner, 'CommandFileRunner')
runner = class_()
# Read the command file
try:
runner.read_command_file(command_file_absolute)
except IOError as e:
# Would be nice to use FileNotFoundError but that is Python 3.3+
message = 'Error: Command file "' + command_file_absolute + '" was not found.'
print(message)
logger.error(message, exc_info=True)
except Exception as e2:
message = 'Error reading command file.'
logger.error(message, exc_info=True)
print(message)
# Run the command file
try:
runner.run_commands()
except Exception as e:
message = 'Error running command file.'
print(message)
logger.error(message, exc_info=True)
logger.info("GeoProcessor properties after running:")
for property_name, property_value in runner.get_processor(
).properties.items():
logger.info(property_name + " = " + str(property_value))
print("See log file for more information.")
def run_command(self):
"""
Run the command. Restart the lof file with the given name.
Subsequent logging messages will go to the file until another StartLog() command is run.
Returns:
Nothing.
Raises:
RuntimeError if any exception occurs running the command.
"""
logger = logging.getLogger(__name__)
warning_count = 0
pv_LogFile = self.get_parameter_value('LogFile')
log_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_LogFile)))
try:
# Change the GeoProcessor logger to use the specified file
# - The initial application log file will be closed.
log_util.reset_log_file_handler(log_file_absolute)
# Add the log file to output
self.command_processor.add_output_file(log_file_absolute)
except Exception as e:
warning_count += 1
message = 'Unexpected error (re)starting log file "' + log_file_absolute + '"'
logger.exception(message, e)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"Check the old log file or command window for details."))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command.
Returns:
Nothing.
Raises:
ValueError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_SearchFolder = self.get_parameter_value('SearchFolder')
pv_FilenamePattern = self.get_parameter_value('FilenamePattern')
if pv_FilenamePattern is None or pv_FilenamePattern == "":
# The pattern that is desired is test_*.gp, using globbing syntax
pv_FilenamePattern = "[Tt][Ee][Ss][Tt]-*.gp"
pv_OutputFile = self.get_parameter_value('OutputFile')
# Runtime checks on input
working_dir = self.command_processor.get_property('WorkingDir')
logger.info('working_dir: "' + working_dir + '"')
search_folder_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(working_dir, self.command_processor.expand_parameter_value(pv_SearchFolder, self)))
search_folder_absolute_internal = io_util.verify_path_for_os(search_folder_absolute,
always_use_forward_slashes=True)
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(working_dir, self.command_processor.expand_parameter_value(pv_OutputFile, self)))
output_file_absolute_internal = io_util.verify_path_for_os(output_file_absolute,
always_use_forward_slashes=True)
logger.info('search_folder_absolute: "' + search_folder_absolute + '"')
logger.info('search_folder_absolute_internal: "' + search_folder_absolute_internal + '"')
if not os.path.exists(search_folder_absolute_internal):
message = 'The folder to search "' + search_folder_absolute + '" does not exist.'
logger.warning(message)
warning_count += 1
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE,
message, "Verify that the folder exists at the time the command is run."))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Do the processing
try:
# Output folder is used when constructing filenames for command files to run
output_folder_absolute = os.path.dirname(output_file_absolute_internal)
logger.info('output_folder_absolute: "' + output_folder_absolute + '"')
logger.info('output_folder_absolute_internal: "' + output_file_absolute_internal + '"')
files = [] # List of files to match
# Convert globbing-style wildcards to Pythonic regex
logger.info('Filename pattern using globbing = "' + pv_FilenamePattern + '"')
# filename_pattern_regex = pv_FilenamePattern.replace("*", "[^/]*")
# Match any character of any length, making sure that special characters are dealt with
filename_pattern_regex = pv_FilenamePattern.replace(".", "\\.") # So .gp is handled literally
# The following is used to match between "test-" and ".gp"
filename_pattern_regex = filename_pattern_regex.replace("*", ".*")
logger.info('Filename pattern using Python regex = "' + filename_pattern_regex + '"')
filename_pattern_regex_compiled = re.compile(filename_pattern_regex)
CreateRegressionTestCommandFile.__get_matching_filenames_in_tree(files, search_folder_absolute_internal,
filename_pattern_regex_compiled,
pattern_string=filename_pattern_regex)
# Sort the list
files = sorted(files)
# Open the output file...
# TODO smalers 2018-10-20 decide whether to support append mode
out = open(output_file_absolute_internal, "w")
# Write a standard header to the file so that it is clear when the file was created
io_util.print_standard_file_header(out, comment_line_prefix="#")
# Include the setup command file if requested
# logger.info('Adding commands from setup command file "' + setup_command_file_absolute + '"')
# include_setup_file(out, setup_file_absolute, "setup")
# Include the matching test cases
# Python 2...
# nl = os.linesep # newline character for operating system
# Python 3 translates \n into the OS-specific end of line so os.linesep introduces extra end of lines
nl = "\n"
out.write("#" + nl)
out.write("# The following " + str(len(files)) +
" test cases will be run to compare results with expected results." + nl)
out.write("# Individual log files are generally created for each test." + nl)
# TODO smalers 2018-01-20 evaluate how to handle output table
# - Currently the GeoProcessor does not have table commands
table_param = ""
# Use absolute path since each developer will regenerate this file.
out.write('StartRegressionTestResultsReport(OutputFile="' +
output_file_absolute + '.out.txt"' + table_param + ")" + nl)
# Find the list of matching files...
# logger.debug('output_folder_absolute"' + output_folder_absolute + '"')
for a_file in files:
logger.info('Adding command file "' + a_file + '"')
# The command files to run are relative to the commands file being created.
# - use the operating system separator
command_file_to_run = io_util.verify_path_for_os(
io_util.to_relative_path(output_folder_absolute, a_file))
# Determine if the command file has @expectedStatus in it. If so, define an ExpectedStatus
# parameter for the command.
logger.info('Relative path to command file is "' + command_file_to_run + '"')
out.write('RunCommands(CommandFile="' + command_file_to_run + '"' +
CreateRegressionTestCommandFile.__determine_expected_status_parameter(a_file) + ')' + nl)
# TODO smalers 2018-12-30 Maybe the output file is relative to the working folder
# output_file_relative = io_util.to_relative_path(working_dir, output_file_absolute)
# out.write('WriteCommandSummaryToFile(OutputFile="' + output_file_relative + '.summary.html")' + nl)
out.write('WriteCommandSummaryToFile(OutputFile="' + output_file_absolute + '.summary.html")' + nl)
# TODO smalers 2018-01-28 evaluate whether to support end content
# Include the end command file if requested
# Message.printStatus ( 2, routine, "Adding commands from end command file \"" + EndCommandFile_full + "\"")
# includeCommandFile ( out, EndCommandFile_full, "end" );
out.close()
# Add the log file to output
self.command_processor.add_output_file(output_file_absolute)
except Exception as e:
warning_count += 1
message = 'Unexpected error creating regression test command file "' + output_file_absolute + '"'
traceback.print_exc(file=sys.stdout)
logger.exception(message, e)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Create a file summarizing command log messages, status, etc.
Returns:
None.
Raises:
RuntimeError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_OutputFile = self.get_parameter_value('OutputFile')
# Runtime checks on input
pv_OutputFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Create the command summary file
try:
pv_OutputFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
# Open the file and write command summary
logger.info('Writing summary to "' + pv_OutputFile_absolute + '"')
fp = open(pv_OutputFile_absolute, "w")
self.write_file_header(fp)
self.write_command_summary(fp)
self.write_file_footer(fp)
# Close the file
fp.close()
except Exception as e:
warning_count += 1
message = 'Unexpected error writing file "' + pv_OutputFile_absolute + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
except:
warning_count += 1
message = 'Unexpected error writing file "' + pv_OutputFile_absolute + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
logger.warning(message)
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Run a separate command file and save results to the current processor.
Returns:
None.
Raises:
ValueError: if a runtime input error occurs.
RuntimeError: if a runtime error occurs.
"""
# The following import is deferred until runtime because if included at the top of the module
# it causes a circular dependency and the GeoProcessor won't load
from geoprocessor.core.CommandFileRunner import CommandFileRunner
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_CommandFile = self.get_parameter_value('CommandFile')
pv_ExpectedStatus = self.get_parameter_value('ExpectedStatus')
expected_status = pv_ExpectedStatus
if pv_ExpectedStatus == "":
pv_ExpectedStatus = None # Default - was not specified in the command
# Runtime checks on input
pv_CommandFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_CommandFile, self)))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Write the output file
try:
command_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_CommandFile, self)))
logger.info('Processing commands from file "' + command_file_absolute + '" using command file runner.')
runner = CommandFileRunner()
# This will set the initial working directory of the runner to that of the command file...
file_found = True
try:
runner.read_command_file(command_file_absolute)
except FileNotFoundError as e:
warning_count += 1
message = 'File does not exist: "' + command_file_absolute + '"'
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"Confirm that the command file exists."))
# Set the following to skip code below
file_found = False
# If the command file is not enabled, don't need to initialize or process
# TODO SAM 2013-04-20 Even if disabled, will still run discovery above
# - need to disable discovery in this case
is_enabled = runner.is_command_file_enabled()
expected_status = str(CommandStatusType.SUCCESS)
if pv_ExpectedStatus is not None:
expected_status = pv_ExpectedStatus
if not file_found:
# Need to add logic to indicate a failed test
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(
CommandStatusType.FAILURE, "Command file does not exist.",
"Confirm that the command file exists."))
# Set the results to fail
test_pass_fail = self.__FAIL
elif is_enabled:
# TODO smalers, 2018-01-26 Java code set datastores here
# TODO SAM 2010-09-30 Need to evaluate how to share properties - issue is that built-in properties are
# handled explicitly whereas user-defined properties are in a list that can be easily shared.
# Also, some properties like the working directory receive special treatment.
# For now don't bite off the property issue
runner.run_commands(env_properties=self.command_processor.env_properties)
logger.info("Done running commands")
# Total runtime for the commands
# long run_time_total = TSCommandProcessorUtil.getRunTimeTotal(runner.getProcessor().getCommands());
# Set the CommandStatus for this command to the most severe status of the
# commands file that was just run.
max_severity = command_util.get_command_status_max_severity(runner.command_processor)
logger.info("Max severity from commands = " + str(max_severity))
test_pass_fail = "????" # Status for the test, which is not always the same as max_severity
if pv_ExpectedStatus is not None:
expected_status_type = CommandStatusType.value_of(expected_status)
if max_severity is expected_status_type:
# Expected status matches the actual so consider this a success.
# This should generally be used only when running a test that we expect to fail (e.g., run
# obsolete command or testing handling of errors).
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.SUCCESS,
"Severity for RunCommands (" + str(max_severity) +
") is max of commands in command file that was run - matches expected (" +
expected_status + ") so RunCommands status=Success.",
"Additional status messages are omitted to allow test to be success - " +
"refer to log file if warning/failure."))
# TODO SAM 2008-07-09 Need to evaluate how to append all the log messages but still
# have a successful status that shows in the displays.
# DO NOT append the messages from the command because their status will cause the
# error displays to show problem indicators.
test_pass_fail = self.__PASS
else:
# Expected status and it does NOT match the actual status so this is a failure.
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.SUCCESS,
"Severity for RunCommands (" + str(max_severity) +
") is max of commands in command file that was run - " +
"does not match expected (" +
expected_status + ") so RunCommands status=Failure.",
"Check the command to confirm the expected status."))
# TODO SAM 2008-07-09 Need to evaluate how to append all the log messages but still
# have a successful status that shows in the displays.
# DO NOT append the messages from the command because their status will cause the
# error displays to show problem indicators.
test_pass_fail = self.__FAIL
else:
# TODO smalers 2018-01-28 evaluate whether this is needed given that success is default expected
# status
# Expected status is not specified
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(
max_severity, "Severity for RunCommands (" + str(max_severity) +
") is max of commands in command file that was run.",
"Status messages from commands that were run are appended to RunCommand status messages."))
# Append the log records from the command file that was run.
# The status contains lists of CommandLogRecord for each run mode.
# For RunCommands() the log messages should be associated with the originating command,
# not this RunCommand command
logger.info("Appending log records")
command_util.append_command_status_log_records(
self.command_status, runner.command_processor.commands)
if max_severity.value >= CommandStatusType.WARNING.value:
test_pass_fail = self.__FAIL
else:
test_pass_fail = self.__PASS
# Add a record to the regression test report...
logger.info("Adding record to regression test report")
run_time_total = 0
StartRegressionTestResultsReport.append_to_regression_test_report(
is_enabled, run_time_total,
test_pass_fail, expected_status, max_severity, command_file_absolute)
# If it was requested to append the results to the calling processor, get
# the results from the runner and do so...
# if ( (AppendResults != null) && AppendResults.equalsIgnoreCase("true")) {
# TSCommandProcessor processor2 = runner.getProcessor();
# Object o_tslist = processor2.getPropContents("TSResultsList");
# PropList request_params = new PropList ( "" );
# if ( o_tslist != null ) {
# @SuppressWarnings("unchecked")
# List<TS> tslist = (List<TS>)o_tslist;
# int size = tslist.size()
# TS ts;
# for ( int i = 0; i < size; i++ ) {
# ts = tslist.get(i);
# request_params.setUsingObject( "TS", ts );
# processor.processRequest( "AppendTimeSeries", request_params );
# }
# }
# }
logger.info("...done processing commands from file.")
else:
# Add a record to the regression report (the is_enabled value is what is important for the report
# because the test is not actually run)...
# TODO smalers 2018-01-26 finish...
logger.info("Command file is not enabled")
run_time_total = 0
test_pass_fail = ""
max_severity = CommandStatusType.UNKNOWN
StartRegressionTestResultsReport.append_to_regression_test_report(
is_enabled, run_time_total,
test_pass_fail, expected_status, max_severity, command_file_absolute)
pass
except Exception as e:
warning_count += 1
message = 'Unexpected error running command file "' + pv_CommandFile_absolute + '"'
traceback.print_exc(file=sys.stdout)
logger.error(message, e, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
except:
warning_count += 1
message = 'Unexpected error running command file "' + pv_CommandFile_absolute + '"'
traceback.print_exc(file=sys.stdout)
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(warning_count) + " warnings processing the command."
logger.warning(message)
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Download the file from the web and save it on the local computer.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values
pv_URL = self.get_parameter_value("URL")
pv_OutputFile = self.get_parameter_value("OutputFile",
default_value=None)
pv_Username = self.get_parameter_value("Username", default_value=None)
pv_Password = self.get_parameter_value("Password", default_value=None)
# Convert the pv_URL parameter to expand for ${Property} syntax.
url_abs = self.command_processor.expand_parameter_value(pv_URL, self)
# Convert the OutputFile parameter value relative path to an absolute path. Expand for ${Property} syntax.
# If the OutputFile parameter is specified, continue.
if pv_OutputFile:
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
# If the OutputFile parameter is NOT specified, continue.
else:
original_filename = io_util.get_filename(
pv_URL) + io_util.get_extension(pv_URL)
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
original_filename, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_run_webget(output_file_absolute):
try:
# Get the output folder.
output_folder = os.path.dirname(output_file_absolute)
# Get the URL file and convert it into a request Response object
# Authentication Reference: http://docs.python-requests.org/en/master/user/authentication/
r = requests.get(url_abs,
auth=HTTPBasicAuth(pv_Username, pv_Password),
verify=False,
stream=True)
# Get the filename of the URL and the output file
url_filename = io_util.get_filename(url_abs)
output_filename = io_util.get_filename(output_file_absolute)
# Remove the output file if it already exists.
if os.path.exists(output_file_absolute):
os.remove(output_file_absolute)
# If the URL file is a zip file, process as a zip file.
if zip_util.is_zip_file_request(r):
# Create an empty list to hold the files that were downloaded/extracted to the output folder.
downloaded_files = []
with open(
os.path.join(output_folder,
"{}.zip".format(url_filename)),
"wb") as downloaded_zip_file:
downloaded_zip_file.write(r.content)
downloaded_files.append("{}.zip".format(url_filename))
# Determine if the downloaded zip file(s) should be renamed. If the filename is %f then the
# filenames of the downloaded products should be the same as the url filenames
if not output_filename == '%f':
self.__rename_files_in_a_folder(
list_of_files=downloaded_files,
folder_path=output_folder,
new_filename=output_filename)
else:
# Download the file to the output folder.
with open(
os.path.join(output_folder,
os.path.basename(url_abs)),
"wb") as downloaded_file:
downloaded_file.write(r.content)
# Determine if the downloaded file should be renamed. If the filename is %f then the filename
# of the downloaded product should be the same as the url filename
if not output_filename == '%f':
self.__rename_files_in_a_folder(
list_of_files=[os.path.basename(url_abs)],
folder_path=output_folder,
new_filename=output_filename)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error downloading file from URL {}.".format(
url_abs)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
# Obtain the parameter values.
pv_Folder = self.get_parameter_value("Folder")
pv_URL = self.get_parameter_value("URL")
pv_ListFiles = self.get_parameter_value("ListFiles",
default_value="True")
pv_ListFolders = self.get_parameter_value("ListFolders",
default_value="True")
pv_ListProperty = self.get_parameter_value("ListProperty")
pv_IncludePatterns = self.get_parameter_value("IncludePatterns",
default_value="*")
pv_ExcludePatterns = self.get_parameter_value("ExcludePatterns",
default_value="''")
# Convert the IncludeAttributes and ExcludeAttributes to lists.
to_include = string_util.delimited_string_to_list(pv_IncludePatterns)
to_exclude = string_util.delimited_string_to_list(pv_ExcludePatterns)
# Convert the pv_ListFiles and pv_ListFolders to Boolean values.
list_files_bool = string_util.str_to_bool(pv_ListFiles)
list_dirs_bool = string_util.str_to_bool(pv_ListFolders)
# Set the absolute paths for the Folder and the URL to None until proven to exist within the command process.
folder_abs = None
url_abs = None
# If the input is a local folder.
if pv_Folder:
# Convert the Folder parameter value relative path to an absolute path and expand for ${Property} syntax.
folder_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_Folder, self)))
# If the input is a url.
if pv_URL:
# Convert the URL parameter to expand for ${Property} syntax.
url_abs = self.command_processor.expand_parameter_value(
pv_URL, self)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_list_files(folder_abs, url_abs, list_files_bool,
list_dirs_bool, pv_ListProperty):
try:
# If the input is a local folder.
if pv_Folder:
# Get a list of the files in the folder.
files = [
f for f in os.listdir(folder_abs)
if os.path.isfile(os.path.join(folder_abs, f))
]
# Get a list of directories in the folder.
dirs = [
d for d in os.listdir(folder_abs)
if os.path.isdir(os.path.join(folder_abs, d))
]
# If configured to list files and folders, continue.
if list_files_bool and list_dirs_bool:
# Filter the list of files and folders with regards to the IncludePatterns and ExcludePatterns.
output_filtered = string_util.filter_list_of_strings(
files + dirs, to_include, to_exclude)
# If configured to list files, continue.
elif list_files_bool:
# Filter the list of files with regards to the IIncludePatterns and ExcludePatterns.
output_filtered = string_util.filter_list_of_strings(
files, to_include, to_exclude)
# If configured to list folders, continue.
elif list_dirs_bool:
# Filter the list of folders with regards to the IIncludePatterns and ExcludePatterns.
output_filtered = string_util.filter_list_of_strings(
dirs, to_include, to_exclude)
else:
output_filtered = []
# Add the filtered list to the desired ListProperty. Sort the list alphabetically
self.command_processor.set_property(
pv_ListProperty, sorted(output_filtered,
key=str.lower))
# If the input is a url.
if pv_URL:
# If configured to list files and folders, continue.
if list_files_bool and list_dirs_bool:
output_filtered = self.scrape_html(
url_abs, True, True, to_include, to_exclude)
# If configured to list files, continue.
elif list_files_bool:
output_filtered = self.scrape_html(
url_abs, True, False, to_include, to_exclude)
# If configured to list folders, continue.
elif list_dirs_bool:
output_filtered = self.scrape_html(
url_abs, False, True, to_include, to_exclude)
else:
output_filtered = None
# Add the filtered list to the desired ListProperty.
self.command_processor.set_property(
pv_ListProperty, sorted(output_filtered,
key=str.lower))
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error running ListFiles command."
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Read all spatial data files within the folder. For each desired spatial data file (can be
specified by the Subset_Pattern parameter), create a GeoLayer object, and add to the GeoProcessor's geolayer
list.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_SpatialDataFolder = self.get_parameter_value("SpatialDataFolder")
pv_Subset_Pattern = self.get_parameter_value("Subset_Pattern")
pv_GeoLayerID_prefix = self.get_parameter_value("GeoLayerID_prefix")
# Convert the SpatialDataFolder parameter value relative path to an absolute path
sd_folder_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_SpatialDataFolder, self)))
# Run the initial checks on the parameter values. Only continue if the checks passed.
if self.__should_read_folder(sd_folder_abs):
# Determine which files within the folder should be processed. All files will be processed if
# pv_Subset_Pattern is set to None. Otherwise only files that match the given pattern will be processed.
# Check that each file in the folder is:
# 1. a file
# 2. a spatial data file (ends in .shp or .geojson)
# 3. follows the given pattern (if Subset_Pattern parameter value does not equal None)
if pv_Subset_Pattern:
spatial_data_files_abs = [os.path.join(sd_folder_abs, source_file)
for source_file in glob.glob(os.path.join(sd_folder_abs, pv_Subset_Pattern))
if os.path.isfile(os.path.join(sd_folder_abs, source_file))
and (source_file.endswith(".shp") or source_file.endswith(".geojson"))]
else:
spatial_data_files_abs = [os.path.join(sd_folder_abs, source_file)
for source_file in os.listdir(sd_folder_abs)
if os.path.isfile(os.path.join(sd_folder_abs, source_file))
and (source_file.endswith(".shp") or source_file.endswith(".geojson"))]
# Iterate through the desired spatial data files
for spatial_data_file_absolute in spatial_data_files_abs:
# Determine the GeoLayerID.
if pv_GeoLayerID_prefix:
geolayer_id = "{}_{}".format(pv_GeoLayerID_prefix,
io_util.expand_formatter(spatial_data_file_absolute, '%f'))
else:
geolayer_id = io_util.expand_formatter(spatial_data_file_absolute, '%f')
# Run the secondary checks on the parameter values. Only continue if the checks passed.
if self.__should_read_geolayer(geolayer_id):
try:
# Create a QGSVectorLayer object with the GeoJSON SpatialDataFile
qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_file(spatial_data_file_absolute)
# Create a GeoLayer and add it to the geoprocessor's GeoLayers list
geolayer_obj = GeoLayer(geolayer_id, qgs_vector_layer, spatial_data_file_absolute)
self.command_processor.add_geolayer(geolayer_obj)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading GeoLayer {} from" \
" file {}.".format(geolayer_id, spatial_data_file_absolute)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command.
Remove the specified file from the operating system.
Returns:
Nothing.
Raises:
RuntimeError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_SourceFile = self.get_parameter_value('SourceFile')
pv_IfSourceFileNotFound = self.get_parameter_value(
'IfSourceFileNotFound')
if pv_IfSourceFileNotFound is None or pv_IfSourceFileNotFound == "":
pv_IfSourceFileNotFound = 'Warn' # Default
pv_RemoveIfFolder = self.get_parameter_value('RemoveIfFolder',
default_value="False")
pv_RemoveIfFolder = string_util.str_to_bool(pv_RemoveIfFolder)
# Runtime checks on input
pv_SourceFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_SourceFile, self)))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Remove the file
try:
input_count = 1
if not os.path.exists(pv_SourceFile_absolute):
message = 'The source file does not exist: "' + pv_SourceFile_absolute + '"'
if pv_IfSourceFileNotFound == 'Warn':
warning_count += 1
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(
CommandStatusType.WARNING, message,
"Verify that the source exists at the time the command is run."
))
logger.warning(message)
elif pv_IfSourceFileNotFound == 'Fail':
warning_count += 1
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(
CommandStatusType.FAILURE, message,
"Verify that the source exists at the time the command is run."
))
logger.warning(message)
input_count -= 1
if input_count == 1:
try:
logger.info('Removing file "' + pv_SourceFile_absolute +
'"')
os.remove(pv_SourceFile_absolute)
except:
if pv_RemoveIfFolder:
shutil.rmtree(pv_SourceFile_absolute)
except Exception as e:
warning_count += 1
message = 'Unexpected error removing file "' + pv_SourceFile_absolute + '"'
traceback.print_exc(file=sys.stdout)
logger.exception(message, e)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Compare the input files.
Returns:
Nothing.
Raises:
RuntimeError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get runtime data for the command
pv_InputFile1 = self.get_parameter_value('InputFile1')
pv_InputFile2 = self.get_parameter_value('InputFile2')
pv_CommentLineChar = self.get_parameter_value('CommentLineChar')
if pv_CommentLineChar is None or pv_CommentLineChar == "":
pv_CommentLineChar = "#" # Default value
pv_MatchCase = self.get_parameter_value('MatchCase')
if pv_MatchCase is None or pv_MatchCase == "":
pv_MatchCase = True # Default value
pv_IgnoreWhitespace = self.get_parameter_value('IgnoreWhitespace')
if pv_IgnoreWhitespace is None or pv_IgnoreWhitespace == "":
pv_IgnoreWhitespace = True # Default value
pv_AllowedDiffCount = self.get_parameter_value('AllowedDiffCount')
if pv_AllowedDiffCount is None or pv_AllowedDiffCount == "":
allowed_diff_count = 0 # Default value
else:
allowed_diff_count = int(pv_AllowedDiffCount)
# Convert IfDifferent and IfSame to internal types, Ignore will convert to None, which is OK
pv_IfDifferent = self.get_parameter_value('IfDifferent')
if pv_IfDifferent is None or pv_IfDifferent == "":
pv_IfDifferent = "Ignore" # Default value
pv_IfDifferent_CommandStatusType = CommandStatusType.value_of(
pv_IfDifferent, True)
pv_IfSame = self.get_parameter_value('IfSame')
if pv_IfSame is None or pv_IfSame == "":
pv_IfSame = "Ignore" # Default value
pv_IfSame_CommandStatusType = CommandStatusType.value_of(
pv_IfSame, True)
# Runtime checks on input
pv_InputFile1_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_InputFile1, self)))
pv_InputFile2_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_InputFile2, self)))
if warning_count > 0:
message = "There were " + warning_count + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Do the processing
diff_count = 0
line_count_compared = 0
try:
input_count = 2
if not os.path.exists(pv_InputFile1_absolute):
warning_count += 1
message = 'The first input file does not exist: "' + pv_InputFile1_absolute + '"'
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(
CommandStatusType.FAILURE, message,
"Verify that the first input file exists at the time the command is run."
))
logger.warning(message)
input_count -= 1
if not os.path.exists(pv_InputFile2_absolute):
warning_count += 1
message = 'The second input file does not exist: "' + pv_InputFile2_absolute + '"'
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(
CommandStatusType.FAILURE, message,
"Verify that the second input file exists at the time the command is run."
))
logger.warning(message)
input_count -= 1
if input_count == 2:
# Try to do the comparison, just inline the logic
# Open the files...
in1 = open(pv_InputFile1_absolute, 'r')
in2 = open(pv_InputFile2_absolute, 'r')
# Loop through the files, comparing non-comment lines...
while True:
# The following will discard comments and only return non-comment lines
# Therefore, comparisons are made on chunks of non-comment lines.
# TODO smalers 2018-01-08 Could make this comparison more intelligent if the # of comments varies
iline1 = CompareFiles.__read_line(in1, pv_CommentLineChar,
pv_IgnoreWhitespace)
iline2 = CompareFiles.__read_line(in2, pv_CommentLineChar,
pv_IgnoreWhitespace)
if iline1 is None and iline2 is None:
# Both are done at the same time...
break
# TODO smalers 2006-04-20 The following needs to handle comments at the end...
if iline1 is None and iline2 is not None:
# First file is done (second is not) so files are different...
diff_count += 1
break
if iline2 is None and iline1 is not None:
# Second file is done (first is not) so files are different...
diff_count += 1
break
line_count_compared += 1
if pv_MatchCase:
# Compare the lines as is since case-specific
if iline1 != iline2:
diff_count += 1
else:
# Compare by ignoring case
if iline1.upper() != iline2.upper():
diff_count += 1
# logger.debug('Compared:\n"' + iline1 + '"\n"' + iline2 + '"\ndiff_count=' + str(diff_count))
in1.close()
in2.close()
if line_count_compared == 0:
line_count_compared = 1 # to avoid divide by zero below.
logger.info("There are " + str(diff_count) +
" lines that are different, " +
'{:4f}'.format(100.0 * float(diff_count) /
float(line_count_compared)) +
'% (compared ' + str(line_count_compared) +
' lines).')
except Exception as e:
warning_count += 1
message = 'Unexpected error comparing file "' + pv_InputFile1_absolute + '" to "' + \
pv_InputFile2_absolute + '"'
traceback.print_exc(file=sys.stdout)
logger.exception(message, e)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if diff_count > allowed_diff_count and \
((pv_IfDifferent_CommandStatusType == CommandStatusType.WARNING) or
(pv_IfDifferent_CommandStatusType == CommandStatusType.FAILURE)):
message = "" + str(diff_count) + " lines were different, " + \
'{:4f}'.format(100.0*float(diff_count)/float(line_count_compared)) + \
"% (compared " + str(line_count_compared) + " lines)."
if pv_IfDifferent_CommandStatusType == CommandStatusType.WARNING:
logger.warning(message)
else:
logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(
pv_IfDifferent_CommandStatusType, message,
"Check files because difference is not expected."))
raise RuntimeError(message)
if diff_count == 0 and \
((pv_IfSame_CommandStatusType == CommandStatusType.WARNING) or
(pv_IfSame_CommandStatusType == CommandStatusType.FAILURE)):
message = "No lines were different (the files are the same)."
if pv_IfDifferent_CommandStatusType == CommandStatusType.WARNING:
logger.warning(message)
else:
logger.error(message)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(pv_IfSame_CommandStatusType, message,
"Check files because match is not expected."))
raise RuntimeError(message)
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Open the connection to a database and store as a DataStore.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values. The DatabasePort parameter value will be obtained later in the code.
pv_DatabaseDialect = self.get_parameter_value("DatabaseDialect")
pv_DatabaseServer = self.get_parameter_value("DatabaseServer")
pv_DatabaseName = self.get_parameter_value("DatabaseName")
pv_DatabaseUser = self.get_parameter_value("DatabaseUser")
pv_DatabasePassword = self.get_parameter_value("DatabasePassword")
pv_DatabasePort = self.get_parameter_value("DatabasePort")
pv_DataStoreID = self.get_parameter_value("DataStoreID")
pv_ConfigFile = self.get_parameter_value("ConfigFile")
pv_IfDataStoreIDExists = self.get_parameter_value(
"IfDataStoreIDExists", default_value="Replace")
# Expand for ${Property} syntax.
pv_DatabaseServer = self.command_processor.expand_parameter_value(
pv_DatabaseServer, self)
pv_DatabaseName = self.command_processor.expand_parameter_value(
pv_DatabaseName, self)
pv_DatabaseUser = self.command_processor.expand_parameter_value(
pv_DatabaseUser, self)
pv_DatabasePassword = self.command_processor.expand_parameter_value(
pv_DatabasePassword, self)
# Convert the File parameter value relative path to an absolute path and expand for ${Property} syntax.
if pv_ConfigFile:
pv_ConfigFile = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_ConfigFile, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_open_datastore(pv_DataStoreID, pv_ConfigFile,
pv_IfDataStoreIDExists):
try:
# If the DataStoreID already exists and the IfDataStoreIDExists parameter is set to "Open", continue.
if self.command_processor.get_datastore(
pv_DataStoreID) and pv_IfDataStoreIDExists.upper(
) == "OPEN":
# Get the DataStore obj from the ID and open the connection.
datastore_obj = self.command_processor.get_datastore(
pv_DataStoreID)
datastore_obj.open_db_connection()
# If the "Configuration file configures datastore" method is used.
elif pv_ConfigFile is not None:
print(
"WARNING: The 'Configuration file configures datastore' method is not currently enabled."
)
# If the "Parameters configure datastore" method is used.
else:
# Create a new DataStore object and assign the DataStore ID.
new_datastore = DataStore(pv_DataStoreID)
# Assign the DataBase URI for the appropriate DataBase dialect.
if pv_DatabaseDialect.upper() == "POSTGRESQL":
new_datastore.get_db_uri_postgres(
pv_DatabaseServer, pv_DatabaseName,
pv_DatabaseUser, pv_DatabasePassword,
pv_DatabasePort)
# Open a connection to the database and add the DataStore object to the GeoProcessor.
new_datastore.open_db_connection()
self.command_processor.add_datastore(new_datastore)
# Raise an exception if an unexpected error occurs during the process.
except Exception as e:
self.warning_count += 1
message = "Unexpected error opening DataStore {}.".format(
pv_DataStoreID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Write the GeoLayer to a delimited file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the command parameter values.
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_OutputFile = self.get_parameter_value("OutputFile")
pv_OutputCRS = self.get_parameter_value("OutputCRS")
pv_OutputGeometryFormat = self.get_parameter_value(
"OutputGeometryFormat", default_value="XY").upper()
pv_OutputDelimiter = self.get_parameter_value(
"OutputDelimiter", default_value="COMMA").upper()
# Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
# Get the filename of the OutputFile with the path but without the extension.
path, filename = os.path.split(output_file_absolute)
path = os.path.split(output_file_absolute)[0]
filename_wo_ext_path = os.path.join(path,
os.path.splitext(filename)[0])
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_write_geolayer(pv_GeoLayerID, filename_wo_ext_path,
pv_OutputCRS, pv_OutputGeometryFormat):
try:
# Get the GeoLayer
geolayer = self.command_processor.get_geolayer(pv_GeoLayerID)
# Get the current coordinate reference system (in EPSG code) of the current GeoLayer
if pv_OutputCRS is None:
pv_OutputCRS = geolayer.get_crs()
# Write the GeoLayer to a delimited spatial data file.
qgis_util.write_qgsvectorlayer_to_delimited_file(
geolayer.qgs_vector_layer, filename_wo_ext_path,
pv_OutputCRS, pv_OutputGeometryFormat, pv_OutputDelimiter)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error writing GeoLayer {} to delimited file format.".format(
pv_GeoLayerID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Write the GeoLayer to a spatial data file in GeoJSON format.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values except for the OutputCRS
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_OutputPrecision = int(self.get_parameter_value("OutputPrecision", default_value=5))
pv_OutputFile = self.get_parameter_value("OutputFile")
# Expand for ${Property} syntax.
pv_GeoLayerID = self.command_processor.expand_parameter_value(pv_GeoLayerID, self)
# Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_OutputFile, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_write_geolayer(pv_GeoLayerID, output_file_absolute, pv_OutputPrecision):
try:
# Get the GeoLayer
geolayer = self.command_processor.get_geolayer(pv_GeoLayerID)
# Get the current coordinate reference system (in EPSG code) of the current GeoLayer
geolayer_crs = geolayer.get_crs()
# Obtain the parameter value of the OutputCRS
pv_OutputCRS = self.get_parameter_value("OutputCRS", default_value=geolayer_crs)
# Write the GeoLayer to a spatial data file in GeoJSONformat
qgis_util.write_qgsvectorlayer_to_geojson(geolayer.qgs_vector_layer,
output_file_absolute,
pv_OutputCRS,
pv_OutputPrecision)
# Save the output file in the processor
self.command_processor.add_output_file(output_file_absolute)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error writing GeoLayer {} to GeoJSON format.".format(pv_GeoLayerID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Read the layer file from a GeoJSON file, create a GeoLayer object, and add to the
GeoProcessor's geolayer list.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
logger = logging.getLogger(__name__)
# Obtain the parameter values.
pv_SpatialDataFile = self.get_parameter_value("SpatialDataFile")
pv_GeoLayerID = self.get_parameter_value("GeoLayerID", default_value='%f')
# Expand for ${Property} syntax.
pv_GeoLayerID = self.command_processor.expand_parameter_value(pv_GeoLayerID, self)
# Convert the SpatialDataFile parameter value relative path to an absolute path and expand for ${Property}
# syntax
spatial_data_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_SpatialDataFile, self)))
# If the pv_GeoLayerID is a valid %-formatter, assign the pv_GeoLayerID the corresponding value.
if pv_GeoLayerID in ['%f', '%F', '%E', '%P', '%p']:
pv_GeoLayerID = io_util.expand_formatter(spatial_data_file_absolute, pv_GeoLayerID)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_read_geolayer(spatial_data_file_absolute, pv_GeoLayerID):
try:
# Create a QGSVectorLayer object with the GeoJSON SpatialDataFile.
qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_file(spatial_data_file_absolute)
# Create a GeoLayer and add it to the geoprocessor's GeoLayers list.
geolayer_obj = GeoLayer(geolayer_id=pv_GeoLayerID,
geolayer_qgs_vector_layer=qgs_vector_layer,
geolayer_source_path=spatial_data_file_absolute)
self.command_processor.add_geolayer(geolayer_obj)
# Raise an exception if an unexpected error occurs during the process.
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading GeoLayer {} from GeoJSON file {}.".format(pv_GeoLayerID,
pv_SpatialDataFile)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(self.warning_count)
logger.error(message)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Write the processor properties to a file.
Returns:
Nothing.
Raises:
RuntimeError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_OutputFile = self.get_parameter_value('OutputFile')
pv_IncludeProperties = self.get_parameter_value('IncludeProperties')
include_properties = [] # Default
if pv_IncludeProperties is not None and len(pv_IncludeProperties) > 0:
include_properties = string_util.delimited_string_to_list(
pv_IncludeProperties)
pv_WriteMode = self.get_parameter_value('WriteMode')
write_mode = pv_WriteMode
if pv_WriteMode is None or pv_WriteMode == "":
write_mode = 'Overwrite' # Default
pv_FileFormat = self.get_parameter_value('FileFormat')
file_format = pv_FileFormat
if pv_FileFormat is None or pv_FileFormat == "":
file_format = 'NameTypeValue' # Default
pv_SortOrder = self.get_parameter_value('SortOrder')
sort_order = 0 # no sort
if pv_SortOrder is not None:
if pv_SortOrder == 'Ascending':
sort_order = 1
elif pv_SortOrder == 'Descending':
sort_order = -1
# Runtime checks on input
pv_OutputFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Write the output file
try:
problems = [] # Empty list of properties
io_util.write_property_file(pv_OutputFile_absolute,
self.command_processor.properties,
include_properties, write_mode,
file_format, sort_order, problems)
# Record any problems that were found
for problem in problems:
warning_count += 1
logger.error(problem)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, problem,
"See the log file for details."))
except Exception as e:
warning_count += 1
traceback.print_exc(
file=sys.stdout) # Formatting of error seems to have issue
message = 'Unexpected error writing file "' + pv_OutputFile_absolute + '"'
logger.error(message, e, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
except:
warning_count += 1
traceback.print_exc(
file=sys.stdout) # Formatting of error seems to have issue
message = 'Unexpected error writing file "' + pv_OutputFile_absolute + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
logger.warning(message)
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Read the Table from the delimited file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_InputFile = self.get_parameter_value("InputFile")
pv_Delimiter = self.get_parameter_value("Delimiter", default_value=",")
pv_TableID = self.get_parameter_value("TableID")
pv_HeaderLines = int(
self.get_parameter_value("HeaderLines", default_value="0"))
pv_NullValues = self.get_parameter_value("NullValues",
default_value="''")
# Convert the InputFile parameter value relative path to an absolute path and expand for ${Property} syntax
input_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_InputFile, self)))
# Convert the NullValues parameter values to a list.
pv_NullValues = string_util.delimited_string_to_list(pv_NullValues)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_read_table(input_file_absolute, pv_TableID):
try:
# Create the table from the delimited file.
table = self.__read_table_from_delimited_file(
input_file_absolute, pv_TableID, pv_Delimiter,
pv_HeaderLines, pv_NullValues)
# Add the table to the GeoProcessor's Tables list.
self.command_processor.add_table(table)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading Table {} from delimited file ({}).".format(
pv_TableID, input_file_absolute)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Read the tabular data from the Excel workbook/worksheet. Create a Table object, and add to the
GeoProcessor's tables list.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_InputFile = self.get_parameter_value("InputFile")
pv_Worksheet = self.get_parameter_value("Worksheet")
pv_TableID = self.get_parameter_value("TableID")
# Convert the InputFile parameter value relative path to an absolute path and expand for ${Property} syntax.
file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_InputFile, self)))
# If the pv_TableID is a valid %-formatter, assign the pv_GeoLayerID the corresponding value.
if pv_TableID in ['%f', '%F', '%E', '%P', '%p']:
pv_TableID = io_util.expand_formatter(file_absolute, pv_TableID)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_read_table(file_absolute, pv_Worksheet, pv_TableID):
try:
# Assign the Worksheet parameter to the name of the first Excel worksheet, if it was not specified.
if pv_Worksheet is None:
pv_Worksheet = pandas_util.create_excel_workbook_obj(
file_absolute).sheet_names[0]
# Assign the TableID parameter to the name of the first Excel worksheet, if it was not specified.
if pv_TableID is None:
pv_TableID = pv_Worksheet
# Create a Pandas Data Frame object.
df = pandas_util.create_data_frame_from_excel(
file_absolute, pv_Worksheet)
# Create a Table and add it to the geoprocessor's Tables list.
table_obj = Table(pv_TableID, df, file_absolute)
self.command_processor.add_table(table_obj)
# Raise an exception if an unexpected error occurs during the process.
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading Table {} from Excel file {}.".format(
pv_TableID, pv_InputFile)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Write the Table to a delimited file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_TableID = self.get_parameter_value("TableID")
pv_OutputFile = self.get_parameter_value("OutputFile")
pv_Delimiter = self.get_parameter_value("Delimiter", default_value=",")
pv_IncludeColumns = self.get_parameter_value("IncludeColumns",
default_value="*")
pv_ExcludeColumns = self.get_parameter_value("ExcludeColumns",
default_value="")
pv_WriteHeaderRow = self.get_parameter_value("WriteHeaderRow",
default_value="True")
pv_WriteIndexColumn = self.get_parameter_value("WriteIndexColumn",
default_value="False")
pv_SortColumns = self.get_parameter_value("SortColumns")
pv_SortOrder = self.get_parameter_value("SortOrder", default_value="")
pv_ArrayFormat = self.get_parameter_value(
"ArrayFormat", default_value="SquareBrackets")
pv_NullValueFormat = self.get_parameter_value("NullValueFormat",
default_value="Null")
# Convert the IncludeColumns, ExcludeColumns, and SortColumns parameter values to lists.
cols_to_include = string_util.delimited_string_to_list(
pv_IncludeColumns)
cols_to_exclude = string_util.delimited_string_to_list(
pv_ExcludeColumns)
sort_cols_list = string_util.delimited_string_to_list(pv_SortColumns)
# Convert the SortOrder to a dictionary.
sort_dictionary = string_util.delimited_string_to_dictionary_one_value(
pv_SortOrder,
entry_delimiter=",",
key_value_delimiter=":",
trim=True)
# Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
# Covert the Boolean parameters from string to Boolean values.
pv_WriteHeaderRow = string_util.str_to_bool(pv_WriteHeaderRow)
pv_WriteIndexColumn = string_util.str_to_bool(pv_WriteIndexColumn)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_write_table(pv_TableID, output_file_absolute,
pv_Delimiter, sort_cols_list):
try:
# Get the Table object
table = self.command_processor.get_table(pv_TableID)
# Determine if square brackets should be used depending on the user input of the ArrayFormat parameter.
use_sq_brackets = False
if pv_ArrayFormat.upper() == "SQUAREBRACKETS":
use_sq_brackets = True
# Determine if the null values should be used depending on the user input of the NullValueFormat
# parameter.
use_null_value = False
if pv_NullValueFormat.upper() == "NULL":
use_null_value = True
# Write the table to the delimited file.
self.__write_table_to_delimited_file(
output_file_absolute, table, pv_Delimiter, cols_to_include,
cols_to_exclude, pv_WriteHeaderRow, pv_WriteIndexColumn,
sort_cols_list, sort_dictionary, use_sq_brackets,
use_null_value)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error writing Table {} to delimited file {}.".format(
pv_TableID, pv_OutputFile)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Run the program, which can generate output files.
Returns:
None.
Raises:
ValueError: if a runtime input error occurs.
RuntimeError: if a runtime error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
logger.info('In RunProgram.run_command')
# Get data for the command
print("command parameters=" +
string_util.format_dict(self.command_parameters))
pv_CommandLine = self.get_parameter_value('CommandLine')
pv_UseCommandShell = self.get_parameter_value('UseCommandShell')
use_command_shell = False # Default
if pv_UseCommandShell is not None and pv_UseCommandShell == 'True':
use_command_shell = True
pv_IncludeParentEnvVars = self.get_parameter_value(
'IncludeParentEnvVars')
include_parent_env_vars = True # Default
if pv_IncludeParentEnvVars is not None and pv_IncludeParentEnvVars == 'False':
include_parent_env_vars = False
pv_IncludeEnvVars = self.get_parameter_value('IncludeEnvVars')
include_env_vars_dict = None
if pv_IncludeEnvVars is not None and pv_IncludeEnvVars != "":
# Have specified environment variables to include
# - expand the environment variable value using processor properties
include_env_vars_dict = string_util.delimited_string_to_dictionary_one_value(
pv_IncludeEnvVars, key_value_delimiter="=", trim=True)
for key, value in include_env_vars_dict.items():
include_env_vars_dict[
key] = self.command_processor.expand_parameter_value(
value, self)
# Add environment variables individually by name
# - these are used when a list of parameters is difficult to parse
# - this is kind of ugly but meets requirements in the short term
pv_IncludeEnvVarName1 = self.get_parameter_value('IncludeEnvVarName1')
pv_IncludeEnvVarValue1 = self.get_parameter_value(
'IncludeEnvVarValue1')
if pv_IncludeEnvVarName1 is not None and pv_IncludeEnvVarName1 != "":
if include_env_vars_dict is None:
include_env_vars_dict = {}
include_env_vars_dict[
pv_IncludeEnvVarName1] = pv_IncludeEnvVarValue1
pv_IncludeEnvVarName2 = self.get_parameter_value('IncludeEnvVarName2')
pv_IncludeEnvVarValue2 = self.get_parameter_value(
'IncludeEnvVarValue2')
if pv_IncludeEnvVarName2 is not None and pv_IncludeEnvVarName2 != "":
if include_env_vars_dict is None:
include_env_vars_dict = {}
include_env_vars_dict[
pv_IncludeEnvVarName2] = pv_IncludeEnvVarValue2
pv_IncludeEnvVarName3 = self.get_parameter_value('IncludeEnvVarName3')
pv_IncludeEnvVarValue3 = self.get_parameter_value(
'IncludeEnvVarValue3')
if pv_IncludeEnvVarName3 is not None and pv_IncludeEnvVarName3 != "":
if include_env_vars_dict is None:
include_env_vars_dict = {}
include_env_vars_dict[
pv_IncludeEnvVarName3] = pv_IncludeEnvVarValue3
pv_IncludeEnvVarName4 = self.get_parameter_value('IncludeEnvVarName4')
pv_IncludeEnvVarValue4 = self.get_parameter_value(
'IncludeEnvVarValue4')
if pv_IncludeEnvVarName4 is not None and pv_IncludeEnvVarName4 != "":
if include_env_vars_dict is None:
include_env_vars_dict = {}
include_env_vars_dict[
pv_IncludeEnvVarName4] = pv_IncludeEnvVarValue4
pv_IncludeEnvVarName5 = self.get_parameter_value('IncludeEnvVarName5')
pv_IncludeEnvVarValue5 = self.get_parameter_value(
'IncludeEnvVarValue5')
if pv_IncludeEnvVarName5 is not None and pv_IncludeEnvVarName5 != "":
if include_env_vars_dict is None:
include_env_vars_dict = {}
include_env_vars_dict[
pv_IncludeEnvVarName5] = pv_IncludeEnvVarValue5
pv_ExcludeEnvVars = self.get_parameter_value('ExcludeEnvVars')
exclude_env_vars_list = None
if pv_ExcludeEnvVars is not None and pv_ExcludeEnvVars != "":
# Have specified environment variables to exclude
exclude_env_vars_list = string_util.delimited_string_to_list(
pv_ExcludeEnvVars, trim=True)
pv_OutputFiles = self.get_parameter_value('OutputFiles')
output_files_list = None
if pv_OutputFiles is not None and pv_OutputFiles != "":
# Have specified output files to add to command output files
output_files_list = string_util.delimited_string_to_list(
pv_OutputFiles, trim=True)
# Expand each output file
ifile = -1
for output_file in output_files_list:
ifile = ifile + 1
output_files_list[ifile] = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
output_file, self)))
logger.info('Command line before expansion="' + pv_CommandLine + '"')
# Runtime checks on input
command_line_expanded = self.command_processor.expand_parameter_value(
pv_CommandLine, self)
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Run the program as a subprocess
try:
logger.info('Running command line "' + command_line_expanded + '"')
# Create the environment dictionary
env_dict = self.create_env_dict(include_parent_env_vars,
include_env_vars_dict,
exclude_env_vars_list)
print("env_dict=" + string_util.format_dict(env_dict))
# TODO smalers 2018-12-16 evaluate using shlex.quote() to handle command string
# TODO smalers 2018-12-16 handle standard input and output
p = subprocess.Popen(command_line_expanded,
shell=use_command_shell,
env=env_dict)
# Wait for the process to terminate since need it to be done before other commands do their work
# with the command output.
p.wait()
return_status = p.poll()
if return_status != 0:
warning_count += 1
message = 'Nonzero return status running program "' + command_line_expanded + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
except Exception as e:
warning_count += 1
message = 'Unexpected error running program "' + command_line_expanded + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
except:
warning_count += 1
message = 'Unexpected error running program "' + command_line_expanded + '"'
logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
# If any output files were indicated, add to the command output if they exist
if output_files_list is not None and len(output_files_list) > 0:
for output_file in output_files_list:
if os.path.isfile(output_file):
# Add the log file to output
self.command_processor.add_output_file(output_file)
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
logger.warning(message)
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command.
Copy the source file to the destination.
Returns:
Nothing.
Raises:
RuntimeError: if a runtime input error occurs.
"""
warning_count = 0
logger = logging.getLogger(__name__)
# Get data for the command
pv_SourceFile = self.get_parameter_value('SourceFile')
pv_DestinationFile = self.get_parameter_value('DestinationFile')
# Runtime checks on input
pv_SourceFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_SourceFile, self)))
pv_DestinationFile_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_DestinationFile, self)))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings about command parameters."
logger.warning(message)
raise ValueError(message)
# Do the processing
try:
input_count = 2
if not os.path.exists(pv_SourceFile_absolute):
warning_count += 1
message = 'The source file does not exist: "' + pv_SourceFile_absolute + '"'
self.command_status.addToLog(
CommandPhaseType.RUN,
CommandLogRecord(
CommandStatusType.FAILURE, message,
"Verify that the source exists at the time the command is run."
))
logger.warning(message)
input_count -= 1
destination_folder = os.path.dirname(pv_DestinationFile_absolute)
if not os.path.exists(destination_folder):
warning_count += 1
message = 'The destination folder does not exist: "' + destination_folder + '"'
self.command_status.addToLog(
CommandPhaseType.RUN,
CommandLogRecord(
CommandStatusType.FAILURE, message,
"Verify that the destination folder exists at the time the command is run."
))
input_count -= 1
if input_count == 2:
# Try to do the copy
logger.info('Copying file "' + pv_SourceFile_absolute +
'" to "' + pv_DestinationFile_absolute + '"')
copyfile(pv_SourceFile_absolute, pv_DestinationFile_absolute)
except Exception as e:
warning_count += 1
message = 'Unexpected error copying file "' + pv_SourceFile_absolute + '" to "' + \
pv_DestinationFile_absolute + '"'
traceback.print_exc(file=sys.stdout)
logger.exception(message, e)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
"See the log file for details."))
if warning_count > 0:
message = "There were " + str(
warning_count) + " warnings processing the command."
raise RuntimeError(message)
self.command_status.refresh_phase_severity(CommandPhaseType.RUN,
CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Read the layer file from a delimited file, create a GeoLayer object, and add to the
GeoProcessor's geolayer list.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_DelimitedFile = self.get_parameter_value("DelimitedFile")
pv_Delimiter = self.get_parameter_value("Delimiter", default_value=',')
pv_GeometryFormat = self.get_parameter_value("GeometryFormat")
pv_XColumn = self.get_parameter_value("XColumn", default_value=None)
pv_YColumn = self.get_parameter_value("YColumn", default_value=None)
pv_WKTColumn = self.get_parameter_value("WKTColumn",
default_value=None)
pv_CRS = self.get_parameter_value("CRS")
pv_GeoLayerID = self.get_parameter_value("GeoLayerID",
default_value='%f')
# Convert the DelimitedFile parameter value relative path to an absolute path and expand for ${Property}
# syntax
delimited_file_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_DelimitedFile, self)))
# If the pv_GeoLayerID is a valid %-formatter, assign the pv_GeoLayerID the corresponding value.
if pv_GeoLayerID in ['%f', '%F', '%E', '%P', '%p']:
pv_GeoLayerID = io_util.expand_formatter(delimited_file_abs,
pv_GeoLayerID)
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_read_geolayer(delimited_file_abs, pv_Delimiter,
pv_GeometryFormat, pv_XColumn,
pv_YColumn, pv_WKTColumn, pv_CRS,
pv_GeoLayerID):
try:
if pv_GeometryFormat.upper() == "XY":
# Create a QGSVectorLayer object with the delimited file.
qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_delimited_file_xy(
delimited_file_abs, pv_Delimiter, pv_CRS, pv_XColumn,
pv_YColumn)
else:
# Create a QGSVectorLayer object with the delimited file.
qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_delimited_file_wkt(
delimited_file_abs, pv_Delimiter, pv_CRS, pv_WKTColumn)
# Create a GeoLayer and add it to the geoprocessor's GeoLayers list.
geolayer_obj = GeoLayer(pv_GeoLayerID, qgs_vector_layer,
delimited_file_abs)
self.command_processor.add_geolayer(geolayer_obj)
# Raise an exception if an unexpected error occurs during the process.
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading GeoLayer {} from delimited file {}.".format(
pv_GeoLayerID, pv_DelimitedFile)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Read the feature classes within a file geodatabase. For each desired feature class (can be
specified by the Subset_Pattern parameter), create a GeoLayer object, and add to the GeoProcessor's geolayer
list.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the required and optional parameter values
pv_SpatialDataFolder = self.get_parameter_value("SpatialDataFolder")
pv_ReadOnlyOneFeatureClass = self.get_parameter_value(
"ReadOnlyOneFeatureClass")
pv_Subset_Pattern = self.get_parameter_value("Subset_Pattern")
pv_GeoLayerID_prefix = self.get_parameter_value("GeoLayerID_prefix")
pv_GeoLayerID = self.get_parameter_value("GeoLayerID")
pv_FeatureClass = self.get_parameter_value("FeatureClass")
# Convert the ReadOnlyOneFeatureClass from a string value to a Boolean value.
pv_ReadOnlyOneFeatureClass = string_util.str_to_bool(
pv_ReadOnlyOneFeatureClass)
# Convert the SpatialDataFolder parameter value relative path to an absolute path
sd_folder_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_SpatialDataFolder, self)))
# Run the initial checks on the parameter values. Only continue if the checks passed.
if self.__should_read_gdb(sd_folder_abs):
# If configured to only read one Feature Class into one GeoLayer.
if pv_ReadOnlyOneFeatureClass:
# Run the check to see if the GeoLayer should be read.
if self.__should_read_geolayer(pv_GeoLayerID, True,
pv_FeatureClass, sd_folder_abs):
try:
# Get the full pathname to the feature class
# TODO egiles 2018-01-04 Need to research how to properly document feature class source path
spatial_data_file_absolute = os.path.join(
sd_folder_abs, str(pv_FeatureClass))
# Create a QgsVectorLayer object from the feature class.
qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_feature_class(
sd_folder_abs, pv_FeatureClass)
# Create a GeoLayer and add it to the geoprocessor's GeoLayers list
geolayer_obj = GeoLayer(pv_GeoLayerID,
qgs_vector_layer,
spatial_data_file_absolute)
self.command_processor.add_geolayer(geolayer_obj)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading feature class ({}) from file geodatabase ({}).".format(
pv_FeatureClass, sd_folder_abs)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE,
message, recommendation))
# If configured to read multiple Feature Classes into multiple GeoLayers.
else:
# Get a list of all of the feature classes in the file geodatabase.
fc_list = self.__return_a_list_of_fc(sd_folder_abs)
# Filter the fc_list to only include feature classes that meet the Subset Pattern configuration. If
# the Subset Pattern configuration is None, all feature classes will remain in the fc_list.
fc_list = string_util.filter_list_of_strings(
fc_list, [pv_Subset_Pattern])
# Iterate over the feature classes in the fc_list.
for feature_class in fc_list:
# Determine the GeoLayerID.
if pv_GeoLayerID_prefix:
geolayer_id = "{}_{}".format(pv_GeoLayerID_prefix,
feature_class)
else:
geolayer_id = feature_class
# Run the secondary checks on the parameter values. Only continue if the checks passed.
if self.__should_read_geolayer(geolayer_id, False):
try:
# Get the full pathname to the feature class
# TODO egiles 2018-01-04 Need to research how to properly document feature class source path
spatial_data_file_absolute = os.path.join(
sd_folder_abs, str(feature_class))
# Create a QgsVectorLayer object from the feature class.
qgs_vector_layer = qgis_util.read_qgsvectorlayer_from_feature_class(
sd_folder_abs, feature_class)
# Create a GeoLayer and add it to the geoprocessor's GeoLayers list
geolayer_obj = GeoLayer(
geolayer_id, qgs_vector_layer,
spatial_data_file_absolute)
self.command_processor.add_geolayer(geolayer_obj)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading feature class ({}) from file geodatabase ({}).".format(
feature_class, sd_folder_abs)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE,
message, recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Execute the Sql statement on the DataStore.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values. The DatabasePort parameter value will be obtained later in the code.
pv_DataStoreID = self.get_parameter_value("DataStoreID")
pv_Sql = self.get_parameter_value("Sql")
pv_SqlFile = self.get_parameter_value("SqlFile")
pv_DataStoreProcedure = self.get_parameter_value("DataStoreProcedure")
# Expand for ${Property} syntax.
pv_DataStoreID = self.command_processor.expand_parameter_value(pv_DataStoreID, self)
pv_Sql = self.command_processor.expand_parameter_value(pv_Sql, self)
if pv_SqlFile:
pv_SqlFile = io_util.verify_path_for_os(io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_SqlFile, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_run_sql(pv_DataStoreID):
try:
# Get the DataStore object
datastore_obj = self.command_processor.get_datastore(pv_DataStoreID)
# If using the Sql method, the sql_statement is the user-provided sql statement.
if pv_Sql:
sql_statement = pv_Sql
# If using the SqlFile method, the sql_statement in read from the provided file.
elif pv_SqlFile:
# Get the SQL statement from the file.
f = open(pv_SqlFile, 'r')
sql_statement = f.read().strip()
# If using the DataStoreProcedure method, ... .
else:
sql_statement = None
# Execute and commit the SQL statement.
datastore_obj.run_sql(sql_statement)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error executing the Sql statement on the {} DataStore.".format(pv_DataStoreID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message, recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def ui_action_select_file(self, event_button):
"""
Open a file selector dialog to select an input or output file (or folder) to be used as a command
parameter value.
Args:
event_button (QPushButton): the instance of the button for which the event is generated.
Use to get the parameter name, so as to get other parameter/component data.
Returns:
None
"""
logger = logging.getLogger(__name__)
# Initialize folder to None and determine below with several checks.
folder_start = None
# Get properties to configure the file selector.
# - use the object name to parse out the parameter name
object_name = event_button.objectName()
print("object_name=" + str(object_name))
parameter_name = object_name.split(".")[0]
# Do the following first because it influences defaults below
request_key = parameter_name + "." + "FileSelector.SelectFolder"
select_folder = False
try:
# The following should match ParameterName.FileSelectorTitle
select_folder = self.command.parameter_input_metadata[request_key]
except KeyError:
# Default was specified above...
pass
request_key = parameter_name + "." + "FileSelector.Title"
if select_folder:
select_file_title = "Select folder"
else:
select_file_title = "Select file"
try:
# The following should match ParameterName.FileSelectorTitle
select_file_title = self.command.parameter_input_metadata[request_key]
except KeyError:
# Default was specified above...
pass
# Get the existing value in the text field, which will correspond to the parameter name value.
# - if specified as absolute path, use it as is
# - if a relative path, append to the working directory or if that is not available the user's home folder
parameter_QLineEdit = None
working_dir = self.command.command_processor.get_property('WorkingDir')
user_folder = self.app_session.get_user_folder()
try:
parameter_QLineEdit = self.input_ui_components[parameter_name]
# Get the parameter value
parameter_value = parameter_QLineEdit.text()
# If the parameter is empty or null
if parameter_value is None or parameter_value == "":
# Try to set the folder to the working directory first
if working_dir is not None:
folder_start = working_dir
else:
# Finally, use the user's home folder
folder_start = self.app_session.get_user_folder()
else:
# The parameter is specified.
if os.path.isabs(parameter_value):
# The input is an absolute path so use as is
folder_start = parameter_value
else:
# The input is relative to the working directory so append to working directory with
# filesystem separator.
if working_dir is not None:
folder_start = io_util.to_absolute_path(working_dir, parameter_value)
else:
folder_start = io_util.to_absolute_path(user_folder, parameter_value)
except KeyError:
# Can't determine the input component so will assume the working directory, if available
if working_dir is not None:
folder_start = working_dir
else:
# Finally, use the user's home folder
folder_start = user_folder
# A browser window will appear to allow the user to browse to the desired file.
# The absolute pathname of the command file is added to the cmd_filepath variable.
use_qt_dialog = True # For now use Qt build-in dialog but may want to try native dialog
filepath_selected = None
if use_qt_dialog:
parameter_QFileDialog = QtWidgets.QFileDialog(self, select_file_title, folder_start)
if select_folder:
# A directory is being selected
parameter_QFileDialog.setFileMode(QtWidgets.QFileDialog.DirectoryOnly)
if parameter_QFileDialog.exec_() == QtWidgets.QFileDialog.Accepted:
filepath_selected = parameter_QFileDialog.selectedFiles()[0]
if not filepath_selected:
# The file selection was canceled
return
# Convert the selected file path to relative path, using the command file folder as the working directory.
if working_dir is not None:
filepath_selected = io_util.to_relative_path(working_dir, filepath_selected)
# Set the file in the text field.
parameter_QLineEdit.setText(filepath_selected)
def run_command(self):
"""
Run the command. Read the Table from the DataStore
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values.
pv_DataStoreID = self.get_parameter_value("DataStoreID")
pv_DataStoreTable = self.get_parameter_value("DataStoreTable")
pv_Sql = self.get_parameter_value("Sql")
pv_SqlFile = self.get_parameter_value("SqlFile")
pv_Top = self.get_parameter_value("Top")
pv_TableID = self.get_parameter_value("TableID")
pv_IncludeColumns = self.get_parameter_value("IncludeColumns",
default_value="*")
pv_ExcludeColumns = self.get_parameter_value("ExcludeColumns",
default_value="")
# Expand for ${Property} syntax.
pv_DataStoreID = self.command_processor.expand_parameter_value(
pv_DataStoreID, self)
pv_DataStoreTable = self.command_processor.expand_parameter_value(
pv_DataStoreTable, self)
pv_Sql = self.command_processor.expand_parameter_value(pv_Sql, self)
pv_TableID = self.command_processor.expand_parameter_value(
pv_TableID, self)
# Convert the IncludeColumns and ExcludeColumns parameter values to lists.
cols_to_include = string_util.delimited_string_to_list(
pv_IncludeColumns)
cols_to_exclude = string_util.delimited_string_to_list(
pv_ExcludeColumns)
# If available, convert the SqlFile parameter value relative path to an absolute path and expand for
# ${Property} syntax.
if pv_SqlFile:
pv_SqlFile = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_SqlFile, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_read_table(pv_SqlFile, pv_TableID, pv_DataStoreID):
try:
# Get the DataStore object
datastore = self.command_processor.get_datastore(
pv_DataStoreID)
# Set the SQL statement to None until proof that SQL statement exists.
sql_statement = None
# If using the Sql method, the sql_statement is the user-provided sql statement.
if pv_Sql:
sql_statement = pv_Sql
if '%' in sql_statement:
sql_statement = sql_statement.replace('%', '%%')
# If using the Sql method, the sql_statement is the user-provided sql statement within a file.
if pv_SqlFile:
# Get the SQL statement from the file.
f = open(pv_SqlFile, 'r')
sql_statement = f.read().strip()
if '%' in sql_statement:
sql_statement = sql_statement.replace('%', '%%')
# Create the Table from the DataStore.
table = self.__read_table_from_datastore(
datastore, pv_DataStoreTable, pv_TableID, pv_Top,
sql_statement, cols_to_include, cols_to_exclude)
# Add the table to the GeoProcessor's Tables list.
self.command_processor.add_table(table)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error reading Table {} from DataStore ({}).".format(
pv_TableID, pv_DataStoreID)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Extract the compressed file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the File and the DeleteFile parameter values.
pv_File = self.get_parameter_value("File")
pv_DeleteFile = self.get_parameter_value("DeleteFile",
default_value="False")
# Convert the File parameter value relative path to an absolute path. Expand for ${Property} syntax.
file_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(pv_File, self)))
# Get the FileType parameter value.
default_file_ext = self.__get_default_file_type(file_abs)
pv_FileType = self.get_parameter_value("FileType",
default_value=default_file_ext)
# Get the OutputFolder parameter value.
parent_folder = io_util.get_path(file_abs)
pv_OutputFolder = self.get_parameter_value("OutputFolder",
default_value=parent_folder)
# Convert the OutputFolder parameter value relative path to an absolute path. Expand for ${Property} syntax.
output_folder_abs = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFolder, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_extract_file(file_abs, output_folder_abs,
pv_FileType):
try:
# If the file is a .zip file, extract the zip file.
if pv_FileType.upper() == "ZIP":
zip_util.unzip_all_files(file_abs, output_folder_abs)
# If the file is a .tar file, extract the tar file.
elif pv_FileType.upper() == "TAR":
zip_util.untar_all_files(file_abs, output_folder_abs)
# If configured, remove the input compressed file.
if string_util.str_to_bool(pv_DeleteFile):
os.remove(file_abs)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error extracting the {} file ({}).".format(
pv_FileType, pv_File)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
def run_command(self):
"""
Run the command. Write the Table to an excel file.
Returns: None.
Raises:
RuntimeError if any warnings occurred during run_command method.
"""
# Obtain the parameter values except for the OutputCRS
pv_TableID = self.get_parameter_value("TableID")
pv_OutputFile = self.get_parameter_value("OutputFile")
pv_OutputWorksheet = self.get_parameter_value("OutputWorksheet")
pv_ColumnsToInclude = self.get_parameter_value("ColumnsToInclude",
default_value="*")
pv_ColumnsToExclude = self.get_parameter_value("ColumnsToExclude",
default_value="")
pv_WriteIndexColumn = self.get_parameter_value("WriteIndexColumn",
default_value="True")
# Convert the Boolean parameters from string to valid Boolean values.
pv_WriteIndexColumn = string_util.str_to_bool(pv_WriteIndexColumn)
# Convert the ColumnsToInclude and ColumnsToExclude parameter values to lists.
cols_to_include = string_util.delimited_string_to_list(
pv_ColumnsToInclude)
cols_to_exclude = string_util.delimited_string_to_list(
pv_ColumnsToExclude)
# Convert the OutputFile parameter value relative path to an absolute path and expand for ${Property} syntax
output_file_absolute = io_util.verify_path_for_os(
io_util.to_absolute_path(
self.command_processor.get_property('WorkingDir'),
self.command_processor.expand_parameter_value(
pv_OutputFile, self)))
# Run the checks on the parameter values. Only continue if the checks passed.
if self.__should_write_table(pv_TableID, output_file_absolute):
try:
# Get the Table object
table = self.command_processor.get_table(pv_TableID)
# Get a list of all the available column names in the Table.
all_cols_names = list(table.df)
# Sort the list to create a second list that only includes the attributes that should be removed.
cols_to_keep = string_util.filter_list_of_strings(
list(table.df),
cols_to_include,
cols_to_exclude,
return_inclusions=True)
# For the columns configured to be written, order them in the same order that they were in with the
# original table. This step ensures that the output table product has the columns in the same order as
# the user would expect.
sorted_cols_to_keep = []
for col in all_cols_names:
if col in cols_to_keep:
sorted_cols_to_keep.append(col)
# Write the tables to an Excel file.
pandas_util.write_df_to_excel(table.df, output_file_absolute,
pv_OutputWorksheet,
sorted_cols_to_keep,
pv_WriteIndexColumn)
# Add the output file to the GeoProcessor's list of output files.
self.command_processor.add_output_file(output_file_absolute)
# Raise an exception if an unexpected error occurs during the process
except Exception as e:
self.warning_count += 1
message = "Unexpected error writing Table {} to Excel workbook file {}.".format(
pv_TableID, pv_OutputFile)
recommendation = "Check the log file for details."
self.logger.error(message, exc_info=True)
self.command_status.add_to_log(
CommandPhaseType.RUN,
CommandLogRecord(CommandStatusType.FAILURE, message,
recommendation))
# Determine success of command processing. Raise Runtime Error if any errors occurred
if self.warning_count > 0:
message = "There were {} warnings proceeding this command.".format(
self.warning_count)
raise RuntimeError(message)
# Set command status type as SUCCESS if there are no errors.
else:
self.command_status.refresh_phase_severity(
CommandPhaseType.RUN, CommandStatusType.SUCCESS)
