def __run_recipe(self, session, recipe):
"""
Run recipe
:param Session session:
:param BaseRecipe recipe:
"""
if session.before_hooks:
log.info(make_bold("Executing before ingestion hook(s)..."))
self.__run_hooks(session, session.before_hooks, False)
recipe.describe()
if session.blocking and not session.is_automated():
input("Press Enter to Continue...: ")
log.title("\nRunning")
if session.blocking:
# It needs to display progress bar (how many files imported)
t = Thread(target=run_status, args=(recipe,))
t.daemon = True
t.start()
recipe.run()
t.join()
else:
# Non-blocking mode, only 1 file is importing, no need to show progress bar (it takes longer time to join threads)
recipe.run()
recipe.importer = None
if session.after_hooks:
log.info(make_bold("Executing after ingestion hook(s)..."))
self.__run_hooks(session, session.after_hooks, True)
def describe(self):
log.info("The recipe has been validated and is ready to run.")
log.info(make_bold("Recipe: ") + self.session.get_recipe()['name'])
log.info(make_bold("WCS Service: ") + ConfigManager.wcs_service)
log.info(make_bold("Mocked: ") + str(ConfigManager.mock))
if ConfigManager.track_files:
log.info(
make_bold("Track files: ") + str(ConfigManager.track_files))
if ConfigManager.skip:
log.info(make_bold("Skip: ") + str(ConfigManager.skip))
if ConfigManager.retry:
log.info(make_bold("Retries: ") + str(ConfigManager.retries))
if ConfigManager.slice_restriction is not None:
log.info(
make_bold("Slice Restriction: ") +
str(ConfigManager.slice_restriction))
multiimporter = self._get_importer()
cov_num = len(multiimporter.importers)
i = 1
for importer in multiimporter.importers:
log.info("Coverage {}/{} - {}: {} files.".format(
i, cov_num, make_bold(importer.coverage.coverage_id),
len(importer.coverage.slices)))
i += 1
def __run_hooks(self, session, hooks, after_ingestion=False):
"""
Run some hooks before/after analyzing input files
:param Session session:
:param dict[str:str] hooks: dictionary of before and after ingestion hooks
"""
# gdal (default), netcdf or grib
recipe_type = GdalToCoverageConverter.RECIPE_TYPE
if session.recipe["name"] == GeneralRecipe.RECIPE_TYPE:
recipe_type = session.recipe["options"]["coverage"]["slicer"]["type"]
for hook in hooks:
abort_on_error = False if "abort_on_error" not in hook else bool(hook["abort_on_error"])
if "description" in hook:
log.info("Executing hook '{}'...".format(make_bold(hook["description"])))
replace_paths = []
replace_path_template = None
if "replace_path" in hook:
# All replaced input files share same template format (e.g: file:path -> file:path.projected)
replace_path_template = hook["replace_path"][0]
# Evaluate shell command expression to get a runnable shell command
cmd_template = hook["cmd"]
files = session.files
if after_ingestion is True:
files = session.imported_files
for file in files:
evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(recipe_type, file)
cmd = self.sentence_evaluator.evaluate(cmd_template, evaluator_slice)
self.__run_shell_command(cmd, abort_on_error)
if FileExpressionEvaluator.PREFIX not in cmd_template:
# Only need to run hook once if ${...} does not exist in cmd command,
# otherwise it runs duplicate commands multiple times (!)
if replace_path_template is not None:
replace_path = self.sentence_evaluator.evaluate(replace_path_template, evaluator_slice)
replace_paths.append(FilePair(replace_path, file.filepath))
break
if replace_path_template is not None:
# Evaluate replace path expression to get a valid file input path
replace_path = self.sentence_evaluator.evaluate(replace_path_template, evaluator_slice)
tmp_files = glob.glob(replace_path, recursive=True)
for tmp_file in tmp_files:
if not isinstance(file, FilePair):
# The first replacement (must keep original input file path)
replace_paths.append(FilePair(tmp_file, file.filepath))
else:
# From the second replacement
replace_paths.append(FilePair(tmp_file, file.original_file_path))
if len(replace_paths) > 0:
# Use replaced file paths instead of original input file paths to analyze and create coverage slices
session.files = replace_paths
def execute(self, request):
"""
Prints the service call that would be executed if a real executor would be used
:param WCSTRequest request: the request to be executed
:rtype str
"""
request = self.prepare_request(request)
service_call = self.base_url + "?" + request.get_query_string()
log.info(
make_bold(
"This is just a mocked request, no data will be changed."))
log.info(service_call)
def execute(self, request, output=False, mock=False, input_base_url=None):
"""
Executes a WCST request and returns the response to it
:param WCSTRequest request: the request to be executed
:rtype str
"""
request = self.prepare_request(request)
base_url_tmp = self.base_url
if input_base_url is not None:
base_url_tmp = input_base_url
service_call = base_url_tmp + "?" + request.get_query_string()
if output:
log.info(service_call)
if mock:
log.info(
make_bold(
"This is just a mocked request, no data will be changed."))
log.info(service_call)
return
try:
response = decode_res(
validate_and_read_url(base_url_tmp,
request.get_query_string()))
except Exception as ex:
raise WCSTException(
404, "Failed reading response from WCS service. "
"Detailed error: {}.".format(str(ex)), service_call)
namespaces = ""
# check if a WMS error occurred and parse accordingly
if response.find("ServiceExceptionReport") != -1:
namespaces = None
# if WCST error then using ows:ExceptionReport
elif response.find("ExceptionReport") != -1:
namespaces = {"ows": "http://www.opengis.net/ows/2.0"}
# Check error from response
self.__check_request_for_errors(response, namespaces, service_call)
try:
if str(response) != "" and str(response) != "None":
return encode_res(response)
return ""
except Exception as ex:
raise WCSTException(
0, "General exception while executing the request: " + str(ex),
service_call)
def describe(self):
"""
This methods is called before insert or update is run. You should override the method and add any comments
regarding the operations that you will perform via log.info to inform the user. You should explicitly state
the information that you deduced (e.g. timestamps for a timeseries) so that the consequences are clear.
"""
cov = CoverageUtil(self.session.get_coverage_id())
operation_type = "UPDATE" if cov.exists() else "INSERT"
log.info("The recipe has been validated and is ready to run.")
log.info(make_bold("Recipe: ") + self.session.get_recipe()['name'])
log.info(make_bold("Coverage: ") + self.session.get_coverage_id())
log.info(make_bold("WCS Service: ") + ConfigManager.wcs_service)
log.info(make_bold("Operation: ") + operation_type)
log.info(
make_bold("Subset Correction: ") +
str(ConfigManager.subset_correction))
log.info(make_bold("Mocked: ") + str(ConfigManager.mock))
log.info(make_bold("WMS Import: ") + str(self.session.wms_import))
# Blocking means analyzing all input files before importing all coverage slices
# Non-blocking means analazying 1 file then import 1 file then continue with next file.
import_mode = "Blocking"
if not self.session.blocking:
import_mode = "Non-blocking"
log.info(make_bold("Import mode: ") + import_mode)
if ConfigManager.track_files:
log.info(
make_bold("Track files: ") + str(ConfigManager.track_files))
if ConfigManager.skip:
log.info(make_bold("Skip: ") + str(ConfigManager.skip))
if ConfigManager.retry:
log.info(make_bold("Retries: ") + str(ConfigManager.retries))
if ConfigManager.slice_restriction is not None:
log.info(
make_bold("Slice Restriction: ") +
str(ConfigManager.slice_restriction))
pass
