def __init__(self):
"""
The registry contains all the recipes available for the wcst import
To add one please register it in the _init_registry method
"""
self.registry = {}
self._init_registry()
self.sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
def _get_coverage(self):
"""
Returns the coverage to be used for the importer
"""
gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(self.session.get_files())
crs = gdal_dataset.get_crs()
general_recipe = GeneralRecipe(self.session)
global_metadata_fields = general_recipe._global_metadata_fields()
local_metadata_fields = general_recipe._local_metadata_fields()
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
gdal_coverage_converter = GdalToCoverageConverter(self.resumer, self.session.get_default_null_values(),
self.recipe_type, sentence_evaluator,
self.session.get_coverage_id(),
None, self.session.get_files(),
crs, None, None,
global_metadata_fields, local_metadata_fields,
None, None, general_recipe._metadata_type(),
None, None)
coverage_slices = self._get_coverage_slices(crs, gdal_coverage_converter)
fields = GdalRangeFieldsGenerator(gdal_dataset, self.options['band_names']).get_range_fields()
global_metadata = None
if len(coverage_slices) > 0:
global_metadata = gdal_coverage_converter._generate_global_metadata(coverage_slices[0], self.evaluator_slice)
coverage = Coverage(self.session.get_coverage_id(), coverage_slices, fields, gdal_dataset.get_crs(),
gdal_dataset.get_band_gdal_type(), self.options['tiling'], global_metadata)
return coverage
def _get_grib_coverage(self, recipe_type):
"""
Returns a coverage that uses the grib slicer
:param: string recipe_type the type of grib
:rtype: master.importer.coverage.Coverage
"""
crs = self._resolve_crs(self.options['coverage']['crs'])
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
pixel_is_point = False
if 'pixelIsPoint' in self.options['coverage']['slicer'] and self.options['coverage']['slicer']['pixelIsPoint']:
pixel_is_point = True
coverage = GRIBToCoverageConverter(self.resumer, self.session.default_null_values,
recipe_type,
sentence_evaluator, self.session.get_coverage_id(),
self._read_bands(),
self.session.get_files(), crs, self._read_axes(crs),
self.options['tiling'], self._global_metadata_fields(),
self._bands_metadata_fields(),
self._local_metadata_fields(),
self._axes_metadata_fields(),
self._metadata_type(),
self.options['coverage']['grid_coverage'], pixel_is_point,
self.options['import_order']).to_coverage()
return coverage
def _get_gdal_coverages(self, recipe_type):
"""
Returns a list of coverage that uses the gdal slicer
:param string: recipe_type the type of recipe
:rtype: master.importer.coverage.Coverage
"""
crs = self._resolve_crs(self.options['coverage']['crs'])
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
input_files = self.session.get_files()
user_axes = self._read_axes(crs)
DEFAULT_NUMBER_GDAL_DIMENSIONS = 2
self.__validate_data_bound_axes(user_axes, DEFAULT_NUMBER_GDAL_DIMENSIONS)
coverages = GdalToCoverageConverter(self.resumer, self.session.default_null_values,
recipe_type, sentence_evaluator, self.session.get_coverage_id(),
self._read_bands(),
input_files, crs, user_axes,
self.options['tiling'], self._global_metadata_fields(),
self._local_metadata_fields(),
self._bands_metadata_fields(),
self._axes_metadata_fields(),
self._metadata_type(),
self.options['coverage']['grid_coverage'],
self.options['import_order'],
self.session).to_coverages()
return coverages
def _create_convertor(self, convertors, cov_id, crs_code, level, res):
recipe_type = GdalToCoverageConverter.RECIPE_TYPE
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
files = []
crs = self._get_crs(crs_code)
bands_metadata_fields = {}
axis_metadata_fields = {}
default_null_values = [self.DEFAULT_NULL_VALUE]
return GdalToCoverageConverter(self.resumer,
default_null_values,
recipe_type,
sentence_evaluator,
cov_id,
self.BANDS[level][res],
files,
crs,
self._read_axes(crs),
self.options['tiling'],
self._global_metadata_fields(),
self._local_metadata_fields(),
bands_metadata_fields,
axis_metadata_fields,
self._metadata_type(),
self.grid_cov,
self.import_order,
self.session)
def _get_coverages(self):
"""
Returns the list of coverages to be used for the importer
"""
gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(
self.session.get_files())
crs = CRSUtil.get_compound_crs(
[self.options['time_crs'],
gdal_dataset.get_crs()])
general_recipe = GeneralRecipe(self.session)
global_metadata_fields = general_recipe._global_metadata_fields()
local_metadata_fields = general_recipe._local_metadata_fields()
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
gdal_coverage_converter = GdalToCoverageConverter(
self.resumer, self.session.get_default_null_values(),
self.recipe_type,
sentence_evaluator, self.session.get_coverage_id(), None,
self.session.get_files(), crs, None, None,
global_metadata_fields, local_metadata_fields, None, None,
general_recipe._metadata_type(), None, None, self.session)
coverage_slices_dict = self._get_coverage_slices(
crs, gdal_coverage_converter)
fields = GdalRangeFieldsGenerator(
gdal_dataset, self.options['band_names']).get_range_fields()
global_metadata = None
if len(coverage_slices_dict["base"]) > 0:
global_metadata = gdal_coverage_converter._generate_global_metadata(
coverage_slices_dict["base"][0], self.evaluator_slice)
results = []
base_coverage_id = self.session.get_coverage_id()
for key, value in coverage_slices_dict.items():
if key == "base":
# base coverage
coverage = Coverage(base_coverage_id,
coverage_slices_dict[key], fields, crs,
gdal_dataset.get_band_gdal_type(),
self.options['tiling'], global_metadata)
else:
# overview coverage (key = overview_index)
coverage_id = create_coverage_id_for_overview(
base_coverage_id, key)
coverage = Coverage(coverage_id, coverage_slices_dict[key],
fields, crs,
gdal_dataset.get_band_gdal_type(),
self.options['tiling'], global_metadata,
base_coverage_id, key)
results.append(coverage)
return results
def _get_gdal_coverage(self, recipe_type):
"""
Returns a coverage that uses the gdal slicer
:param string: recipe_type the type of recipe
:rtype: master.importer.coverage.Coverage
"""
crs = self._resolve_crs(self.options['coverage']['crs'])
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
coverage = GdalToCoverageConverter(recipe_type, sentence_evaluator, self.session.get_coverage_id(),
self._read_bands(),
self.session.get_files(), crs, self._read_axes(crs),
self.options['tiling'], self._global_metadata_fields(),
self._local_metadata_fields(), self._bands_metadata_fields(),
self._axes_metadata_fields(),
self._metadata_type(),
self.options['coverage']['grid_coverage']).to_coverage()
return coverage
def _create_convertor(self, cov_id):
recipe_type = GdalToCoverageConverter.RECIPE_TYPE
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
files = []
bands_metadata_fields = {}
axis_metadata_fields = {}
default_null_values = []
if self.product == self.DEFAULT_PRODUCT:
default_null_values = [self.DEFAULT_NULL_VALUE]
return GdalToCoverageConverter(
self.resumer, default_null_values, recipe_type,
sentence_evaluator, cov_id, [self.BAND], files, self.crs,
self._read_axes(self.crs), self.options['tiling'],
self._global_metadata_fields(), self._local_metadata_fields(),
bands_metadata_fields, axis_metadata_fields, self._metadata_type(),
self.grid_cov, self.import_order)
def _get_netcdf_coverage(self, recipe_type):
"""
Returns a coverage that uses the netcdf slicer
:param: string recipe_type the type of netcdf
:rtype: master.importer.coverage.Coverage
"""
crs = self._resolve_crs(self.options['coverage']['crs'])
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
# by default pixelIsPoint is not set to true in ingredient file
pixel_is_point = False
if 'pixelIsPoint' in self.options['coverage']['slicer'] and self.options['coverage']['slicer']['pixelIsPoint']:
pixel_is_point = True
coverage = NetcdfToCoverageConverter(recipe_type, sentence_evaluator, self.session.get_coverage_id(),
self._read_bands(),
self.session.get_files(), crs, self._read_axes(crs),
self.options['tiling'], self._netcdf_global_metadata_fields(),
self._local_metadata_fields(),
self._netcdf_bands_metadata_fields(),
self._netcdf_axes_metadata_fields(),
self._metadata_type(),
self.options['coverage']['grid_coverage'], pixel_is_point).to_coverage()
return coverage
def _get_netcdf_coverage(self, recipe_type):
"""
Returns a coverage that uses the netcdf slicer
:param: string recipe_type the type of netcdf
:rtype: master.importer.coverage.Coverage
"""
crs = self._resolve_crs(self.options['coverage']['crs'])
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
# by default pixelIsPoint is not set to true in ingredient file
pixel_is_point = False
if 'pixelIsPoint' in self.options['coverage']['slicer'] and self.options['coverage']['slicer']['pixelIsPoint']:
pixel_is_point = True
input_files = self.session.get_files()
user_axes = self._read_axes(crs)
bands = self._read_bands()
first_band = bands[0]
first_band_variable_identifier = first_band.identifier
netCDF4 = import_netcdf4()
# Get the number of dimensions of band variable to validate with number of axes specified in the ingredients file
number_of_dimensions = len(netCDF4.Dataset(input_files[0], 'r').variables[first_band_variable_identifier].dimensions)
self.__validate_data_bound_axes(user_axes, number_of_dimensions)
coverage = NetcdfToCoverageConverter(self.resumer, self.session.default_null_values,
recipe_type, sentence_evaluator, self.session.get_coverage_id(),
bands,
input_files, crs, user_axes,
self.options['tiling'], self._global_metadata_fields(),
self._local_metadata_fields(),
self._netcdf_bands_metadata_fields(),
self._netcdf_axes_metadata_fields(),
self._metadata_type(),
self.options['coverage']['grid_coverage'], pixel_is_point,
self.options['import_order'],
self.session).to_coverages()
return coverage
class RecipeRegistry:
def __init__(self):
"""
The registry contains all the recipes available for the wcst import
To add one please register it in the _init_registry method
"""
self.registry = {}
self._init_registry()
self.sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
def _init_registry(self):
"""
Initializes the registry with all the recipes available by looking into the recipes folder
"""
util = ReflectionUtil()
# load "official" rasdaman recipes manually
import recipes.general_coverage
import recipes.map_mosaic
import recipes.time_series_irregular
import recipes.time_series_regular
import recipes.wcs_extract
util.import_submodules(recipes.general_coverage)
util.import_submodules(recipes.map_mosaic)
util.import_submodules(recipes.time_series_irregular)
util.import_submodules(recipes.time_series_regular)
util.import_submodules(recipes.wcs_extract)
# user-created recipes are put in the recipes_custom directory, so
# wcst_import needs to load all possible recipes in this directory.
import recipes_custom
util.import_submodules(recipes_custom)
# register all loaded recipes
all_recipes = util.get_all_subclasses(BaseRecipe)
for recipe in all_recipes:
self.registry[recipe.get_name()] = recipe
def __run_shell_command(self, command, abort_on_error=False):
"""
Run a shell command and exit wcst_import if needed
:param str command: shell command to run
"""
try:
log.info("Executing shell command '{}'...".format(command))
output = subprocess.check_output(command, stderr=subprocess.STDOUT, shell=True)
output = decode_res(output)
if output != "":
log.info("Output result '{}'".format(output))
except subprocess.CalledProcessError as exc:
log.warn("Failed, status code '{}', error message '{}'.".format(exc.returncode, str(exc.output).strip()))
if abort_on_error:
log.error("wcst_import terminated on running hook command.")
exit(1)
def __run_hooks(self, session, hooks, after_ingestion=False):
"""
Run some hooks before/after analyzing input files
:param Session session:
:param dict[str:str] hooks: dictionary of before and after ingestion hooks
"""
# gdal (default), netcdf or grib
recipe_type = GdalToCoverageConverter.RECIPE_TYPE
if session.recipe["name"] == GeneralRecipe.RECIPE_TYPE:
recipe_type = session.recipe["options"]["coverage"]["slicer"]["type"]
for hook in hooks:
abort_on_error = False if "abort_on_error" not in hook else bool(hook["abort_on_error"])
if "description" in hook:
log.info("Executing hook '{}'...".format(make_bold(hook["description"])))
replace_paths = []
replace_path_template = None
if "replace_path" in hook:
# All replaced input files share same template format (e.g: file:path -> file:path.projected)
replace_path_template = hook["replace_path"][0]
# Evaluate shell command expression to get a runnable shell command
cmd_template = hook["cmd"]
files = session.files
if after_ingestion is True:
files = session.imported_files
for file in files:
evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(recipe_type, file)
cmd = self.sentence_evaluator.evaluate(cmd_template, evaluator_slice)
self.__run_shell_command(cmd, abort_on_error)
if FileExpressionEvaluator.PREFIX not in cmd_template:
# Only need to run hook once if ${...} does not exist in cmd command,
# otherwise it runs duplicate commands multiple times (!)
if replace_path_template is not None:
replace_path = self.sentence_evaluator.evaluate(replace_path_template, evaluator_slice)
replace_paths.append(FilePair(replace_path, file.filepath))
break
if replace_path_template is not None:
# Evaluate replace path expression to get a valid file input path
replace_path = self.sentence_evaluator.evaluate(replace_path_template, evaluator_slice)
tmp_files = glob.glob(replace_path, recursive=True)
for tmp_file in tmp_files:
if not isinstance(file, FilePair):
# The first replacement (must keep original input file path)
replace_paths.append(FilePair(tmp_file, file.filepath))
else:
# From the second replacement
replace_paths.append(FilePair(tmp_file, file.original_file_path))
if len(replace_paths) > 0:
# Use replaced file paths instead of original input file paths to analyze and create coverage slices
session.files = replace_paths
def __run_recipe(self, session, recipe):
"""
Run recipe
:param Session session:
:param BaseRecipe recipe:
"""
if session.before_hooks:
log.info(make_bold("Executing before ingestion hook(s)..."))
self.__run_hooks(session, session.before_hooks, False)
recipe.describe()
if session.blocking and not session.is_automated():
input("Press Enter to Continue...: ")
log.title("\nRunning")
if session.blocking:
# It needs to display progress bar (how many files imported)
t = Thread(target=run_status, args=(recipe,))
t.daemon = True
t.start()
recipe.run()
t.join()
else:
# Non-blocking mode, only 1 file is importing, no need to show progress bar (it takes longer time to join threads)
recipe.run()
recipe.importer = None
if session.after_hooks:
log.info(make_bold("Executing after ingestion hook(s)..."))
self.__run_hooks(session, session.after_hooks, True)
def run_recipe(self, session):
"""
Recipe session
:param Session session: the session of the import
:rtype BaseRecipe
"""
recipe_name = session.get_recipe()['name']
if recipe_name not in self.registry:
raise RecipeValidationException("Recipe '" + session.get_recipe()['name'] + "' not found; "
"if it's a custom recipe, please put it in the "
"'$RMANHOME/share/rasdaman/wcst_import/recipes_custom' folder.")
else:
recipe = self.registry[session.get_recipe()['name']](session)
log.title("Initialization")
if recipe_name != virtual_coverage_recipe.RECIPE_NAME:
number_of_files = len(session.get_files())
if number_of_files > 10:
number_of_files = 10
log.info("Collected first " + str(number_of_files) + " files: "
+ str([str(f) for f in session.get_files()[:10]]) + "...")
log.title("\nValidation")
recipe.validate()
# Show what recipe and coverage are imported only once
super(recipe.__class__, recipe).describe()
if session.blocking is True:
# Default blocking import mode (analyze all files -> import)
self.__run_recipe(session, recipe)
else:
# Non blocking import mode (analyze 1 file -> import then next file)
files = list(session.get_files())
for file in files:
session.files = [file]
self.__run_recipe(session, recipe)
log.success("Recipe executed successfully")