def _get_slices(self, crs):
"""
Returns the slices for the collection of files given
"""
files = self.session.get_files()
crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id)
slices = []
count = 1
for file in files:
# NOTE: don't process any imported file from *.resume.json as it is just waisted time
if not self.resumer.is_file_imported(file.filepath):
timer = Timer()
# print which file is analyzing
FileUtil.print_feedback(count, len(files), file.filepath)
if not FileUtil.validate_file_path(file.filepath):
continue
valid_coverage_slice = True
try:
subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill()
except Exception as ex:
# If skip: true then just ignore this file from importing, else raise exception
FileUtil.ignore_coverage_slice_from_file_if_possible(file.get_filepath(), ex)
valid_coverage_slice = False
if valid_coverage_slice:
slices.append(Slice(subsets, FileDataProvider(file)))
timer.print_elapsed_time()
count += 1
return slices
def _get_coverage_slices(self, crs, gdal_coverage_converter):
"""
Returns the slices for the collection of files given
"""
crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id)
slices_dict = self.create_dict_of_slices(self.session.import_overviews)
timeseries = self._generate_timeseries_tuples()
count = 1
for tpair in timeseries:
file = tpair.file
file_path = tpair.file.get_filepath()
timer = Timer()
# print which file is analyzing
FileUtil.print_feedback(count, len(timeseries), file_path)
if not FileUtil.validate_file_path(file_path):
continue
valid_coverage_slice = True
gdal_file = GDALGmlUtil(file.get_filepath())
try:
subsets = GdalAxisFiller(crs_axes, gdal_file).fill(True)
subsets = self._fill_time_axis(tpair, subsets)
except Exception as ex:
# If skip: true then just ignore this file from importing, else raise exception
FileUtil.ignore_coverage_slice_from_file_if_possible(
file_path, ex)
valid_coverage_slice = False
if valid_coverage_slice:
# Generate local metadata string for current coverage slice
self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
self.recipe_type, tpair.file)
local_metadata = gdal_coverage_converter._generate_local_metadata(
subsets, self.evaluator_slice)
if self.session.import_overviews_only is False:
slices_dict["base"].append(
Slice(subsets, FileDataProvider(tpair.file),
local_metadata))
# Then, create slices for selected overviews from user
for overview_index in self.session.import_overviews:
subsets_overview = self.create_subsets_for_overview(
subsets, overview_index, gdal_file)
slices_dict[str(overview_index)].append(
Slice(subsets_overview, FileDataProvider(file),
local_metadata))
timer.print_elapsed_time()
count += 1
return slices_dict
def validate_base(self, ignore_no_files=False):
"""
Validates the configuration and the input files
:param bool ignore_no_files: if the extending recipe does not work with files, set this to true to skip
the validation check for no files (used in wcs_extract recipe).
"""
if self.session.get_wcs_service(
) is None or self.session.get_wcs_service() == "":
raise RecipeValidationException("No valid wcs endpoint provided")
if self.session.get_crs_resolver(
) is None or self.session.get_crs_resolver() == "":
raise RecipeValidationException("No valid crs resolver provided")
if self.session.get_coverage_id(
) is None or self.session.get_coverage_id() == "":
raise RecipeValidationException("No valid coverage id provided")
if not FileUtil.check_dir_writable(ConfigManager.tmp_directory):
raise RecipeValidationException(
"Cannot write to tmp directory '{}'".format(
ConfigManager.tmp_directory))
checked_files = []
for file in self.session.get_files():
if FileUtil.validate_file_path(file.get_filepath()):
checked_files.append(file)
if not ignore_no_files:
# If no input file is available, exit wcst_import.
FileUtil.validate_input_file_paths(checked_files)
self.session.files = checked_files
if 'wms_import' not in self.options:
self.options['wms_import'] = False
else:
self.options['wms_import'] = bool(self.options['wms_import'])
if 'tiling' not in self.options:
self.options['tiling'] = None
if 'scale_levels' not in self.options:
self.options['scale_levels'] = None
if "import_order" in self.options:
if self.options['import_order'] != AbstractToCoverageConverter.IMPORT_ORDER_ASCENDING \
and self.options['import_order'] != AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING:
error_message = "'import_order' option must be '{}' or '{}', given '{}'.".\
format(AbstractToCoverageConverter.IMPORT_ORDER_ASCENDING,
AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING,
self.options['import_order'])
raise RecipeValidationException(error_message)
else:
self.options['import_order'] = None
def _get_convertors(self):
"""
Returns a map of coverage id -> GdalToCoverageConverter
"""
convertors = {}
band_data_type = self.DEFAULT_BAND_DATA_TYPE
if self.product == self.SLC_PRODUCT:
band_data_type = self.SLC_BAND_DATA_TYPE
for file in self.session.get_files():
# Check if this file still exists when preparing to import
if not FileUtil.validate_file_path(file.get_filepath()):
continue
# Check if this file belongs to this coverage id
modebeam, polarisation = self._get_modebeam_polarisation(
file.filepath)
cov_id = self._get_coverage_id(self.coverage_id, modebeam,
polarisation)
conv = self._get_convertor(convertors, cov_id)
file_pair = FilePair(file.filepath, file.filepath)
conv.files = [file_pair]
crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id)
# Different file contains different datetime from its name
evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
GdalToCoverageConverter.RECIPE_TYPE, file)
conv.data_type = band_data_type
slices_dict = conv._create_coverage_slices(crs_axes,
evaluator_slice)
slices_dict = self.__filter_invalid_geo_bounds(slices_dict)
if conv.coverage_slices == {}:
conv.coverage_slices = slices_dict
else:
for key, val in slices_dict.items():
conv.coverage_slices[key] += slices_dict[key]
if len(conv.coverage_slices) > 0 and len(
conv.coverage_slices["base"]) > 0:
first_slice = conv.coverage_slices["base"][0]
# This needs one available file to extract metadata later
conv.files = [first_slice.data_provider.file]
return convertors
def collect_local_metadata_file(self, local_metadata, sentence):
"""
Collect local metadata from external metadata files if it is set in the ingredient
:param dict local_metadata
"""
if "root_element" not in sentence or "path" not in sentence:
raise RuntimeError("'metadata_file' setting must contain one 'root_element'"
" and one 'path' children settings in the ingredient file")
root_element = str(self.evaluator.evaluate(sentence["root_element"], self.metadata_entry.evalutor_slice))
metadata_file = str(self.evaluator.evaluate(sentence["path"], self.metadata_entry.evalutor_slice))
if not FileUtil.validate_file_path(metadata_file):
raise RuntimeError("Cannot access local metadata file '" + metadata_file + "' to read")
else:
local_metadata[root_element] = XMLUtil.read_file_and_remove_xml_header(metadata_file)
def _create_coverage_slices(self, crs_axes, calculated_evaluator_slice=None, axis_resolutions=None):
"""
Returns all the coverage slices for this coverage
:param crs_axes:
:rtype: list[Slice]
"""
slices = []
count = 1
for file in self.files:
# NOTE: don't process any previously imported file (recorded in *.resume.json)
if not self.resumer.is_file_imported(file.filepath):
timer = Timer()
FileUtil.print_feedback(count, len(self.files), file.filepath)
# print which file is analyzing
if not FileUtil.validate_file_path(file.filepath):
continue
valid_coverage_slice = True
evaluator_slice = None
try:
if calculated_evaluator_slice is None:
# get the evaluator for the current recipe_type (each recipe has different evaluator)
evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(self.recipe_type, file)
else:
evaluator_slice = calculated_evaluator_slice
if self.data_type is None:
self.data_type = evaluator_slice.get_data_type(self)
coverage_slice = self._create_coverage_slice(file, crs_axes, evaluator_slice, axis_resolutions)
except Exception as ex:
# If skip: true then just ignore this file from importing, else raise exception
FileUtil.ignore_coverage_slice_from_file_if_possible(file.get_filepath(), ex)
valid_coverage_slice = False
if valid_coverage_slice:
slices.append(coverage_slice)
timer.print_elapsed_time()
count += 1
# Currently, only sort by datetime to import coverage slices (default is ascending)
reverse = (self.import_order == self.IMPORT_ORDER_DESCENDING)
return sort_slices_by_datetime(slices, reverse)
def _get_coverage_slices(self, crs, gdal_coverage_converter):
"""
Returns the slices for the collection of files given
"""
crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id)
slices = []
timeseries = self._generate_timeseries_tuples()
count = 1
for tpair in timeseries:
file_path = tpair.file.get_filepath()
# NOTE: don't process any imported file from *.resume.json as it is just waisted time
if not self.resumer.is_file_imported(file_path):
timer = Timer()
# print which file is analyzing
FileUtil.print_feedback(count, len(timeseries), file_path)
if not FileUtil.validate_file_path(file_path):
continue
valid_coverage_slice = True
try:
subsets = GdalAxisFiller(crs_axes,
GDALGmlUtil(file_path)).fill(True)
subsets = self._fill_time_axis(tpair, subsets)
except Exception as ex:
# If skip: true then just ignore this file from importing, else raise exception
FileUtil.ignore_coverage_slice_from_file_if_possible(
file_path, ex)
valid_coverage_slice = False
if valid_coverage_slice:
# Generate local metadata string for current coverage slice
self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
self.recipe_type, tpair.file)
local_metadata = gdal_coverage_converter._generate_local_metadata(
subsets, self.evaluator_slice)
slices.append(
Slice(subsets, FileDataProvider(tpair.file),
local_metadata))
timer.print_elapsed_time()
count += 1
return slices
def _get_convertors(self):
"""
Returns a map of coverage id -> GdalToCoverageConverter
"""
convertors = {}
band_data_type = self.DEFAULT_BAND_DATA_TYPE
if self.product == self.SLC_PRODUCT:
band_data_type = self.SLC_BAND_DATA_TYPE
for file in self.session.get_files():
# Check if this file still exists when preparing to import
if not FileUtil.validate_file_path(file.get_filepath()):
continue
# Check if this file belongs to this coverage id
modebeam, polarisation = self._get_modebeam_polarisation(
file.filepath)
cov_id = self._get_coverage_id(self.coverage_id, modebeam,
polarisation)
# This file already imported in coverage_id.resume.json
self.resumer = Resumer(cov_id)
if self.resumer.is_file_imported(file.filepath):
continue
conv = self._get_convertor(convertors, cov_id)
file_pair = FilePair(file.filepath, file.filepath)
conv.files = [file_pair]
crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id)
# Different file contains different datetime from its name
evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
GdalToCoverageConverter.RECIPE_TYPE, file)
conv.data_type = band_data_type
slices = conv._create_coverage_slices(crs_axes, evaluator_slice)
conv.coverage_slices += slices
return convertors
def _get_convertors(self):
"""
Returns a map of coverage id -> GdalToCoverageConverter
"""
convertors = {}
for f in self.session.get_files():
# This one does not contain any information for geo bounds
if not FileUtil.validate_file_path(f.get_filepath()):
continue
gdal_ds = GDALGmlUtil(f.get_filepath())
subdatasets = self._get_subdatasets(gdal_ds, f)
gdal_ds.close()
level = self._get_level(f.get_filepath())
if len(self.levels) > 0 and level not in self.levels:
# skip file, as it's not in the list of levels provided in the ingredients file
log.debug("Skipping " + level + " data")
continue
crs_code = ""
evaluator_slice = None
for res in self.resolutions:
subds_file = self._get_subdataset_file(subdatasets, res)
crs_code = self._get_crs_code(subds_file.get_filepath(),
crs_code)
if len(self.crss) > 0 and crs_code not in self.crss:
# skip CRS, it's not in the list of CRSs provided in the ingredients file
log.debug("Skipping data with CRS " + crs_code)
continue
cov_id = self._get_coverage_id(self.coverage_id, crs_code,
level, res)
conv = self._get_convertor(convertors, cov_id, crs_code, level,
res)
file_pair = FilePair(subds_file.filepath, f.filepath)
conv.files = [file_pair]
crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id)
if evaluator_slice is None:
# This one contains information for geo bounds
evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
GdalToCoverageConverter.RECIPE_TYPE, subds_file)
# Resolution 10m, 20m and 60m have same data type (UInt16) while TCI has data type (Byte)
if res == self.RES_TCI:
conv.data_type = "Byte"
else:
conv.data_type = "UInt16"
# Fixed values for 3 axes of Sentinel 2 coverage
axis_resolutions = self.RES_DICT[res]
slices_dict = conv._create_coverage_slices(
crs_axes, evaluator_slice, axis_resolutions)
if conv.coverage_slices == {}:
conv.coverage_slices = slices_dict
else:
for key, val in slices_dict.items():
conv.coverage_slices[key] += slices_dict[key]
if len(conv.coverage_slices) != 0:
first_slice = conv.coverage_slices["base"][0]
# This needs one available file to extract metadata later
conv.files = [first_slice.data_provider.file]
return convertors
def validate_base(self, ignore_no_files=False):
"""
Validates the configuration and the input files
:param bool ignore_no_files: if the extending recipe does not work with files, set this to true to skip
the validation check for no files (used in wcs_extract recipe).
"""
if self.session.get_wcs_service(
) is None or self.session.get_wcs_service() == "":
raise RecipeValidationException("No valid wcs endpoint provided")
if self.session.get_crs_resolver(
) is None or self.session.get_crs_resolver() == "":
raise RecipeValidationException("No valid crs resolver provided")
if self.session.get_coverage_id(
) is None or self.session.get_coverage_id() == "":
raise RecipeValidationException("No valid coverage id provided")
import recipes.virtual_coverage.recipe as super_coverage
if self.session.get_recipe_name() == super_coverage.Recipe.RECIPE_NAME:
# NOTE: virtual_coverage recipe does not require any input files
return
if not FileUtil.check_dir_writable(ConfigManager.tmp_directory):
raise RecipeValidationException(
"Cannot write to tmp directory '{}'".format(
ConfigManager.tmp_directory))
checked_files = []
for file in self.session.get_files():
if FileUtil.validate_file_path(file.get_filepath()):
checked_files.append(file)
if not ignore_no_files:
# If no input file is available, exit wcst_import.
FileUtil.validate_input_file_paths(checked_files)
self.session.files = checked_files
if 'wms_import' not in self.options:
self.options['wms_import'] = False
else:
self.options['wms_import'] = bool(self.options['wms_import'])
if 'tiling' not in self.options:
self.options['tiling'] = None
if 'scale_levels' not in self.options:
self.options['scale_levels'] = None
if 'scale_factors' not in self.options:
self.options['scale_factors'] = None
if self.options['scale_levels'] is not None \
and self.options['scale_factors'] is not None:
raise RecipeValidationException(
"Only one of 'scale_levels' or 'scale_factors' "
"setting can exist in the ingredients file.")
if self.options['scale_factors'] is not None:
# as scale_factors and scale_levels are only valid when initializing a new coverage
cov = CoverageUtil(self.session.get_coverage_id())
if not cov.exists():
for obj in self.options['scale_factors']:
if 'coverage_id' not in obj or 'factors' not in obj:
raise RecipeValidationException(
"All elements of 'scale_factors' list must contain "
"'coverage_id' and 'factors' properties")
coverage_id = obj['coverage_id']
cov = CoverageUtil(coverage_id)
if cov.exists():
raise RecipeValidationException(
"Downscaled level coverage '" + coverage_id +
"' already exists, "
"please use a different 'coverage_id' in 'scale_factors' list"
)
self.validate_pyramid_members()
self.validate_pyramid_bases()
if "import_order" in self.options:
if self.options['import_order'] != AbstractToCoverageConverter.IMPORT_ORDER_ASCENDING \
and self.options['import_order'] != AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING:
error_message = "'import_order' option must be '{}' or '{}', given '{}'.".\
format(AbstractToCoverageConverter.IMPORT_ORDER_ASCENDING,
AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING,
self.options['import_order'])
raise RecipeValidationException(error_message)
else:
self.options['import_order'] = None
def _create_coverage_slices(self,
crs_axes,
calculated_evaluator_slice=None,
axis_resolutions=None):
"""
Returns the slices for the collection of files given
:param crs_axes:
:rtype: list[Slice]
"""
from master.recipe.base_recipe import BaseRecipe
slices_dict = BaseRecipe.create_dict_of_slices(
self.session.import_overviews)
count = 1
for file in self.files:
timer = Timer()
FileUtil.print_feedback(count, len(self.files), file.filepath)
# print which file is analyzing
if not FileUtil.validate_file_path(file.filepath):
continue
valid_coverage_slice = True
try:
if calculated_evaluator_slice is None:
# get the evaluator for the current recipe_type (each recipe has different evaluator)
self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
self.recipe_type, file)
else:
self.evaluator_slice = calculated_evaluator_slice
if self.data_type is None:
self.data_type = self.evaluator_slice.get_data_type(self)
coverage_slice = self._create_coverage_slice(
file, crs_axes, self.evaluator_slice, axis_resolutions)
except Exception as ex:
# If skip: true then just ignore this file from importing, else raise exception
FileUtil.ignore_coverage_slice_from_file_if_possible(
file.get_filepath(), ex)
valid_coverage_slice = False
if valid_coverage_slice:
if self.session.import_overviews_only is False:
slices_dict["base"].append(coverage_slice)
if self.session.recipe["options"]["coverage"]["slicer"][
"type"] == "gdal":
gdal_file = GDALGmlUtil(file.get_filepath())
# Then, create slices for selected overviews from user
for overview_index in self.session.import_overviews:
axis_subsets_overview = BaseRecipe.create_subsets_for_overview(
coverage_slice.axis_subsets, overview_index,
gdal_file)
coverage_slice_overview = copy.deepcopy(coverage_slice)
coverage_slice_overview.axis_subsets = axis_subsets_overview
slices_dict[str(overview_index)].append(
coverage_slice_overview)
timer.print_elapsed_time()
count += 1
# Currently, only sort by datetime to import coverage slices (default is ascending)
reverse = (self.import_order == self.IMPORT_ORDER_DESCENDING)
for key, value in slices_dict.items():
slices_dict[key] = sort_slices_by_datetime(value, reverse)
return slices_dict
