def _generate_timeseries_tuples(self, limit=None):
"""
Generate the timeseries tuples from the original files based on the recipe
:rtype: list[TimeFileTuple]
"""
ret = []
if limit is None:
limit = len(self.session.get_files())
time_format = None
if 'datetime_format' in self.options['time_parameter']:
time_format = self.options['time_parameter']['datetime_format']
if 'metadata_tag' in self.options['time_parameter']:
mtag = self.options['time_parameter']['metadata_tag']['tag_name']
for tfile in self.session.get_files():
if len(ret) == limit:
break
gdal_file = GDALGmlUtil(tfile.get_filepath())
dtutil = DateTimeUtil(gdal_file.get_datetime(mtag), time_format, self.options['time_crs'])
ret.append(TimeFileTuple(dtutil, tfile))
elif 'filename' in self.options['time_parameter'] and len(ret) < limit:
regex = self.options['time_parameter']['filename']['regex']
group = int(self.options['time_parameter']['filename']['group'])
for tfile in self.session.get_files():
if len(ret) == limit:
break
dtutil = DateTimeUtil(re.search(regex, tfile.filepath).group(group),
time_format, self.options['time_crs'])
ret.append(TimeFileTuple(dtutil, tfile))
else:
raise RecipeValidationException("No method to get the time parameter, you should either choose "
"metadata_tag or filename.")
return sorted(ret)
def _get_slices(self, crs):
# Let's first extract all the axes from our crs
crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id)
# Prepare a list container for our slices
slices = []
# Iterate over the files and create a slice for each one
for infile in self.session.get_files():
# We need to create the exact position in time and space in which to place this slice
# For the space coordinates we can use the GDAL helper to extract it for us
# The helper will return a list of subsets based on the crs axes that we extracted
# and will fill the coordinates for the ones that it can (the easting and northing axes)
subsets = GdalAxisFiller(crs_axes,
GDALGmlUtil(
infile.get_filepath())).fill()
# Now we must fill the time axis as well and indicate the position in time
for subset in subsets:
# Find the time axis
if subset.coverage_axis.axis.crs_axis.is_time_axis():
# Set the time position for it. Our recipe extracts it from a GDAL tag provided by the user
# datetime format needs enquoted (e.g: "2015-01")
subset.interval.low = '"' + GDALGmlUtil(
infile).get_datetime(self.options["time_tag"]) + '"'
slices.append(Slice(subsets, FileDataProvider(infile)))
return slices
def _get_coverage(self):
"""
Returns the coverage to be used for the importer
"""
gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath())
slices = self._get_slices(gdal_dataset)
fields = GdalRangeFieldsGenerator(gdal_dataset, self.options['band_names']).get_range_fields()
coverage = Coverage(self.session.get_coverage_id(), slices, fields, gdal_dataset.get_crs(),
gdal_dataset.get_band_gdal_type(), self.options['tiling'])
return coverage
def _generate_timeseries_tuples(self, limit=None):
"""
Generate the timeseries tuples from the original files based on the recipe.
And sort the files in order of time.
:rtype: list[TimeFileTuple]
"""
ret = []
if limit is None:
limit = len(self.session.get_files())
time_format = None
if 'datetime_format' in self.options['time_parameter']:
time_format = self.options['time_parameter']['datetime_format']
if 'metadata_tag' in self.options['time_parameter']:
mtag = self.options['time_parameter']['metadata_tag']['tag_name']
for tfile in self.session.get_files():
if len(ret) == limit:
break
valid_file = True
try:
gdal_file = GDALGmlUtil(tfile.get_filepath())
except Exception as ex:
FileUtil.ignore_coverage_slice_from_file_if_possible(
tfile.get_filepath(), ex)
valid_file = False
if valid_file:
dtutil = DateTimeUtil(gdal_file.get_datetime(mtag),
time_format,
self.options['time_crs'])
ret.append(TimeFileTuple(dtutil, tfile))
elif 'filename' in self.options['time_parameter'] and len(ret) < limit:
regex = self.options['time_parameter']['filename']['regex']
group = int(self.options['time_parameter']['filename']['group'])
for tfile in self.session.get_files():
if len(ret) == limit:
break
dtutil = DateTimeUtil(
re.search(regex, tfile.filepath).group(group), time_format,
self.options['time_crs'])
ret.append(TimeFileTuple(dtutil, tfile))
else:
raise RecipeValidationException(
"No method to get the time parameter, you should either choose "
"metadata_tag or filename.")
# Currently, only sort by datetime to import coverage slices (default is ascending), option: to sort descending
if self.options[
"import_order"] == AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING:
return sorted(ret, reverse=True)
return sorted(ret)
def _get_coverage(self):
# Get the crs of one of the images using a GDAL helper class. We are assuming all images have the same CRS
gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath())
# Get the crs of the coverage by compounding the two crses
crs = CRSUtil.get_compound_crs(
[gdal_dataset.get_crs(), self.options['time_crs']])
fields = GdalRangeFieldsGenerator(gdal_dataset).get_range_fields()
pixel_type = gdal_dataset.get_band_gdal_type()
coverage_id = self.session.get_coverage_id()
slices = self._get_slices(crs)
return Coverage(coverage_id, slices, fields, crs, pixel_type)
def parse_gdal_global_metadata(file_path):
"""
Parse the first file of importing gdal files to extract the global metadata for the coverage
str file_path: path to first gdal input file
:return: dict: global_metadata
"""
# NOTE: all files should have same global metadata for each file
gdal_dataset = GDALGmlUtil(file_path)
global_metadata = gdal_dataset.get_metadata()
return global_metadata
def _get_coverage(self):
"""
Returns the coverage to be used for the importer
"""
gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath())
slices = self._get_slices(gdal_dataset)
fields = GdalRangeFieldsGenerator(
gdal_dataset, self.options['band_names']).get_range_fields()
coverage = Coverage(self.session.get_coverage_id(), slices, fields,
gdal_dataset.get_crs(),
gdal_dataset.get_band_gdal_type(),
self.options['tiling'])
return coverage
def _init_epsg_xy_crs(self):
"""
From the first file of input file, detect its EPSG code for XY axes
"""
for file in self.session.get_files():
try:
gdal_ds = GDALGmlUtil(file.get_filepath())
self.epsg_xy_crs = gdal_ds.get_crs()
break
except Exception as e:
if ConfigManager.skip == True:
pass
else:
raise e
def _resolve(self, expression, gdal_dataset):
"""
Resolves the expression in the context of the gdal dataset
:param str expression: the expression to resolve
:param util.gdal_util.GDALGmlUtil gdal_dataset: the dataset for which to resolve
:return:
"""
if expression.startswith("metadata:"):
value = gdal_dataset.get_metadata_tag(
expression.replace("metadata:", ""))
else:
user_bands = []
for band in GDALGmlUtil.get_fields_range_type(gdal_dataset):
user_band = UserBand(band.field_name, "", "", "", "",
band.nill_values, band.uom_code)
user_bands.append(user_band)
gdal_dictionary = {
"resolutionX": gdal_dataset.get_offset_vectors()[0],
"resolutionY": gdal_dataset.get_offset_vectors()[1],
"originX": gdal_dataset.get_origin_x(),
"originY": gdal_dataset.get_origin_y(),
"minX": gdal_dataset.get_extents_x()[0],
"maxX": gdal_dataset.get_extents_x()[1],
"minY": gdal_dataset.get_extents_y()[0],
"maxY": gdal_dataset.get_extents_y()[1],
"bands": user_bands
}
if expression in gdal_dictionary:
value = gdal_dictionary[expression]
else:
raise RuntimeException(
"Cannot evaluate the given grib expression {} on the given container."
.format(str(expression)))
return value
def _get_slices(self, crs):
"""
Returns the slices for the collection of files given
"""
files = self.session.get_files()
crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id)
slices = []
count = 1
for file in files:
# NOTE: don't process any imported file from *.resume.json as it is just waisted time
if not self.resumer.is_file_imported(file.filepath):
timer = Timer()
# print which file is analyzing
FileUtil.print_feedback(count, len(files), file.filepath)
if not FileUtil.validate_file_path(file.filepath):
continue
valid_coverage_slice = True
try:
subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill()
except Exception as ex:
# If skip: true then just ignore this file from importing, else raise exception
FileUtil.ignore_coverage_slice_from_file_if_possible(file.get_filepath(), ex)
valid_coverage_slice = False
if valid_coverage_slice:
slices.append(Slice(subsets, FileDataProvider(file)))
timer.print_elapsed_time()
count += 1
return slices
def _get_coverage(self):
"""
Returns the coverage to be used for the importer
"""
gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(self.session.get_files())
crs = gdal_dataset.get_crs()
general_recipe = GeneralRecipe(self.session)
global_metadata_fields = general_recipe._global_metadata_fields()
local_metadata_fields = general_recipe._local_metadata_fields()
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
gdal_coverage_converter = GdalToCoverageConverter(self.resumer, self.session.get_default_null_values(),
self.recipe_type, sentence_evaluator,
self.session.get_coverage_id(),
None, self.session.get_files(),
crs, None, None,
global_metadata_fields, local_metadata_fields,
None, None, general_recipe._metadata_type(),
None, None)
coverage_slices = self._get_coverage_slices(crs, gdal_coverage_converter)
fields = GdalRangeFieldsGenerator(gdal_dataset, self.options['band_names']).get_range_fields()
global_metadata = None
if len(coverage_slices) > 0:
global_metadata = gdal_coverage_converter._generate_global_metadata(coverage_slices[0], self.evaluator_slice)
coverage = Coverage(self.session.get_coverage_id(), coverage_slices, fields, gdal_dataset.get_crs(),
gdal_dataset.get_band_gdal_type(), self.options['tiling'], global_metadata)
return coverage
def _get_coverage_slices(self, crs, gdal_coverage_converter):
"""
Returns the slices for the collection of files given
"""
crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id)
slices_dict = self.create_dict_of_slices(self.session.import_overviews)
timeseries = self._generate_timeseries_tuples()
count = 1
for tpair in timeseries:
file = tpair.file
file_path = tpair.file.get_filepath()
timer = Timer()
# print which file is analyzing
FileUtil.print_feedback(count, len(timeseries), file_path)
if not FileUtil.validate_file_path(file_path):
continue
valid_coverage_slice = True
gdal_file = GDALGmlUtil(file.get_filepath())
try:
subsets = GdalAxisFiller(crs_axes, gdal_file).fill(True)
subsets = self._fill_time_axis(tpair, subsets)
except Exception as ex:
# If skip: true then just ignore this file from importing, else raise exception
FileUtil.ignore_coverage_slice_from_file_if_possible(
file_path, ex)
valid_coverage_slice = False
if valid_coverage_slice:
# Generate local metadata string for current coverage slice
self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
self.recipe_type, tpair.file)
local_metadata = gdal_coverage_converter._generate_local_metadata(
subsets, self.evaluator_slice)
if self.session.import_overviews_only is False:
slices_dict["base"].append(
Slice(subsets, FileDataProvider(tpair.file),
local_metadata))
# Then, create slices for selected overviews from user
for overview_index in self.session.import_overviews:
subsets_overview = self.create_subsets_for_overview(
subsets, overview_index, gdal_file)
slices_dict[str(overview_index)].append(
Slice(subsets_overview, FileDataProvider(file),
local_metadata))
timer.print_elapsed_time()
count += 1
return slices_dict
def _get_coverages(self):
"""
Returns the list of coverages to be used for the importer
"""
gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(
self.session.get_files())
crs = CRSUtil.get_compound_crs(
[self.options['time_crs'],
gdal_dataset.get_crs()])
general_recipe = GeneralRecipe(self.session)
global_metadata_fields = general_recipe._global_metadata_fields()
local_metadata_fields = general_recipe._local_metadata_fields()
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
gdal_coverage_converter = GdalToCoverageConverter(
self.resumer, self.session.get_default_null_values(),
self.recipe_type,
sentence_evaluator, self.session.get_coverage_id(), None,
self.session.get_files(), crs, None, None,
global_metadata_fields, local_metadata_fields, None, None,
general_recipe._metadata_type(), None, None, self.session)
coverage_slices_dict = self._get_coverage_slices(
crs, gdal_coverage_converter)
fields = GdalRangeFieldsGenerator(
gdal_dataset, self.options['band_names']).get_range_fields()
global_metadata = None
if len(coverage_slices_dict["base"]) > 0:
global_metadata = gdal_coverage_converter._generate_global_metadata(
coverage_slices_dict["base"][0], self.evaluator_slice)
results = []
base_coverage_id = self.session.get_coverage_id()
for key, value in coverage_slices_dict.items():
if key == "base":
# base coverage
coverage = Coverage(base_coverage_id,
coverage_slices_dict[key], fields, crs,
gdal_dataset.get_band_gdal_type(),
self.options['tiling'], global_metadata)
else:
# overview coverage (key = overview_index)
coverage_id = create_coverage_id_for_overview(
base_coverage_id, key)
coverage = Coverage(coverage_id, coverage_slices_dict[key],
fields, crs,
gdal_dataset.get_band_gdal_type(),
self.options['tiling'], global_metadata,
base_coverage_id, key)
results.append(coverage)
return results
def _get_coverage(self):
# Get the crs of one of the images using a GDAL helper class. We are assuming all images have the same CRS
for file in self.session.get_files():
try:
file_path = file.get_filepath()
gdal_dataset = GDALGmlUtil(file_path)
break
except Exception as e:
if ConfigManager.skip == True:
pass
else:
raise e
# Get the crs of the coverage by compounding the two crses
crs = CRSUtil.get_compound_crs([gdal_dataset.get_crs(), self.options['time_crs']])
fields = GdalRangeFieldsGenerator(gdal_dataset).get_range_fields()
pixel_type = gdal_dataset.get_band_gdal_type()
coverage_id = self.session.get_coverage_id()
slices = self._get_slices(crs)
return Coverage(coverage_id, slices, fields, crs, pixel_type)
def get_dataset(self):
"""
Returns the dataset of the file
NOTE: gdal cannot open too many files (1989 files with error too many file opens)
when getting dataset object and store inside list, so only open dataset when it is needed.
:rtype: gdal Dataset
"""
if self.dataset is None:
self.dataset = GDALGmlUtil(self.get_file().filepath)
return self.dataset
def _data_type(self):
"""
Returns the data type for this netcdf dataset
:rtype: str
"""
if len(self.files) < 1:
raise RuntimeException("No files to import were specified.")
netCDF4 = import_netcdf4()
nci = netCDF4.Dataset(self.files[0].get_filepath(), 'r')
netcdf_data_type = nci.variables[self.bands[0].identifier].dtype.name
return GDALGmlUtil.data_type_to_gdal_type(netcdf_data_type)
def _get_data_type(self, slice):
"""
Returns the data type of the slice by downloading the slice and trying to guess it with GDAL
:param Slice slice: slice
:rtype: str
"""
if isinstance(slice.data_provider, UrlDataProvider):
# Do this only for coverages that have more than one axis
if len(slice.axis_subsets) > 1:
fu = FileUtil()
contents = validate_and_read_url(slice.data_provider.get_url())
file_path = fu.write_to_tmp_file(contents, "tif")
return GDALGmlUtil(file_path).get_band_gdal_type()
return None
def _get_coverage(self):
"""
Returns the coverage to be used for the importer
"""
gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(
self.session.get_files())
crs = CRSUtil.get_compound_crs(
[self.options['time_crs'],
gdal_dataset.get_crs()])
slices = self._get_slices(crs)
fields = GdalRangeFieldsGenerator(
gdal_dataset, self.options['band_names']).get_range_fields()
coverage = Coverage(self.session.get_coverage_id(), slices, fields,
crs, gdal_dataset.get_band_gdal_type(),
self.options['tiling'])
return coverage
def _get_coverage_slices(self, crs, gdal_coverage_converter):
"""
Returns the slices for the collection of files given
"""
crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id)
slices = []
timeseries = self._generate_timeseries_tuples()
count = 1
for tpair in timeseries:
file_path = tpair.file.get_filepath()
# NOTE: don't process any imported file from *.resume.json as it is just waisted time
if not self.resumer.is_file_imported(file_path):
timer = Timer()
# print which file is analyzing
FileUtil.print_feedback(count, len(timeseries), file_path)
if not FileUtil.validate_file_path(file_path):
continue
valid_coverage_slice = True
try:
subsets = GdalAxisFiller(crs_axes,
GDALGmlUtil(file_path)).fill(True)
subsets = self._fill_time_axis(tpair, subsets)
except Exception as ex:
# If skip: true then just ignore this file from importing, else raise exception
FileUtil.ignore_coverage_slice_from_file_if_possible(
file_path, ex)
valid_coverage_slice = False
if valid_coverage_slice:
# Generate local metadata string for current coverage slice
self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
self.recipe_type, tpair.file)
local_metadata = gdal_coverage_converter._generate_local_metadata(
subsets, self.evaluator_slice)
slices.append(
Slice(subsets, FileDataProvider(tpair.file),
local_metadata))
timer.print_elapsed_time()
count += 1
return slices
def get_data_type(self, netcdf_recipe_converter):
"""
Return the data type of opened dataset
:param: AbstractToCoverageConverter netcdf_recipe: Converter for netCDF recipe
:return: str data_type
"""
collected_data_type = []
for band in netcdf_recipe_converter.bands:
band_id = band.identifier
netcdf_data_type = self.get_dataset().variables[band_id].dtype.name
band_data_type = GDALGmlUtil.data_type_to_gdal_type(
netcdf_data_type)
collected_data_type.append(band_data_type)
data_type = ",".join(collected_data_type)
return data_type
def _get_slices(self, gdal_dataset):
"""
Returns the slices for the collection of files given
"""
files = self.session.get_files()
crs = gdal_dataset.get_crs()
crs_axes = CRSUtil(crs).get_axes()
slices = []
count = 1
for file in files:
# print which file is analyzing
FileUtil.print_feedback(count, len(files), file.filepath)
subsets = GdalAxisFiller(crs_axes,
GDALGmlUtil(file.get_filepath())).fill()
slices.append(Slice(subsets, FileDataProvider(file)))
count += 1
return slices
def _get_slices(self, crs):
"""
Returns the slices for the collection of files given
"""
crs_axes = CRSUtil(crs).get_axes()
slices = []
timeseries = self._generate_timeseries_tuples()
count = 1
for tpair in timeseries:
# print which file is analyzing
FileUtil.print_feedback(count, len(timeseries),
tpair.file.filepath)
subsets = GdalAxisFiller(crs_axes,
GDALGmlUtil(
tpair.file.get_filepath())).fill()
subsets = self._fill_time_axis(tpair, subsets)
slices.append(Slice(subsets, FileDataProvider(tpair.file)))
count += 1
return slices
def _file_band_nil_values(self, index):
"""
This is used to get the null values (Only 1) from the given band index if one exists when nilValue was not defined
in ingredient file
:param integer index: the current band index to get the nilValues
:rtype: List[RangeTypeNilValue] with only 1 element
"""
if len(self.files) < 1:
raise RuntimeException("No gdal files given for import!")
if len(self.default_null_values) > 0:
return self.default_null_values
# NOTE: all files should have same bands's metadata, so 1 file is ok
gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(self.files)
# band in gdal starts with 1
gdal_band = gdal_dataset.get_raster_band(index + 1)
nil_value = gdal_band.GetNoDataValue()
if nil_value is None:
return None
else:
return [nil_value]
def __add_color_palette_table_to_global_metadata(self, metadata_dict, file_path):
"""
If colorPaletteTable is added in ingredient file, then add it to coverage's global metadata
"""
supported_recipe = (self.options['coverage']['slicer']['type'] == "gdal")
color_palette_table = None
if "metadata" in self.options['coverage']:
if "colorPaletteTable" in self.options['coverage']['metadata']:
value = self.options['coverage']['metadata']['colorPaletteTable']
if value.strip() != "":
if value == "auto" and not supported_recipe:
raise RecipeValidationException("colorPaletteTable auto is only supported"
" in general recipe with slicer's type: gdal.")
elif value == "auto":
# Get colorPaletteTable automatically from first file
gdal_dataset = GDALGmlUtil(file_path)
color_palette_table = gdal_dataset.get_color_table()
else:
# file_path can be relative path or full path
file_paths = FileUtil.get_file_paths_by_regex(self.session.get_ingredients_dir_path(), value)
if len(file_paths) == 0:
raise RecipeValidationException(
"Color palette table file does not exist, given: '" + value + "'.")
else:
file_path = file_paths[0]
# Add the content of colorPaletteTable to coverage's metadata
with open(file_path, 'r') as file_reader:
color_palette_table = file_reader.read()
elif supported_recipe:
# If colorPaletteTable is not mentioned in the ingredient, automatically fetch it
gdal_dataset = GDALGmlUtil(file_path)
color_palette_table = gdal_dataset.get_color_table()
if color_palette_table is not None:
metadata_dict["colorPaletteTable"] = color_palette_table
def _create_coverage_slices(self,
crs_axes,
calculated_evaluator_slice=None,
axis_resolutions=None):
"""
Returns the slices for the collection of files given
:param crs_axes:
:rtype: list[Slice]
"""
from master.recipe.base_recipe import BaseRecipe
slices_dict = BaseRecipe.create_dict_of_slices(
self.session.import_overviews)
count = 1
for file in self.files:
timer = Timer()
FileUtil.print_feedback(count, len(self.files), file.filepath)
# print which file is analyzing
if not FileUtil.validate_file_path(file.filepath):
continue
valid_coverage_slice = True
try:
if calculated_evaluator_slice is None:
# get the evaluator for the current recipe_type (each recipe has different evaluator)
self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
self.recipe_type, file)
else:
self.evaluator_slice = calculated_evaluator_slice
if self.data_type is None:
self.data_type = self.evaluator_slice.get_data_type(self)
coverage_slice = self._create_coverage_slice(
file, crs_axes, self.evaluator_slice, axis_resolutions)
except Exception as ex:
# If skip: true then just ignore this file from importing, else raise exception
FileUtil.ignore_coverage_slice_from_file_if_possible(
file.get_filepath(), ex)
valid_coverage_slice = False
if valid_coverage_slice:
if self.session.import_overviews_only is False:
slices_dict["base"].append(coverage_slice)
if self.session.recipe["options"]["coverage"]["slicer"][
"type"] == "gdal":
gdal_file = GDALGmlUtil(file.get_filepath())
# Then, create slices for selected overviews from user
for overview_index in self.session.import_overviews:
axis_subsets_overview = BaseRecipe.create_subsets_for_overview(
coverage_slice.axis_subsets, overview_index,
gdal_file)
coverage_slice_overview = copy.deepcopy(coverage_slice)
coverage_slice_overview.axis_subsets = axis_subsets_overview
slices_dict[str(overview_index)].append(
coverage_slice_overview)
timer.print_elapsed_time()
count += 1
# Currently, only sort by datetime to import coverage slices (default is ascending)
reverse = (self.import_order == self.IMPORT_ORDER_DESCENDING)
for key, value in slices_dict.items():
slices_dict[key] = sort_slices_by_datetime(value, reverse)
return slices_dict
def _init_epsg_xy_crs(self):
"""
From the first file of input file, detect its EPSG code for XY axes
"""
gdal_ds = GDALGmlUtil(self.session.get_files()[0].get_filepath())
self.epsg_xy_crs = gdal_ds.get_crs()
def _get_convertors(self):
"""
Returns a map of coverage id -> GdalToCoverageConverter
"""
convertors = {}
for f in self.session.get_files():
# This one does not contain any information for geo bounds
if not FileUtil.validate_file_path(f.get_filepath()):
continue
gdal_ds = GDALGmlUtil(f.get_filepath())
subdatasets = self._get_subdatasets(gdal_ds, f)
gdal_ds.close()
level = self._get_level(f.get_filepath())
if len(self.levels) > 0 and level not in self.levels:
# skip file, as it's not in the list of levels provided in the ingredients file
log.debug("Skipping " + level + " data")
continue
crs_code = ""
evaluator_slice = None
for res in self.resolutions:
subds_file = self._get_subdataset_file(subdatasets, res)
crs_code = self._get_crs_code(subds_file.get_filepath(),
crs_code)
if len(self.crss) > 0 and crs_code not in self.crss:
# skip CRS, it's not in the list of CRSs provided in the ingredients file
log.debug("Skipping data with CRS " + crs_code)
continue
cov_id = self._get_coverage_id(self.coverage_id, crs_code,
level, res)
conv = self._get_convertor(convertors, cov_id, crs_code, level,
res)
file_pair = FilePair(subds_file.filepath, f.filepath)
conv.files = [file_pair]
crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id)
if evaluator_slice is None:
# This one contains information for geo bounds
evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
GdalToCoverageConverter.RECIPE_TYPE, subds_file)
# Resolution 10m, 20m and 60m have same data type (UInt16) while TCI has data type (Byte)
if res == self.RES_TCI:
conv.data_type = "Byte"
else:
conv.data_type = "UInt16"
# Fixed values for 3 axes of Sentinel 2 coverage
axis_resolutions = self.RES_DICT[res]
slices_dict = conv._create_coverage_slices(
crs_axes, evaluator_slice, axis_resolutions)
if conv.coverage_slices == {}:
conv.coverage_slices = slices_dict
else:
for key, val in slices_dict.items():
conv.coverage_slices[key] += slices_dict[key]
if len(conv.coverage_slices) != 0:
first_slice = conv.coverage_slices["base"][0]
# This needs one available file to extract metadata later
conv.files = [first_slice.data_provider.file]
return convertors
def _axis_subset(self, crs_axis, gdal_file):
"""
Returns an axis subset using the given crs axis in the context of the gdal file
:param CRSAxis crs_axis: the crs definition of the axis
:param File gdal_file: the gdal file
:rtype AxisSubset
"""
user_axis = self._user_axis(
self._get_user_axis_by_crs_axis_name(crs_axis.label),
GDALEvaluatorSlice(GDALGmlUtil(gdal_file.get_filepath())))
high = user_axis.interval.high if user_axis.interval.high else user_axis.interval.low
if isinstance(user_axis, RegularUserAxis):
geo_axis = RegularAxis(crs_axis.label, crs_axis.uom,
user_axis.interval.low, high,
user_axis.interval.low, crs_axis)
else:
# if irregular axis value is fetched from fileName so the coefficient is [0] as slicing
if user_axis.directPositions == AbstractToCoverageConverter.DIRECT_POSITIONS_SLICING:
user_axis.directPositions = AbstractToCoverageConverter.COEFFICIENT_SLICING
geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom,
user_axis.interval.low, high,
user_axis.interval.low,
user_axis.directPositions, crs_axis)
if not crs_axis.is_easting() and not crs_axis.is_northing():
# GDAL model is 2D so on any axis except x/y we expect to have only one value
grid_low = 0
grid_high = 0
else:
grid_low = 0
number_of_grid_points = decimal.Decimal(str(user_axis.interval.high)) \
- decimal.Decimal(str(user_axis.interval.low))
# number_of_grid_points = (geo_max - geo_min) / resolution
grid_high = grid_low + number_of_grid_points / decimal.Decimal(
user_axis.resolution)
grid_high = HighPixelAjuster.adjust_high(grid_high)
# Negative axis, e.g: Latitude (min <--- max)
if user_axis.resolution < 0:
grid_high = int(abs(math.floor(grid_high)))
else:
# Positive axis, e.g: Longitude (min --> max)
grid_high = int(abs(math.ceil(grid_high)))
# NOTE: Grid Coverage uses the direct intervals as in Rasdaman
if self.grid_coverage is False:
if grid_high > grid_low:
grid_high -= 1
grid_axis = GridAxis(user_axis.order, crs_axis.label,
user_axis.resolution, grid_low, grid_high)
geo_axis.origin = PointPixelAdjuster.get_origin(user_axis, crs_axis)
if user_axis.type == UserAxisType.DATE:
self._translate_decimal_to_datetime(user_axis, geo_axis)
# NOTE: current, gdal recipe supports only has 2 axes which are "bounded" (i.e: they exist as 2D axes in file)
# and 1 or more another axes gotten (i.e: from fileName) which are not "bounded" to create 3D+ coverage.
data_bound = crs_axis.is_northing() or crs_axis.is_easting()
return AxisSubset(
CoverageAxis(geo_axis, grid_axis, data_bound),
Interval(user_axis.interval.low, user_axis.interval.high))
def _read_bands(self):
"""
Returns a user band extracted from the ingredients if specified (required for netCDF/GRIB)
:rtype: list[UserBand]
"""
if "bands" in self.options['coverage']['slicer']:
bands = self.options['coverage']['slicer']['bands']
number_of_bands = len(bands)
# NOTE: rasdaman supports 1 band grib only to import
recipe_type = self.options['coverage']['slicer']['type']
if recipe_type == GRIBToCoverageConverter.RECIPE_TYPE and number_of_bands > 1:
raise RuntimeError("Only single band grib files are currently supported. "
"Given " + str(number_of_bands) + " bands in ingredient file.")
ret_bands = []
i = 0
for band in bands:
identifier = self._read_or_empty_string(band, "identifier")
if recipe_type == GdalToCoverageConverter.RECIPE_TYPE:
# NOTE: for old ingredients with wrong defined "identifier" with band name instead of index 0-based
if not identifier.isdigit():
identifier = str(i)
ret_bands.append(UserBand(
identifier,
self._read_or_empty_string(band, "name"),
self._read_or_empty_string(band, "description"),
self._read_or_empty_string(band, "definition"),
self._read_or_empty_string(band, "nilReason"),
self._read_or_empty_string(band, "nilValue").split(","),
self._read_or_empty_string(band, "uomCode")
))
i += 1
return ret_bands
else:
if self.options['coverage']['slicer']['type'] == GdalToCoverageConverter.RECIPE_TYPE:
# If gdal does not specify bands in ingredient file, just fetch all bands from first file
for file in self.session.get_files():
try:
gdal_util = GDALGmlUtil(file)
gdal_fields = gdal_util.get_fields_range_type()
ret_bands = []
for field in gdal_fields:
ret_bands.append(UserBand(
field.field_name,
field.field_name,
None,
None,
None,
field.nill_values
))
break
except Exception as e:
if ConfigManager.skip == True:
pass
else:
raise e
return ret_bands
else:
raise RuntimeError("'bands' must be specified in ingredient file for netCDF/GRIB recipes.")
def __get_import_overviews(self):
"""
Get the OVERVIEWs in the ingredients file if user wants to import
"""
if "options" in self.recipe:
if "import_all_overviews" in self.recipe["options"] and "import_overviews" in self.recipe["options"] \
and self.recipe["options"]["import_all_overviews"] is True:
raise RuntimeException(
"Both settings '{}' or '{}' cannot exist in the ingredients file, "
"please specify only one of them.".format(
"import_all_overviews", "import_overviews"))
self.import_overviews = self.recipe["options"]["import_overviews"] \
if "import_overviews" in self.recipe["options"] else []
if len(self.files) > 0 and (
"import_all_overviews" in self.recipe["options"]
or "import_overviews" in self.recipe["options"]):
from util.gdal_util import GDALGmlUtil
first_input_file_gdal = GDALGmlUtil(self.files[0].filepath)
if "import_all_overviews" in self.recipe["options"] \
and bool(self.recipe["options"]["import_all_overviews"]) is True:
# import all overviews
number_of_overviews = first_input_file_gdal.get_number_of_overviews(
)
self.import_overviews = []
if number_of_overviews > 0:
self.import_overviews = range(0, number_of_overviews)
for overview_index in self.import_overviews:
if is_integer(overview_index) is False or int(
overview_index) < 0:
raise RuntimeException(
"'{}' must contain non-negative integers integer values. "
"Given: {}".format("import_overviews",
overview_index))
elif self.recipe["name"] != "sentinel2":
# NOTE: sentinel 2 custom recipe to import .zip file has special format (xml to combine subdatasets (10m, 20m, 60m))
# e.g: SENTINEL2_L2A:/vsizip//home/vagrant/S2A_..._20210601T140140.zip/S2A_..._20210601T140140.SAFE/MTD_MSIL2A.xml:10m:EPSG_32632
overview = first_input_file_gdal.get_overview(
overview_index)
if overview is None:
raise RuntimeException(
"Overview index '{}' does not exist in the input file '{}'."
.format(overview_index,
first_input_file_gdal.gdal_file_path))
from util.gdal_util import GDALGmlUtil
# e.g 101140
gdal_version = int(GDALGmlUtil.get_gdal_version()[0:2])
if gdal_version < 20:
from osgeo import gdal
# e.g: 1.11.4
version = gdal.__version__
raise RuntimeException(
"NOTE: Importing overviews is only supported since gdal version 2.0, "
"and your system has GDAL version '" + version + "'.")
def _data_type(self):
if len(self.files) < 1:
raise RuntimeException("No files to import were specified.")
# as all files should have same data type, so first file is ok
return GDALGmlUtil(self.files[0].get_filepath()).get_band_gdal_type()