def _get_coverages(self):
"""
Returns the list of coverages to be used for the importer
"""
gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(
self.session.get_files())
crs = CRSUtil.get_compound_crs(
[self.options['time_crs'],
gdal_dataset.get_crs()])
general_recipe = GeneralRecipe(self.session)
global_metadata_fields = general_recipe._global_metadata_fields()
local_metadata_fields = general_recipe._local_metadata_fields()
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
gdal_coverage_converter = GdalToCoverageConverter(
self.resumer, self.session.get_default_null_values(),
self.recipe_type,
sentence_evaluator, self.session.get_coverage_id(), None,
self.session.get_files(), crs, None, None,
global_metadata_fields, local_metadata_fields, None, None,
general_recipe._metadata_type(), None, None, self.session)
coverage_slices_dict = self._get_coverage_slices(
crs, gdal_coverage_converter)
fields = GdalRangeFieldsGenerator(
gdal_dataset, self.options['band_names']).get_range_fields()
global_metadata = None
if len(coverage_slices_dict["base"]) > 0:
global_metadata = gdal_coverage_converter._generate_global_metadata(
coverage_slices_dict["base"][0], self.evaluator_slice)
results = []
base_coverage_id = self.session.get_coverage_id()
for key, value in coverage_slices_dict.items():
if key == "base":
# base coverage
coverage = Coverage(base_coverage_id,
coverage_slices_dict[key], fields, crs,
gdal_dataset.get_band_gdal_type(),
self.options['tiling'], global_metadata)
else:
# overview coverage (key = overview_index)
coverage_id = create_coverage_id_for_overview(
base_coverage_id, key)
coverage = Coverage(coverage_id, coverage_slices_dict[key],
fields, crs,
gdal_dataset.get_band_gdal_type(),
self.options['tiling'], global_metadata,
base_coverage_id, key)
results.append(coverage)
return results
def _get_coverage(self):
"""
Returns the coverage to be used for the importer
"""
gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(self.session.get_files())
crs = gdal_dataset.get_crs()
general_recipe = GeneralRecipe(self.session)
global_metadata_fields = general_recipe._global_metadata_fields()
local_metadata_fields = general_recipe._local_metadata_fields()
sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory())
gdal_coverage_converter = GdalToCoverageConverter(self.resumer, self.session.get_default_null_values(),
self.recipe_type, sentence_evaluator,
self.session.get_coverage_id(),
None, self.session.get_files(),
crs, None, None,
global_metadata_fields, local_metadata_fields,
None, None, general_recipe._metadata_type(),
None, None)
coverage_slices = self._get_coverage_slices(crs, gdal_coverage_converter)
fields = GdalRangeFieldsGenerator(gdal_dataset, self.options['band_names']).get_range_fields()
global_metadata = None
if len(coverage_slices) > 0:
global_metadata = gdal_coverage_converter._generate_global_metadata(coverage_slices[0], self.evaluator_slice)
coverage = Coverage(self.session.get_coverage_id(), coverage_slices, fields, gdal_dataset.get_crs(),
gdal_dataset.get_band_gdal_type(), self.options['tiling'], global_metadata)
return coverage
def to_coverages(self, coverage_slices_dict=None):
"""
Returns list coverages from all the importing files (gdal|grib|netcdf)
:rtype: Array[Coverage]
"""
crs_axes = CRSUtil(self.crs).get_axes(self.coverage_id)
if coverage_slices_dict is None:
# Build list of coverage slices from input files
coverage_slices_dict = self._create_coverage_slices(crs_axes)
global_metadata = None
if len(coverage_slices_dict) > 0:
for coverage_level, slices in coverage_slices_dict.items():
if len(slices) > 0:
first_coverage_slice = slices[0]
break
# generate coverage extra_metadata from ingredient file based on first input file of first coverage slice.
global_metadata = self._generate_global_metadata(
first_coverage_slice)
# Evaluate all the swe bands's metadata (each file should have same swe bands's metadata), so first file is ok
self._evaluate_swe_bands_metadata(self.files[0], self.bands)
results = []
base_coverage_id = self.coverage_id
for key, value in coverage_slices_dict.items():
slices = coverage_slices_dict[key]
if key == "base":
coverage = Coverage(base_coverage_id, slices,
self._range_fields(), self.crs,
self._data_type(), self.tiling,
global_metadata)
else:
# overview coverage (key = overview_index)
coverage_id = create_coverage_id_for_overview(
self.coverage_id, key)
coverage = Coverage(coverage_id, slices, self._range_fields(),
self.crs, self._data_type(), self.tiling,
global_metadata, base_coverage_id, key)
results.append(coverage)
return results
def _get_coverage(self):
# Get the crs of one of the images using a GDAL helper class. We are assuming all images have the same CRS
gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath())
# Get the crs of the coverage by compounding the two crses
crs = CRSUtil.get_compound_crs(
[gdal_dataset.get_crs(), self.options['time_crs']])
fields = GdalRangeFieldsGenerator(gdal_dataset).get_range_fields()
pixel_type = gdal_dataset.get_band_gdal_type()
coverage_id = self.session.get_coverage_id()
slices = self._get_slices(crs)
return Coverage(coverage_id, slices, fields, crs, pixel_type)
def _get_coverage(self):
"""
Returns the coverage to be used for the importer
"""
gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(self.session.get_files())
crs = gdal_dataset.get_crs()
slices = self._get_slices(crs)
fields = GdalRangeFieldsGenerator(gdal_dataset, self.options['band_names']).get_range_fields()
coverage = Coverage(self.session.get_coverage_id(), slices, fields, gdal_dataset.get_crs(),
gdal_dataset.get_band_gdal_type(), self.options['tiling'])
return coverage
def _get_coverage(self):
"""
Returns the coverage to be used for the importer
"""
gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath())
crs = CRSUtil.get_compound_crs(
[gdal_dataset.get_crs(), self.options['time_crs']])
slices = self._get_slices(crs)
fields = GdalRangeFieldsGenerator(
gdal_dataset, self.options['band_names']).get_range_fields()
coverage = Coverage(self.session.get_coverage_id(), slices, fields,
crs, gdal_dataset.get_band_gdal_type(),
self.options['tiling'])
return coverage
def ingest(self):
"""
Starts the ingesting process
"""
coverage = Coverage(self.coverage_id, [], None, None, None)
importer = Importer(self.resumer, coverage, self.import_wms)
if importer._is_insert():
self.__insert_coverage_request()
else:
self.__update_coverage_request()
if self.import_wms:
importer._insert_update_into_wms()
def to_coverage(self):
"""
Returns a Coverage from all the importing files (gdal|grib|netcdf)
:rtype: Coverage
"""
crs_axes = CRSUtil(self.crs).get_axes()
slices = self._slices(crs_axes)
# generate coverage extra_metadata from ingredient files
metadata = self._metadata(slices)
# Evaluate all the swe bands's metadata (each file should have same swe bands's metadata), so first file is ok
self._evaluate_swe_bands_metadata(self.files[0], self.bands)
coverage = Coverage(self.coverage_id, slices, self._range_fields(),
self.crs, self._data_type(), self.tiling, metadata)
return coverage
def _get_coverage(self):
# Get the crs of one of the images using a GDAL helper class. We are assuming all images have the same CRS
for file in self.session.get_files():
try:
file_path = file.get_filepath()
gdal_dataset = GDALGmlUtil(file_path)
break
except Exception as e:
if ConfigManager.skip == True:
pass
else:
raise e
# Get the crs of the coverage by compounding the two crses
crs = CRSUtil.get_compound_crs([gdal_dataset.get_crs(), self.options['time_crs']])
fields = GdalRangeFieldsGenerator(gdal_dataset).get_range_fields()
pixel_type = gdal_dataset.get_band_gdal_type()
coverage_id = self.session.get_coverage_id()
slices = self._get_slices(crs)
return Coverage(coverage_id, slices, fields, crs, pixel_type)
def _read(self):
"""
Reads the metadata from the describe coverage and creates the virtual coverage object
:rtype: Coverage
"""
try:
xmlstr = self.description()
root = etree.fromstring(xmlstr)
crs = self._get_crs(root)
crs_axes = CRSUtil(crs).get_axes(self.coverage_id)
range_type = self._get_range_types(root)
raster_coords = self._get_raster_coords(root)
geo_coords = self._get_geo_coords(root)
coverage_id = self._get_coverage_id(root)
resolutions = self._get_resolutions(root, crs_axes)
origin = self._get_origin(root)
coverage_axes = self._get_coverage_axes(geo_coords, raster_coords,
origin, crs_axes,
resolutions)
intervals = self._get_intervals(coverage_axes,
self.partitioning_scheme)
slices = self._get_slices(coverage_axes, intervals)
pixel_data_type, generated_file_path = self._get_data_type(
slices[0])
coverage = Coverage(coverage_id, slices, range_type, crs,
pixel_data_type)
self.coverage = coverage
except IOError as e:
os.remove(generated_file_path)
raise RuntimeException(
"Could not read the coverage description for url: " +
self._get_description_url() +
". Check that the url is accessible and try again. More details: "
+ str(e))
except XMLSyntaxError as e:
os.remove(generated_file_path)
raise RuntimeException(
"Could not decode the xml description for url " +
self._get_description_url() +
". Check that the url is correct and try again. More details: "
+ str(e))
def to_coverage(self, coverage_slices=None):
"""
Returns a Coverage from all the importing files (gdal|grib|netcdf)
:rtype: Coverage
"""
crs_axes = CRSUtil(self.crs).get_axes(self.coverage_id)
if coverage_slices is None:
# Build list of coverage slices from input files
coverage_slices = self._create_coverage_slices(crs_axes)
global_metadata = None
if len(coverage_slices) > 0:
first_coverage_slice = coverage_slices[0]
# generate coverage extra_metadata from ingredient file based on first input file of first coverage slice.
global_metadata = self._generate_global_metadata(first_coverage_slice)
# Evaluate all the swe bands's metadata (each file should have same swe bands's metadata), so first file is ok
self._evaluate_swe_bands_metadata(self.files[0], self.bands)
coverage = Coverage(self.coverage_id, coverage_slices, self._range_fields(), self.crs,
self._data_type(),
self.tiling, global_metadata)
return coverage