def filter_imported_files(self):
"""
Filer all imported files from coverage_id.resume.json
"""
resumer = Resumer(self.coverage_id)
not_imported_files = resumer.get_not_imported_files(self.files)
self.files = not_imported_files
def __init__(self, session):
"""
:param Session session: the session for this import
"""
super(Recipe, self).__init__(session)
self.options = session.get_recipe()['options'] if "options" in session.get_recipe() else {}
self.importer = None
self.resumer = Resumer(self.session.get_coverage_id())
def __init__(self, session):
"""
The recipe class for my_custom_recipe (check wcst_import guide from rasdaman web page for more details).
:param Session session: the session for the import tun
"""
super(Recipe, self).__init__(session)
self.options = session.get_recipe()['options']
self.importer = None
self.resumer = Resumer(self.session.get_coverage_id())
def __init__(self, session):
"""
The recipe class for wcs. To get an overview of the ingredients needed for this
recipe check ingredients/map_mosaic
:param Session session: the session for this import
"""
super(Recipe, self).__init__(session)
self.options = session.get_recipe()['options'] if "options" in session.get_recipe() else {}
self.importer = None
self.resumer = Resumer(self.session.get_coverage_id())
self.coverage = None
def __init__(self, session):
"""
The recipe class for irregular timeseries. To get an overview of the ingredients needed for this
recipe check ingredients/time_series_irregular
"""
super(Recipe, self).__init__(session)
self.options = session.get_recipe()['options'] if "options" in session.get_recipe() else {}
self.importer = None
self.resumer = Resumer(self.session.get_coverage_id())
self.recipe_type = GdalToCoverageConverter.RECIPE_TYPE
if "coverage" in self.options:
self.options['coverage']['slicer'] = {}
self.options['coverage']['slicer']['type'] = GdalToCoverageConverter.RECIPE_TYPE
def __init__(self, session):
"""
The recipe class for map_mosaic. To get an overview of the ingredients needed for this
recipe check ingredients/map_mosaic
:param Session session: the session for this import
"""
super(Recipe, self).__init__(session)
self.options = session.get_recipe()['options'] if "options" in session.get_recipe() else {}
self.importer = None
self.resumer = Resumer(self.session.get_coverage_id())
validator = GDALValidator(self.session.files)
if ConfigManager.skip == True:
self.session.files = validator.get_valid_files()
def __init__(self, session):
"""
The recipe class for regular timeseries. To get an overview of the ingredients needed for this
recipe check ingredients/time_series_regular
"""
super(Recipe, self).__init__(session)
self.options = session.get_recipe(
)['options'] if "options" in session.get_recipe() else {}
self.importer = None
self.resumer = Resumer(self.session.get_coverage_id())
validator = GDALValidator(self.session.files)
if ConfigManager.skip:
self.session.files = validator.get_valid_files()
def __init__(self, session):
"""
:param Session session: the session for this import
"""
super(Recipe, self).__init__(session)
self.session = session
self.options = session.get_recipe()['options']
self.validate()
self.coverage_id = self.session.get_input()["coverage_id"]
self.resumer = Resumer(self.coverage_id)
self.source_coverage_ids = self.parse_source_coverage_ids(
self.session.get_input()["source_coverage_ids"])
self.envelope = self.options["envelope"]
self.srs_name = XMLUtil.escape(self.envelope["srsName"])
# array of axis
self.axes = self.envelope["axis"]
self.import_wms = self.options[
"wms_import"] if "wms_import" in self.options else None
def _get_convertors(self):
"""
Returns a map of coverage id -> GdalToCoverageConverter
"""
convertors = {}
band_data_type = self.DEFAULT_BAND_DATA_TYPE
if self.product == self.SLC_PRODUCT:
band_data_type = self.SLC_BAND_DATA_TYPE
for file in self.session.get_files():
# Check if this file still exists when preparing to import
if not FileUtil.validate_file_path(file.get_filepath()):
continue
# Check if this file belongs to this coverage id
modebeam, polarisation = self._get_modebeam_polarisation(
file.filepath)
cov_id = self._get_coverage_id(self.coverage_id, modebeam,
polarisation)
# This file already imported in coverage_id.resume.json
self.resumer = Resumer(cov_id)
if self.resumer.is_file_imported(file.filepath):
continue
conv = self._get_convertor(convertors, cov_id)
file_pair = FilePair(file.filepath, file.filepath)
conv.files = [file_pair]
crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id)
# Different file contains different datetime from its name
evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
GdalToCoverageConverter.RECIPE_TYPE, file)
conv.data_type = band_data_type
slices = conv._create_coverage_slices(crs_axes, evaluator_slice)
conv.coverage_slices += slices
return convertors
def __init__(self,
coverage,
insert_into_wms=False,
scale_levels=None,
grid_coverage=False):
"""
Imports a coverage into wcst
:param Coverage coverage: the coverage to be imported
"""
self.coverage = coverage
self.resumer = Resumer(coverage.coverage_id)
self.coverage.slices = SliceRestricter(
self.resumer.eliminate_already_imported_slices(
self.coverage.slices)).get_slices()
self.processed = 0
self.total = len(coverage.slices)
self.insert_into_wms = insert_into_wms
self.scale_levels = scale_levels
self.grid_coverage = grid_coverage
def _get_convertors(self):
"""
Returns a map of coverage id -> GdalToCoverageConverter
"""
convertors = {}
for f in self.session.get_files():
# This one does not contain any information for geo bounds
if not FileUtil.validate_file_path(f.get_filepath()):
continue
gdal_ds = GDALGmlUtil(f.get_filepath())
subdatasets = self._get_subdatasets(gdal_ds, f)
gdal_ds.close()
level = self._get_level(f.get_filepath())
if len(self.levels) > 0 and level not in self.levels:
# skip file, as it's not in the list of levels provided in the ingredients file
log.debug("Skipping " + level + " data")
continue
crs_code = ""
evaluator_slice = None
for res in self.resolutions:
subds_file = self._get_subdataset_file(subdatasets, res)
crs_code = self._get_crs_code(subds_file.get_filepath(),
crs_code)
if len(self.crss) > 0 and crs_code not in self.crss:
# skip CRS, it's not in the list of CRSs provided in the ingredients file
log.debug("Skipping data with CRS " + crs_code)
continue
cov_id = self._get_coverage_id(self.coverage_id, crs_code,
level, res)
# This file already imported in coverage_id.resume.json
self.resumer = Resumer(cov_id)
if self.resumer.is_file_imported(f.filepath):
continue
conv = self._get_convertor(convertors, cov_id, crs_code, level,
res)
file_pair = FilePair(subds_file.filepath, f.filepath)
conv.files = [file_pair]
crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id)
if evaluator_slice is None:
# This one contains information for geo bounds
evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(
GdalToCoverageConverter.RECIPE_TYPE, subds_file)
# Resolution 10m, 20m and 60m have same data type (UInt16) while TCI has data type (Byte)
if res == self.RES_TCI:
conv.data_type = "Byte"
else:
conv.data_type = "UInt16"
# Fixed values for 3 axes of Sentinel 2 coverage
axis_resolutions = self.RES_DICT[res]
slices = conv._create_coverage_slices(crs_axes,
evaluator_slice,
axis_resolutions)
conv.coverage_slices += slices
return convertors