def _get_slices(self, crs): """ Returns the slices for the collection of files given """ files = self.session.get_files() crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices = [] count = 1 for file in files: # NOTE: don't process any imported file from *.resume.json as it is just waisted time if not self.resumer.is_file_imported(file.filepath): timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(files), file.filepath) if not FileUtil.validate_file_path(file.filepath): continue valid_coverage_slice = True try: subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill() except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible(file.get_filepath(), ex) valid_coverage_slice = False if valid_coverage_slice: slices.append(Slice(subsets, FileDataProvider(file))) timer.print_elapsed_time() count += 1 return slices
def _get_coverage_slices(self, crs, gdal_coverage_converter): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices_dict = self.create_dict_of_slices(self.session.import_overviews) timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: file = tpair.file file_path = tpair.file.get_filepath() timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), file_path) if not FileUtil.validate_file_path(file_path): continue valid_coverage_slice = True gdal_file = GDALGmlUtil(file.get_filepath()) try: subsets = GdalAxisFiller(crs_axes, gdal_file).fill(True) subsets = self._fill_time_axis(tpair, subsets) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file_path, ex) valid_coverage_slice = False if valid_coverage_slice: # Generate local metadata string for current coverage slice self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, tpair.file) local_metadata = gdal_coverage_converter._generate_local_metadata( subsets, self.evaluator_slice) if self.session.import_overviews_only is False: slices_dict["base"].append( Slice(subsets, FileDataProvider(tpair.file), local_metadata)) # Then, create slices for selected overviews from user for overview_index in self.session.import_overviews: subsets_overview = self.create_subsets_for_overview( subsets, overview_index, gdal_file) slices_dict[str(overview_index)].append( Slice(subsets_overview, FileDataProvider(file), local_metadata)) timer.print_elapsed_time() count += 1 return slices_dict
def validate_base(self, ignore_no_files=False): """ Validates the configuration and the input files :param bool ignore_no_files: if the extending recipe does not work with files, set this to true to skip the validation check for no files (used in wcs_extract recipe). """ if self.session.get_wcs_service( ) is None or self.session.get_wcs_service() == "": raise RecipeValidationException("No valid wcs endpoint provided") if self.session.get_crs_resolver( ) is None or self.session.get_crs_resolver() == "": raise RecipeValidationException("No valid crs resolver provided") if self.session.get_coverage_id( ) is None or self.session.get_coverage_id() == "": raise RecipeValidationException("No valid coverage id provided") if not FileUtil.check_dir_writable(ConfigManager.tmp_directory): raise RecipeValidationException( "Cannot write to tmp directory '{}'".format( ConfigManager.tmp_directory)) checked_files = [] for file in self.session.get_files(): if FileUtil.validate_file_path(file.get_filepath()): checked_files.append(file) if not ignore_no_files: # If no input file is available, exit wcst_import. FileUtil.validate_input_file_paths(checked_files) self.session.files = checked_files if 'wms_import' not in self.options: self.options['wms_import'] = False else: self.options['wms_import'] = bool(self.options['wms_import']) if 'tiling' not in self.options: self.options['tiling'] = None if 'scale_levels' not in self.options: self.options['scale_levels'] = None if "import_order" in self.options: if self.options['import_order'] != AbstractToCoverageConverter.IMPORT_ORDER_ASCENDING \ and self.options['import_order'] != AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING: error_message = "'import_order' option must be '{}' or '{}', given '{}'.".\ format(AbstractToCoverageConverter.IMPORT_ORDER_ASCENDING, AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING, self.options['import_order']) raise RecipeValidationException(error_message) else: self.options['import_order'] = None
def _get_convertors(self): """ Returns a map of coverage id -> GdalToCoverageConverter """ convertors = {} band_data_type = self.DEFAULT_BAND_DATA_TYPE if self.product == self.SLC_PRODUCT: band_data_type = self.SLC_BAND_DATA_TYPE for file in self.session.get_files(): # Check if this file still exists when preparing to import if not FileUtil.validate_file_path(file.get_filepath()): continue # Check if this file belongs to this coverage id modebeam, polarisation = self._get_modebeam_polarisation( file.filepath) cov_id = self._get_coverage_id(self.coverage_id, modebeam, polarisation) conv = self._get_convertor(convertors, cov_id) file_pair = FilePair(file.filepath, file.filepath) conv.files = [file_pair] crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id) # Different file contains different datetime from its name evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( GdalToCoverageConverter.RECIPE_TYPE, file) conv.data_type = band_data_type slices_dict = conv._create_coverage_slices(crs_axes, evaluator_slice) slices_dict = self.__filter_invalid_geo_bounds(slices_dict) if conv.coverage_slices == {}: conv.coverage_slices = slices_dict else: for key, val in slices_dict.items(): conv.coverage_slices[key] += slices_dict[key] if len(conv.coverage_slices) > 0 and len( conv.coverage_slices["base"]) > 0: first_slice = conv.coverage_slices["base"][0] # This needs one available file to extract metadata later conv.files = [first_slice.data_provider.file] return convertors
def collect_local_metadata_file(self, local_metadata, sentence): """ Collect local metadata from external metadata files if it is set in the ingredient :param dict local_metadata """ if "root_element" not in sentence or "path" not in sentence: raise RuntimeError("'metadata_file' setting must contain one 'root_element'" " and one 'path' children settings in the ingredient file") root_element = str(self.evaluator.evaluate(sentence["root_element"], self.metadata_entry.evalutor_slice)) metadata_file = str(self.evaluator.evaluate(sentence["path"], self.metadata_entry.evalutor_slice)) if not FileUtil.validate_file_path(metadata_file): raise RuntimeError("Cannot access local metadata file '" + metadata_file + "' to read") else: local_metadata[root_element] = XMLUtil.read_file_and_remove_xml_header(metadata_file)
def _create_coverage_slices(self, crs_axes, calculated_evaluator_slice=None, axis_resolutions=None): """ Returns all the coverage slices for this coverage :param crs_axes: :rtype: list[Slice] """ slices = [] count = 1 for file in self.files: # NOTE: don't process any previously imported file (recorded in *.resume.json) if not self.resumer.is_file_imported(file.filepath): timer = Timer() FileUtil.print_feedback(count, len(self.files), file.filepath) # print which file is analyzing if not FileUtil.validate_file_path(file.filepath): continue valid_coverage_slice = True evaluator_slice = None try: if calculated_evaluator_slice is None: # get the evaluator for the current recipe_type (each recipe has different evaluator) evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(self.recipe_type, file) else: evaluator_slice = calculated_evaluator_slice if self.data_type is None: self.data_type = evaluator_slice.get_data_type(self) coverage_slice = self._create_coverage_slice(file, crs_axes, evaluator_slice, axis_resolutions) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible(file.get_filepath(), ex) valid_coverage_slice = False if valid_coverage_slice: slices.append(coverage_slice) timer.print_elapsed_time() count += 1 # Currently, only sort by datetime to import coverage slices (default is ascending) reverse = (self.import_order == self.IMPORT_ORDER_DESCENDING) return sort_slices_by_datetime(slices, reverse)
def _get_coverage_slices(self, crs, gdal_coverage_converter): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices = [] timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: file_path = tpair.file.get_filepath() # NOTE: don't process any imported file from *.resume.json as it is just waisted time if not self.resumer.is_file_imported(file_path): timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), file_path) if not FileUtil.validate_file_path(file_path): continue valid_coverage_slice = True try: subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file_path)).fill(True) subsets = self._fill_time_axis(tpair, subsets) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file_path, ex) valid_coverage_slice = False if valid_coverage_slice: # Generate local metadata string for current coverage slice self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, tpair.file) local_metadata = gdal_coverage_converter._generate_local_metadata( subsets, self.evaluator_slice) slices.append( Slice(subsets, FileDataProvider(tpair.file), local_metadata)) timer.print_elapsed_time() count += 1 return slices
def _get_convertors(self): """ Returns a map of coverage id -> GdalToCoverageConverter """ convertors = {} band_data_type = self.DEFAULT_BAND_DATA_TYPE if self.product == self.SLC_PRODUCT: band_data_type = self.SLC_BAND_DATA_TYPE for file in self.session.get_files(): # Check if this file still exists when preparing to import if not FileUtil.validate_file_path(file.get_filepath()): continue # Check if this file belongs to this coverage id modebeam, polarisation = self._get_modebeam_polarisation( file.filepath) cov_id = self._get_coverage_id(self.coverage_id, modebeam, polarisation) # This file already imported in coverage_id.resume.json self.resumer = Resumer(cov_id) if self.resumer.is_file_imported(file.filepath): continue conv = self._get_convertor(convertors, cov_id) file_pair = FilePair(file.filepath, file.filepath) conv.files = [file_pair] crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id) # Different file contains different datetime from its name evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( GdalToCoverageConverter.RECIPE_TYPE, file) conv.data_type = band_data_type slices = conv._create_coverage_slices(crs_axes, evaluator_slice) conv.coverage_slices += slices return convertors
def _get_convertors(self): """ Returns a map of coverage id -> GdalToCoverageConverter """ convertors = {} for f in self.session.get_files(): # This one does not contain any information for geo bounds if not FileUtil.validate_file_path(f.get_filepath()): continue gdal_ds = GDALGmlUtil(f.get_filepath()) subdatasets = self._get_subdatasets(gdal_ds, f) gdal_ds.close() level = self._get_level(f.get_filepath()) if len(self.levels) > 0 and level not in self.levels: # skip file, as it's not in the list of levels provided in the ingredients file log.debug("Skipping " + level + " data") continue crs_code = "" evaluator_slice = None for res in self.resolutions: subds_file = self._get_subdataset_file(subdatasets, res) crs_code = self._get_crs_code(subds_file.get_filepath(), crs_code) if len(self.crss) > 0 and crs_code not in self.crss: # skip CRS, it's not in the list of CRSs provided in the ingredients file log.debug("Skipping data with CRS " + crs_code) continue cov_id = self._get_coverage_id(self.coverage_id, crs_code, level, res) conv = self._get_convertor(convertors, cov_id, crs_code, level, res) file_pair = FilePair(subds_file.filepath, f.filepath) conv.files = [file_pair] crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id) if evaluator_slice is None: # This one contains information for geo bounds evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( GdalToCoverageConverter.RECIPE_TYPE, subds_file) # Resolution 10m, 20m and 60m have same data type (UInt16) while TCI has data type (Byte) if res == self.RES_TCI: conv.data_type = "Byte" else: conv.data_type = "UInt16" # Fixed values for 3 axes of Sentinel 2 coverage axis_resolutions = self.RES_DICT[res] slices_dict = conv._create_coverage_slices( crs_axes, evaluator_slice, axis_resolutions) if conv.coverage_slices == {}: conv.coverage_slices = slices_dict else: for key, val in slices_dict.items(): conv.coverage_slices[key] += slices_dict[key] if len(conv.coverage_slices) != 0: first_slice = conv.coverage_slices["base"][0] # This needs one available file to extract metadata later conv.files = [first_slice.data_provider.file] return convertors
def validate_base(self, ignore_no_files=False): """ Validates the configuration and the input files :param bool ignore_no_files: if the extending recipe does not work with files, set this to true to skip the validation check for no files (used in wcs_extract recipe). """ if self.session.get_wcs_service( ) is None or self.session.get_wcs_service() == "": raise RecipeValidationException("No valid wcs endpoint provided") if self.session.get_crs_resolver( ) is None or self.session.get_crs_resolver() == "": raise RecipeValidationException("No valid crs resolver provided") if self.session.get_coverage_id( ) is None or self.session.get_coverage_id() == "": raise RecipeValidationException("No valid coverage id provided") import recipes.virtual_coverage.recipe as super_coverage if self.session.get_recipe_name() == super_coverage.Recipe.RECIPE_NAME: # NOTE: virtual_coverage recipe does not require any input files return if not FileUtil.check_dir_writable(ConfigManager.tmp_directory): raise RecipeValidationException( "Cannot write to tmp directory '{}'".format( ConfigManager.tmp_directory)) checked_files = [] for file in self.session.get_files(): if FileUtil.validate_file_path(file.get_filepath()): checked_files.append(file) if not ignore_no_files: # If no input file is available, exit wcst_import. FileUtil.validate_input_file_paths(checked_files) self.session.files = checked_files if 'wms_import' not in self.options: self.options['wms_import'] = False else: self.options['wms_import'] = bool(self.options['wms_import']) if 'tiling' not in self.options: self.options['tiling'] = None if 'scale_levels' not in self.options: self.options['scale_levels'] = None if 'scale_factors' not in self.options: self.options['scale_factors'] = None if self.options['scale_levels'] is not None \ and self.options['scale_factors'] is not None: raise RecipeValidationException( "Only one of 'scale_levels' or 'scale_factors' " "setting can exist in the ingredients file.") if self.options['scale_factors'] is not None: # as scale_factors and scale_levels are only valid when initializing a new coverage cov = CoverageUtil(self.session.get_coverage_id()) if not cov.exists(): for obj in self.options['scale_factors']: if 'coverage_id' not in obj or 'factors' not in obj: raise RecipeValidationException( "All elements of 'scale_factors' list must contain " "'coverage_id' and 'factors' properties") coverage_id = obj['coverage_id'] cov = CoverageUtil(coverage_id) if cov.exists(): raise RecipeValidationException( "Downscaled level coverage '" + coverage_id + "' already exists, " "please use a different 'coverage_id' in 'scale_factors' list" ) self.validate_pyramid_members() self.validate_pyramid_bases() if "import_order" in self.options: if self.options['import_order'] != AbstractToCoverageConverter.IMPORT_ORDER_ASCENDING \ and self.options['import_order'] != AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING: error_message = "'import_order' option must be '{}' or '{}', given '{}'.".\ format(AbstractToCoverageConverter.IMPORT_ORDER_ASCENDING, AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING, self.options['import_order']) raise RecipeValidationException(error_message) else: self.options['import_order'] = None
def _create_coverage_slices(self, crs_axes, calculated_evaluator_slice=None, axis_resolutions=None): """ Returns the slices for the collection of files given :param crs_axes: :rtype: list[Slice] """ from master.recipe.base_recipe import BaseRecipe slices_dict = BaseRecipe.create_dict_of_slices( self.session.import_overviews) count = 1 for file in self.files: timer = Timer() FileUtil.print_feedback(count, len(self.files), file.filepath) # print which file is analyzing if not FileUtil.validate_file_path(file.filepath): continue valid_coverage_slice = True try: if calculated_evaluator_slice is None: # get the evaluator for the current recipe_type (each recipe has different evaluator) self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, file) else: self.evaluator_slice = calculated_evaluator_slice if self.data_type is None: self.data_type = self.evaluator_slice.get_data_type(self) coverage_slice = self._create_coverage_slice( file, crs_axes, self.evaluator_slice, axis_resolutions) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file.get_filepath(), ex) valid_coverage_slice = False if valid_coverage_slice: if self.session.import_overviews_only is False: slices_dict["base"].append(coverage_slice) if self.session.recipe["options"]["coverage"]["slicer"][ "type"] == "gdal": gdal_file = GDALGmlUtil(file.get_filepath()) # Then, create slices for selected overviews from user for overview_index in self.session.import_overviews: axis_subsets_overview = BaseRecipe.create_subsets_for_overview( coverage_slice.axis_subsets, overview_index, gdal_file) coverage_slice_overview = copy.deepcopy(coverage_slice) coverage_slice_overview.axis_subsets = axis_subsets_overview slices_dict[str(overview_index)].append( coverage_slice_overview) timer.print_elapsed_time() count += 1 # Currently, only sort by datetime to import coverage slices (default is ascending) reverse = (self.import_order == self.IMPORT_ORDER_DESCENDING) for key, value in slices_dict.items(): slices_dict[key] = sort_slices_by_datetime(value, reverse) return slices_dict