def _get_update_crs(self, slice, crs): """ Returns the crs corresponding to the axes that are data bound :param slice: the slice for which the gml should be created :param crs: the crs of the coverage :return: String """ crsAxes = [] for axis_subset in slice.axis_subsets: if axis_subset.coverage_axis.data_bound: crsAxes.append(axis_subset.coverage_axis.axis.crs_axis) crsUtil = CRSUtil(crs) return crsUtil.get_crs_for_axes(crsAxes)
def _get_slices(self, crs): # Let's first extract all the axes from our crs crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) # Prepare a list container for our slices slices = [] # Iterate over the files and create a slice for each one for infile in self.session.get_files(): # We need to create the exact position in time and space in which to place this slice # For the space coordinates we can use the GDAL helper to extract it for us # The helper will return a list of subsets based on the crs axes that we extracted # and will fill the coordinates for the ones that it can (the easting and northing axes) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil( infile.get_filepath())).fill() # Now we must fill the time axis as well and indicate the position in time for subset in subsets: # Find the time axis if subset.coverage_axis.axis.crs_axis.is_time_axis(): # Set the time position for it. Our recipe extracts it from a GDAL tag provided by the user # datetime format needs enquoted (e.g: "2015-01") subset.interval.low = '"' + GDALGmlUtil( infile).get_datetime(self.options["time_tag"]) + '"' slices.append(Slice(subsets, FileDataProvider(infile))) return slices
def _get_slices(self, crs): """ Returns the slices for the collection of files given """ files = self.session.get_files() crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices = [] count = 1 for file in files: # NOTE: don't process any imported file from *.resume.json as it is just waisted time if not self.resumer.is_file_imported(file.filepath): timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(files), file.filepath) if not FileUtil.validate_file_path(file.filepath): continue valid_coverage_slice = True try: subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill() except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible(file.get_filepath(), ex) valid_coverage_slice = False if valid_coverage_slice: slices.append(Slice(subsets, FileDataProvider(file))) timer.print_elapsed_time() count += 1 return slices
def _get_coverage_slices(self, crs, gdal_coverage_converter): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices_dict = self.create_dict_of_slices(self.session.import_overviews) timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: file = tpair.file file_path = tpair.file.get_filepath() timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), file_path) if not FileUtil.validate_file_path(file_path): continue valid_coverage_slice = True gdal_file = GDALGmlUtil(file.get_filepath()) try: subsets = GdalAxisFiller(crs_axes, gdal_file).fill(True) subsets = self._fill_time_axis(tpair, subsets) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file_path, ex) valid_coverage_slice = False if valid_coverage_slice: # Generate local metadata string for current coverage slice self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, tpair.file) local_metadata = gdal_coverage_converter._generate_local_metadata( subsets, self.evaluator_slice) if self.session.import_overviews_only is False: slices_dict["base"].append( Slice(subsets, FileDataProvider(tpair.file), local_metadata)) # Then, create slices for selected overviews from user for overview_index in self.session.import_overviews: subsets_overview = self.create_subsets_for_overview( subsets, overview_index, gdal_file) slices_dict[str(overview_index)].append( Slice(subsets_overview, FileDataProvider(file), local_metadata)) timer.print_elapsed_time() count += 1 return slices_dict
def __init__(self, datetime, dt_format=None, time_crs=None): """ :param str datetime: the datetime value :param str dt_format: the datetime format, if none is given we'll try to guess """ try: if dt_format is None: self.datetime = arrow.get(datetime) else: self.datetime = arrow.get(datetime, dt_format) if time_crs is None: self.time_crs_code = self.CRS_CODE_ANSI_DATE else: tmp_crs = CRSUtil(time_crs) self.time_crs_code = tmp_crs.get_crs_code() except ParserError as pe: dt_format_err = "auto" if dt_format is None else dt_format raise RuntimeException("Failed to parse the date " + datetime + " using format " + dt_format_err)
def _get_coverages(self): """ Returns the list of coverages to be used for the importer """ gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file( self.session.get_files()) crs = CRSUtil.get_compound_crs( [self.options['time_crs'], gdal_dataset.get_crs()]) general_recipe = GeneralRecipe(self.session) global_metadata_fields = general_recipe._global_metadata_fields() local_metadata_fields = general_recipe._local_metadata_fields() sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory()) gdal_coverage_converter = GdalToCoverageConverter( self.resumer, self.session.get_default_null_values(), self.recipe_type, sentence_evaluator, self.session.get_coverage_id(), None, self.session.get_files(), crs, None, None, global_metadata_fields, local_metadata_fields, None, None, general_recipe._metadata_type(), None, None, self.session) coverage_slices_dict = self._get_coverage_slices( crs, gdal_coverage_converter) fields = GdalRangeFieldsGenerator( gdal_dataset, self.options['band_names']).get_range_fields() global_metadata = None if len(coverage_slices_dict["base"]) > 0: global_metadata = gdal_coverage_converter._generate_global_metadata( coverage_slices_dict["base"][0], self.evaluator_slice) results = [] base_coverage_id = self.session.get_coverage_id() for key, value in coverage_slices_dict.items(): if key == "base": # base coverage coverage = Coverage(base_coverage_id, coverage_slices_dict[key], fields, crs, gdal_dataset.get_band_gdal_type(), self.options['tiling'], global_metadata) else: # overview coverage (key = overview_index) coverage_id = create_coverage_id_for_overview( base_coverage_id, key) coverage = Coverage(coverage_id, coverage_slices_dict[key], fields, crs, gdal_dataset.get_band_gdal_type(), self.options['tiling'], global_metadata, base_coverage_id, key) results.append(coverage) return results
def __init__(self, datetime, dt_format=None, time_crs=None): """ :param str datetime: the datetime value :param str dt_format: the datetime format, if none is given we'll try to guess """ self.init_cache() try: if dt_format is None: self.datetime = arrow.get(datetime) else: self.datetime = arrow.get(datetime, dt_format) if time_crs is None: self.time_crs_code = self.CRS_CODE_ANSI_DATE else: tmp_crs = CRSUtil(time_crs) self.time_crs_code = tmp_crs.get_crs_code() except ParserError as pe: dt_format_err = "auto" if dt_format is None else dt_format raise RuntimeException("Failed to parse the date " + datetime + " using format " + dt_format_err)
def _get_coverage(self): # Get the crs of one of the images using a GDAL helper class. We are assuming all images have the same CRS gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath()) # Get the crs of the coverage by compounding the two crses crs = CRSUtil.get_compound_crs( [gdal_dataset.get_crs(), self.options['time_crs']]) fields = GdalRangeFieldsGenerator(gdal_dataset).get_range_fields() pixel_type = gdal_dataset.get_band_gdal_type() coverage_id = self.session.get_coverage_id() slices = self._get_slices(crs) return Coverage(coverage_id, slices, fields, crs, pixel_type)
def _get_coverage(self): """ Returns the coverage to be used for the importer """ gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath()) crs = CRSUtil.get_compound_crs([gdal_dataset.get_crs(), self.options['time_crs']]) slices = self._get_slices(crs) fields = GdalRangeFieldsGenerator(gdal_dataset, self.options['band_names']).get_range_fields() coverage = Coverage(self.session.get_coverage_id(), slices, fields, crs, gdal_dataset.get_band_gdal_type(), self.options['tiling']) return coverage
def _resolve_crs(self, crs): """ Resolves a crs string to an url :param str crs: the crs string :rtype: str """ crs_resolver = self.session.get_crs_resolver() + "crs/" if not crs.startswith("http"): crs_parts = crs.split("@") for i in range(0, len(crs_parts)): crs_parts[i] = crs_resolver + crs_parts[i] return CRSUtil.get_compound_crs(crs_parts) return crs
def __get_epsg_xy_axes_labels(self): """ Return a tuple of axis labels for X and Y axes """ axes_labels = CRSUtil.get_axis_labels_from_single_crs(self.epsg_xy_crs) axis_type1 = CRSAxis.get_axis_type_by_name(axes_labels[0]) # XY order (e.g: EPSG:3857) if axis_type1 == CRSAxis.AXIS_TYPE_X: return axes_labels[0], axes_labels[1] else: # YX order (e.g: EPSG:4326) needs to swap order return axes_labels[1], axes_labels[0]
def _resolve_crs(self, crs): """ Resolves a crs string to an url :param str crs: the crs string :rtype: str """ crs_resolver = self.session.get_crs_resolver() + "crs/" if not crs.startswith("http"): crs_parts = crs.split("@") for i in range(0, len(crs_parts)): crs_parts[i] = crs_resolver + crs_parts[i] return CRSUtil.get_compound_crs(crs_parts) return crs
def _get_crs_axis_by_user_axis_name(self, user_axis_name): """ Returns the crs axis from the list by user_axis_name :param user_axis_name: :return: crs_axis """ crs_axes = CRSUtil(self.crs).get_axes(self.coverage_id) for i in range(0, len(crs_axes)): crs_axis = crs_axes[i] if crs_axis.label == user_axis_name: return crs_axis
def _read_axes(self, crs): """ Returns a list of user axes extracted from the ingredients file :param str crs: the crs of the coverage :rtype: list[UserAxis] """ axes = self.options['coverage']['slicer']['axes'] user_axes = [] crs_axes = CRSUtil(crs).get_axes() default_order = 0 for crs_axis in crs_axes: if crs_axis.label not in axes: raise RecipeValidationException( "Could not find a definition for axis " + crs_axis.label + " in the axes parameter.") axis = axes[crs_axis.label] max = axis["max"] if "max" in axis else None if "type" in axis: type = axis["type"] elif crs_axis.is_date(): type = UserAxisType.DATE else: type = UserAxisType.NUMBER order = axis["gridOrder"] if "gridOrder" in axis else default_order irregular = axis["irregular"] if "irregular" in axis else False data_bound = axis["dataBound"] if "dataBound" in axis else True # for irregular axes we consider the resolution 1 / -1 as gmlcov requires resolution for all axis types, # even irregular if "resolution" in axis: resolution = axis["resolution"] else: resolution = 1 default_order += 1 if "statements" in axis: if isinstance(axis["statements"], list): statements = axis["statements"] else: statements = [axis["statements"]] else: statements = [] if not irregular: user_axes.append( RegularUserAxis(crs_axis.label, resolution, order, axis["min"], max, type, data_bound, statements=statements)) else: user_axes.append( IrregularUserAxis(crs_axis.label, resolution, order, axis["min"], axis["directPositions"], max, type, data_bound, statements=statements)) return user_axes
def __filter_invalid_geo_bounds(self, slices_dict): """ Filter any coverage slices (scenes) which have invalid lat and long bounds in EPSG:4326 """ results = OrderedDict() for key, slices in slices_dict.items(): results[key] = [] for slice in slices: input_file = slice.data_provider.file.filepath axis_subsets = slice.axis_subsets is_valid = True for axis_subset in axis_subsets: axis = axis_subset.coverage_axis.axis geo_lower_bound = axis.low geo_upper_bound = axis.high axis_label = axis.label if axis.crs_axis.uri.endswith(self.EPSG_4326): if CRSUtil.is_latitude_axis(axis_label): is_valid = geo_lower_bound >= -90 and geo_upper_bound <= 90 elif CRSUtil.is_longitude_axis(axis_label): is_valid = geo_lower_bound >= -180 and geo_upper_bound <= 180 if not is_valid: log.warn( "File '" + input_file + "' has invalid lat or long axes geo bounds in EPSG:4326 CRS, ignored for further processing." ) break if is_valid: results[key].append(slice) return results
def _get_convertors(self): """ Returns a map of coverage id -> GdalToCoverageConverter """ convertors = {} band_data_type = self.DEFAULT_BAND_DATA_TYPE if self.product == self.SLC_PRODUCT: band_data_type = self.SLC_BAND_DATA_TYPE for file in self.session.get_files(): # Check if this file still exists when preparing to import if not FileUtil.validate_file_path(file.get_filepath()): continue # Check if this file belongs to this coverage id modebeam, polarisation = self._get_modebeam_polarisation( file.filepath) cov_id = self._get_coverage_id(self.coverage_id, modebeam, polarisation) conv = self._get_convertor(convertors, cov_id) file_pair = FilePair(file.filepath, file.filepath) conv.files = [file_pair] crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id) # Different file contains different datetime from its name evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( GdalToCoverageConverter.RECIPE_TYPE, file) conv.data_type = band_data_type slices_dict = conv._create_coverage_slices(crs_axes, evaluator_slice) slices_dict = self.__filter_invalid_geo_bounds(slices_dict) if conv.coverage_slices == {}: conv.coverage_slices = slices_dict else: for key, val in slices_dict.items(): conv.coverage_slices[key] += slices_dict[key] if len(conv.coverage_slices) > 0 and len( conv.coverage_slices["base"]) > 0: first_slice = conv.coverage_slices["base"][0] # This needs one available file to extract metadata later conv.files = [first_slice.data_provider.file] return convertors
def _get_coverage(self): """ Returns the coverage to be used for the importer """ gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath()) crs = CRSUtil.get_compound_crs( [gdal_dataset.get_crs(), self.options['time_crs']]) slices = self._get_slices(crs) fields = GdalRangeFieldsGenerator( gdal_dataset, self.options['band_names']).get_range_fields() coverage = Coverage(self.session.get_coverage_id(), slices, fields, crs, gdal_dataset.get_band_gdal_type(), self.options['tiling']) return coverage
def get_crs(self): """ Returns the CRS associated with this dataset. If none is found the default for the session is returned :rtype: str """ import osgeo.osr as osr wkt = self.gdal_dataset.GetProjection() spatial_ref = osr.SpatialReference() spatial_ref.ImportFromWkt(wkt) crs = ConfigManager.default_crs if spatial_ref.GetAuthorityName(None) is not None: crs = CRSUtil.get_crs_url(spatial_ref.GetAuthorityName(None), spatial_ref.GetAuthorityCode(None)) return crs
def to_coverage(self): """ Returns a Coverage from all the importing files (gdal|grib|netcdf) :rtype: Coverage """ crs_axes = CRSUtil(self.crs).get_axes() slices = self._slices(crs_axes) # generate coverage extra_metadata from ingredient files metadata = self._metadata(slices) # Evaluate all the swe bands's metadata (each file should have same swe bands's metadata), so first file is ok self._evaluate_swe_bands_metadata(self.files[0], self.bands) coverage = Coverage(self.coverage_id, slices, self._range_fields(), self.crs, self._data_type(), self.tiling, metadata) return coverage
def get_crs(self): """ Returns the CRS associated with this dataset. If none is found the default for the session is returned :rtype: str """ wkt = self.gdal_dataset.GetProjection() spatial_ref = self._get_spatial_ref(wkt) crs = ConfigManager.default_crs if spatial_ref.GetAuthorityName(None) is not None: crs = CRSUtil.get_crs_url(spatial_ref.GetAuthorityName(None), spatial_ref.GetAuthorityCode(None)) if crs is None: raise RuntimeException("Cannot implicitly detect EPSG code from WKT of input file. " "Please explicitly specify the CRS in the ingredient (option \"default_crs\").") return crs
def get_crs(self): """ Returns the CRS associated with this dataset. If none is found the default for the session is returned :rtype: str """ import osgeo.osr as osr wkt = self.gdal_dataset.GetProjection() spatial_ref = osr.SpatialReference() spatial_ref.ImportFromWkt(wkt) crs = ConfigManager.default_crs if spatial_ref.GetAuthorityName(None) is not None: crs = CRSUtil.get_crs_url(spatial_ref.GetAuthorityName(None), spatial_ref.GetAuthorityCode(None)) return crs
def _get_coverage_slices(self, crs, gdal_coverage_converter): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices = [] timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: file_path = tpair.file.get_filepath() # NOTE: don't process any imported file from *.resume.json as it is just waisted time if not self.resumer.is_file_imported(file_path): timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), file_path) if not FileUtil.validate_file_path(file_path): continue valid_coverage_slice = True try: subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file_path)).fill(True) subsets = self._fill_time_axis(tpair, subsets) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file_path, ex) valid_coverage_slice = False if valid_coverage_slice: # Generate local metadata string for current coverage slice self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, tpair.file) local_metadata = gdal_coverage_converter._generate_local_metadata( subsets, self.evaluator_slice) slices.append( Slice(subsets, FileDataProvider(tpair.file), local_metadata)) timer.print_elapsed_time() count += 1 return slices
def to_coverages(self, coverage_slices_dict=None): """ Returns list coverages from all the importing files (gdal|grib|netcdf) :rtype: Array[Coverage] """ crs_axes = CRSUtil(self.crs).get_axes(self.coverage_id) if coverage_slices_dict is None: # Build list of coverage slices from input files coverage_slices_dict = self._create_coverage_slices(crs_axes) global_metadata = None if len(coverage_slices_dict) > 0: for coverage_level, slices in coverage_slices_dict.items(): if len(slices) > 0: first_coverage_slice = slices[0] break # generate coverage extra_metadata from ingredient file based on first input file of first coverage slice. global_metadata = self._generate_global_metadata( first_coverage_slice) # Evaluate all the swe bands's metadata (each file should have same swe bands's metadata), so first file is ok self._evaluate_swe_bands_metadata(self.files[0], self.bands) results = [] base_coverage_id = self.coverage_id for key, value in coverage_slices_dict.items(): slices = coverage_slices_dict[key] if key == "base": coverage = Coverage(base_coverage_id, slices, self._range_fields(), self.crs, self._data_type(), self.tiling, global_metadata) else: # overview coverage (key = overview_index) coverage_id = create_coverage_id_for_overview( self.coverage_id, key) coverage = Coverage(coverage_id, slices, self._range_fields(), self.crs, self._data_type(), self.tiling, global_metadata, base_coverage_id, key) results.append(coverage) return results
def _get_slices(self, gdal_dataset): """ Returns the slices for the collection of files given """ files = self.session.get_files() crs = gdal_dataset.get_crs() crs_axes = CRSUtil(crs).get_axes() slices = [] count = 1 for file in files: # print which file is analyzing FileUtil.print_feedback(count, len(files), file.filepath) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill() slices.append(Slice(subsets, FileDataProvider(file))) count += 1 return slices
def _get_slices(self, crs): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes() slices = [] timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), tpair.file.filepath) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil( tpair.file.get_filepath())).fill() subsets = self._fill_time_axis(tpair, subsets) slices.append(Slice(subsets, FileDataProvider(tpair.file))) count += 1 return slices
def _get_coverage(self): # Get the crs of one of the images using a GDAL helper class. We are assuming all images have the same CRS for file in self.session.get_files(): try: file_path = file.get_filepath() gdal_dataset = GDALGmlUtil(file_path) break except Exception as e: if ConfigManager.skip == True: pass else: raise e # Get the crs of the coverage by compounding the two crses crs = CRSUtil.get_compound_crs([gdal_dataset.get_crs(), self.options['time_crs']]) fields = GdalRangeFieldsGenerator(gdal_dataset).get_range_fields() pixel_type = gdal_dataset.get_band_gdal_type() coverage_id = self.session.get_coverage_id() slices = self._get_slices(crs) return Coverage(coverage_id, slices, fields, crs, pixel_type)
def _get_convertors(self): """ Returns a map of coverage id -> GdalToCoverageConverter """ convertors = {} band_data_type = self.DEFAULT_BAND_DATA_TYPE if self.product == self.SLC_PRODUCT: band_data_type = self.SLC_BAND_DATA_TYPE for file in self.session.get_files(): # Check if this file still exists when preparing to import if not FileUtil.validate_file_path(file.get_filepath()): continue # Check if this file belongs to this coverage id modebeam, polarisation = self._get_modebeam_polarisation( file.filepath) cov_id = self._get_coverage_id(self.coverage_id, modebeam, polarisation) # This file already imported in coverage_id.resume.json self.resumer = Resumer(cov_id) if self.resumer.is_file_imported(file.filepath): continue conv = self._get_convertor(convertors, cov_id) file_pair = FilePair(file.filepath, file.filepath) conv.files = [file_pair] crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id) # Different file contains different datetime from its name evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( GdalToCoverageConverter.RECIPE_TYPE, file) conv.data_type = band_data_type slices = conv._create_coverage_slices(crs_axes, evaluator_slice) conv.coverage_slices += slices return convertors
def _read(self): """ Reads the metadata from the describe coverage and creates the virtual coverage object :rtype: Coverage """ try: xmlstr = self.description() root = etree.fromstring(xmlstr) crs = self._get_crs(root) crs_axes = CRSUtil(crs).get_axes(self.coverage_id) range_type = self._get_range_types(root) raster_coords = self._get_raster_coords(root) geo_coords = self._get_geo_coords(root) coverage_id = self._get_coverage_id(root) resolutions = self._get_resolutions(root, crs_axes) origin = self._get_origin(root) coverage_axes = self._get_coverage_axes(geo_coords, raster_coords, origin, crs_axes, resolutions) intervals = self._get_intervals(coverage_axes, self.partitioning_scheme) slices = self._get_slices(coverage_axes, intervals) pixel_data_type, generated_file_path = self._get_data_type( slices[0]) coverage = Coverage(coverage_id, slices, range_type, crs, pixel_data_type) self.coverage = coverage except IOError as e: os.remove(generated_file_path) raise RuntimeException( "Could not read the coverage description for url: " + self._get_description_url() + ". Check that the url is accessible and try again. More details: " + str(e)) except XMLSyntaxError as e: os.remove(generated_file_path) raise RuntimeException( "Could not decode the xml description for url " + self._get_description_url() + ". Check that the url is correct and try again. More details: " + str(e))
def _get_coverage(self): """ Returns the coverage to be used for the importer """ gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file( self.session.get_files()) crs = CRSUtil.get_compound_crs( [self.options['time_crs'], gdal_dataset.get_crs()]) general_recipe = GeneralRecipe(self.session) global_metadata_fields = general_recipe._global_metadata_fields() local_metadata_fields = general_recipe._local_metadata_fields() sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory()) gdal_coverage_converter = GdalToCoverageConverter( self.resumer, self.session.get_default_null_values(), self.recipe_type, sentence_evaluator, self.session.get_coverage_id(), None, self.session.get_files(), crs, None, None, global_metadata_fields, local_metadata_fields, None, None, general_recipe._metadata_type(), None, None) coverage_slices = self._get_coverage_slices(crs, gdal_coverage_converter) fields = GdalRangeFieldsGenerator( gdal_dataset, self.options['band_names']).get_range_fields() global_metadata = None if len(coverage_slices) > 0: global_metadata = gdal_coverage_converter._generate_global_metadata( coverage_slices[0], self.evaluator_slice) coverage = Coverage(self.session.get_coverage_id(), coverage_slices, fields, crs, gdal_dataset.get_band_gdal_type(), self.options['tiling'], global_metadata) return coverage
def to_coverage(self, coverage_slices=None): """ Returns a Coverage from all the importing files (gdal|grib|netcdf) :rtype: Coverage """ crs_axes = CRSUtil(self.crs).get_axes(self.coverage_id) if coverage_slices is None: # Build list of coverage slices from input files coverage_slices = self._create_coverage_slices(crs_axes) global_metadata = None if len(coverage_slices) > 0: first_coverage_slice = coverage_slices[0] # generate coverage extra_metadata from ingredient file based on first input file of first coverage slice. global_metadata = self._generate_global_metadata(first_coverage_slice) # Evaluate all the swe bands's metadata (each file should have same swe bands's metadata), so first file is ok self._evaluate_swe_bands_metadata(self.files[0], self.bands) coverage = Coverage(self.coverage_id, coverage_slices, self._range_fields(), self.crs, self._data_type(), self.tiling, global_metadata) return coverage
def get_running_crs_resolver(self, crs_resolvers, embedded_petascope_port): """ From a list of SECORE configured in petascope.properties, find the first running SECORE to be used for wcst_import :param string[] crs_resolvers: list of SECORE URLs :return: string (the running SECORE) """ i = 0 crs_resolvers_tmp = [] for url_prefix in crs_resolvers: url_prefix = url_prefix.strip() # NOTE: if secore_urls=internal in petascope.properties, it means # wcst_import will use the internal SECORE inside petascope with the sub endpoint at /rasdaman/def if url_prefix == INTERNAL_SECORE or url_prefix == DEFAULT_SECORE_URL: url_prefix = INTERNAL_SECORE_URL crs_resolvers_tmp.append(url_prefix) # Also, in case petascope runs at different port than 8080 if embedded_petascope_port != "8080": url_prefix = "http://localhost:" + embedded_petascope_port + INTERNAL_SECORE_URL_CONTEXT_PATH crs_resolvers_tmp.append(url_prefix) else: crs_resolvers_tmp.append(url_prefix) for url_prefix in crs_resolvers_tmp: try: url_prefix = url_prefix.strip() test_url = url_prefix + "/crs/EPSG/0/4326" from util.crs_util import CRSUtil Session.RUNNING_SECORE_URL = url_prefix CRSUtil.get_axis_labels_from_single_crs(test_url) # e.g: http://localhost:8080/def return url_prefix except Exception as ex: log.warn("CRS resolver '" + url_prefix + "' is not working.") if i < len(crs_resolvers) - 1: log.warn("Trying with another fallback resolver...") i += 1 # none of resolvers work, then assume there is SECORE embedded in petascope internal_secore_url = self.wcs_service.replace( "/rasdaman/ows", "/rasdaman/def").strip() try: test_url = internal_secore_url + "/crs/EPSG/0/4326" from util.crs_util import CRSUtil Session.RUNNING_SECORE_URL = internal_secore_url CRSUtil.get_axis_labels_from_single_crs(test_url) log.warn( "None of the configured secore_urls in petascope.properties respond to requests. " "wcst_import will use the internal CRS resolver at endpoint '" + internal_secore_url + "'. " "Hint: set secore_urls=internal in petascope.properties to suppress this warning." ) return internal_secore_url except Exception as ex: raise RuntimeException( "No configured CRS resolvers in petascope.properties work. Given: " + ",".join(crs_resolvers))
def _read_axes(self, crs): """ Returns a list of user axes extracted from the ingredients file :param str crs: the crs of the coverage :rtype: list[UserAxis] """ axes_configurations = self.options['coverage']['slicer']['axes'] user_axes = [] crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id, axes_configurations) default_order = 0 for index, crs_axis in enumerate(crs_axes): exist = False for axis_label, axis_configuration_dicts in axes_configurations.items(): # If axis label configured in ingredient file does not exist in CRS definition, # then "crsOrder" configuration must match with the crs axis order. if crs_axis.label == axis_label \ or ("crsOrder" in axis_configuration_dicts and int(axis_configuration_dicts["crsOrder"]) == index): crs_axes[index].label = axis_label exist = True break if not exist: raise RecipeValidationException( "Could not find a definition for axis '" + crs_axis.label + "' in the axes parameter.") axis = axes_configurations[crs_axis.label] max = axis["max"] if "max" in axis else None if "type" in axis: type = axis["type"] elif crs_axis.is_date_axis(): type = UserAxisType.DATE else: type = UserAxisType.NUMBER order = axis["gridOrder"] if "gridOrder" in axis else default_order irregular = axis["irregular"] if "irregular" in axis else False data_bound = axis["dataBound"] if "dataBound" in axis else True # for irregular axes we consider the resolution 1 / -1 as gmlcov requires resolution for all axis types, # even irregular if "resolution" in axis: resolution = axis["resolution"] else: resolution = 1 default_order += 1 if "statements" in axis: if isinstance(axis["statements"], list): statements = axis["statements"] else: statements = [axis["statements"]] else: statements = [] slice_group_size = None if "sliceGroupSize" in axis: if not irregular: raise RuntimeException("Cannot set 'sliceGroupSize' for regular axis '{}' in ingredient file.".format(crs_axis.label)) else: # Irregular axis with dataBound:false only can use sliceGroupSize (!) value_str = axis["sliceGroupSize"] if "dataBound" not in axis or axis["dataBound"] is True: raise RuntimeException("Option 'sliceGroupSize' can be set only for irregular axis '{}'" " with \"dataBound\": false in ingredient file.".format(crs_axis.label)) try: slice_group_size = float(value_str) if slice_group_size <= 0: raise ValueError except ValueError: raise RuntimeException("Option 'sliceGroupSize' for irregular axis '{}'" " in ingredient file must be positive number. Given '{}'.".format(crs_axis.label, value_str)) if not irregular: user_axes.append( RegularUserAxis(crs_axis.label, resolution, order, axis["min"], max, type, data_bound, statements)) else: # NOTE: irregular axis cannot set any resolution != 1 if int(resolution) != IrregularUserAxis.DEFAULT_RESOLUTION: raise RuntimeException("Cannot set 'resolution' value for irregular axis '{}' in ingredient file." " Given '{}'.".format(crs_axis.label, resolution)) user_axes.append( IrregularUserAxis(crs_axis.label, resolution, order, axis["min"], axis["directPositions"], max, type, data_bound, statements, slice_group_size)) number_of_specified_axes = len(axes_configurations.items()) number_of_crs_axes = len(crs_axes) if number_of_specified_axes != number_of_crs_axes: raise RuntimeException("Number of axes in the coverage CRS ({}) does not match " "the number of axes specified in the ingredients file ({}).".format(number_of_crs_axes, number_of_specified_axes)) return user_axes
def _get_convertors(self): """ Returns a map of coverage id -> GdalToCoverageConverter """ convertors = {} for f in self.session.get_files(): # This one does not contain any information for geo bounds if not FileUtil.validate_file_path(f.get_filepath()): continue gdal_ds = GDALGmlUtil(f.get_filepath()) subdatasets = self._get_subdatasets(gdal_ds, f) gdal_ds.close() level = self._get_level(f.get_filepath()) if len(self.levels) > 0 and level not in self.levels: # skip file, as it's not in the list of levels provided in the ingredients file log.debug("Skipping " + level + " data") continue crs_code = "" evaluator_slice = None for res in self.resolutions: subds_file = self._get_subdataset_file(subdatasets, res) crs_code = self._get_crs_code(subds_file.get_filepath(), crs_code) if len(self.crss) > 0 and crs_code not in self.crss: # skip CRS, it's not in the list of CRSs provided in the ingredients file log.debug("Skipping data with CRS " + crs_code) continue cov_id = self._get_coverage_id(self.coverage_id, crs_code, level, res) conv = self._get_convertor(convertors, cov_id, crs_code, level, res) file_pair = FilePair(subds_file.filepath, f.filepath) conv.files = [file_pair] crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id) if evaluator_slice is None: # This one contains information for geo bounds evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( GdalToCoverageConverter.RECIPE_TYPE, subds_file) # Resolution 10m, 20m and 60m have same data type (UInt16) while TCI has data type (Byte) if res == self.RES_TCI: conv.data_type = "Byte" else: conv.data_type = "UInt16" # Fixed values for 3 axes of Sentinel 2 coverage axis_resolutions = self.RES_DICT[res] slices_dict = conv._create_coverage_slices( crs_axes, evaluator_slice, axis_resolutions) if conv.coverage_slices == {}: conv.coverage_slices = slices_dict else: for key, val in slices_dict.items(): conv.coverage_slices[key] += slices_dict[key] if len(conv.coverage_slices) != 0: first_slice = conv.coverage_slices["base"][0] # This needs one available file to extract metadata later conv.files = [first_slice.data_provider.file] return convertors