def _get_slices(self, crs): """ Returns the slices for the collection of files given """ files = self.session.get_files() crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices = [] count = 1 for file in files: # NOTE: don't process any imported file from *.resume.json as it is just waisted time if not self.resumer.is_file_imported(file.filepath): timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(files), file.filepath) if not FileUtil.validate_file_path(file.filepath): continue valid_coverage_slice = True try: subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill() except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible(file.get_filepath(), ex) valid_coverage_slice = False if valid_coverage_slice: slices.append(Slice(subsets, FileDataProvider(file))) timer.print_elapsed_time() count += 1 return slices
def _get_slices(self, crs): # Let's first extract all the axes from our crs crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) # Prepare a list container for our slices slices = [] # Iterate over the files and create a slice for each one for infile in self.session.get_files(): # We need to create the exact position in time and space in which to place this slice # For the space coordinates we can use the GDAL helper to extract it for us # The helper will return a list of subsets based on the crs axes that we extracted # and will fill the coordinates for the ones that it can (the easting and northing axes) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil( infile.get_filepath())).fill() # Now we must fill the time axis as well and indicate the position in time for subset in subsets: # Find the time axis if subset.coverage_axis.axis.crs_axis.is_time_axis(): # Set the time position for it. Our recipe extracts it from a GDAL tag provided by the user # datetime format needs enquoted (e.g: "2015-01") subset.interval.low = '"' + GDALGmlUtil( infile).get_datetime(self.options["time_tag"]) + '"' slices.append(Slice(subsets, FileDataProvider(infile))) return slices
def _get_coverage_slices(self, crs, gdal_coverage_converter): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices_dict = self.create_dict_of_slices(self.session.import_overviews) timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: file = tpair.file file_path = tpair.file.get_filepath() timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), file_path) if not FileUtil.validate_file_path(file_path): continue valid_coverage_slice = True gdal_file = GDALGmlUtil(file.get_filepath()) try: subsets = GdalAxisFiller(crs_axes, gdal_file).fill(True) subsets = self._fill_time_axis(tpair, subsets) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file_path, ex) valid_coverage_slice = False if valid_coverage_slice: # Generate local metadata string for current coverage slice self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, tpair.file) local_metadata = gdal_coverage_converter._generate_local_metadata( subsets, self.evaluator_slice) if self.session.import_overviews_only is False: slices_dict["base"].append( Slice(subsets, FileDataProvider(tpair.file), local_metadata)) # Then, create slices for selected overviews from user for overview_index in self.session.import_overviews: subsets_overview = self.create_subsets_for_overview( subsets, overview_index, gdal_file) slices_dict[str(overview_index)].append( Slice(subsets_overview, FileDataProvider(file), local_metadata)) timer.print_elapsed_time() count += 1 return slices_dict
def _get_update_crs(self, slice, crs): """ Returns the crs corresponding to the axes that are data bound :param slice: the slice for which the gml should be created :param crs: the crs of the coverage :return: String """ crsAxes = [] for axis_subset in slice.axis_subsets: if axis_subset.coverage_axis.data_bound: crsAxes.append(axis_subset.coverage_axis.axis.crs_axis) crsUtil = CRSUtil(crs) return crsUtil.get_crs_for_axes(crsAxes)
def _get_crs_axis_by_user_axis_name(self, user_axis_name): """ Returns the crs axis from the list by user_axis_name :param user_axis_name: :return: crs_axis """ crs_axes = CRSUtil(self.crs).get_axes(self.coverage_id) for i in range(0, len(crs_axes)): crs_axis = crs_axes[i] if crs_axis.label == user_axis_name: return crs_axis
def _read_axes(self, crs): """ Returns a list of user axes extracted from the ingredients file :param str crs: the crs of the coverage :rtype: list[UserAxis] """ axes = self.options['coverage']['slicer']['axes'] user_axes = [] crs_axes = CRSUtil(crs).get_axes() default_order = 0 for crs_axis in crs_axes: if crs_axis.label not in axes: raise RecipeValidationException( "Could not find a definition for axis " + crs_axis.label + " in the axes parameter.") axis = axes[crs_axis.label] max = axis["max"] if "max" in axis else None if "type" in axis: type = axis["type"] elif crs_axis.is_date(): type = UserAxisType.DATE else: type = UserAxisType.NUMBER order = axis["gridOrder"] if "gridOrder" in axis else default_order irregular = axis["irregular"] if "irregular" in axis else False data_bound = axis["dataBound"] if "dataBound" in axis else True # for irregular axes we consider the resolution 1 / -1 as gmlcov requires resolution for all axis types, # even irregular if "resolution" in axis: resolution = axis["resolution"] else: resolution = 1 default_order += 1 if "statements" in axis: if isinstance(axis["statements"], list): statements = axis["statements"] else: statements = [axis["statements"]] else: statements = [] if not irregular: user_axes.append( RegularUserAxis(crs_axis.label, resolution, order, axis["min"], max, type, data_bound, statements=statements)) else: user_axes.append( IrregularUserAxis(crs_axis.label, resolution, order, axis["min"], axis["directPositions"], max, type, data_bound, statements=statements)) return user_axes
def _get_convertors(self): """ Returns a map of coverage id -> GdalToCoverageConverter """ convertors = {} band_data_type = self.DEFAULT_BAND_DATA_TYPE if self.product == self.SLC_PRODUCT: band_data_type = self.SLC_BAND_DATA_TYPE for file in self.session.get_files(): # Check if this file still exists when preparing to import if not FileUtil.validate_file_path(file.get_filepath()): continue # Check if this file belongs to this coverage id modebeam, polarisation = self._get_modebeam_polarisation( file.filepath) cov_id = self._get_coverage_id(self.coverage_id, modebeam, polarisation) conv = self._get_convertor(convertors, cov_id) file_pair = FilePair(file.filepath, file.filepath) conv.files = [file_pair] crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id) # Different file contains different datetime from its name evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( GdalToCoverageConverter.RECIPE_TYPE, file) conv.data_type = band_data_type slices_dict = conv._create_coverage_slices(crs_axes, evaluator_slice) slices_dict = self.__filter_invalid_geo_bounds(slices_dict) if conv.coverage_slices == {}: conv.coverage_slices = slices_dict else: for key, val in slices_dict.items(): conv.coverage_slices[key] += slices_dict[key] if len(conv.coverage_slices) > 0 and len( conv.coverage_slices["base"]) > 0: first_slice = conv.coverage_slices["base"][0] # This needs one available file to extract metadata later conv.files = [first_slice.data_provider.file] return convertors
def to_coverage(self): """ Returns a Coverage from all the importing files (gdal|grib|netcdf) :rtype: Coverage """ crs_axes = CRSUtil(self.crs).get_axes() slices = self._slices(crs_axes) # generate coverage extra_metadata from ingredient files metadata = self._metadata(slices) # Evaluate all the swe bands's metadata (each file should have same swe bands's metadata), so first file is ok self._evaluate_swe_bands_metadata(self.files[0], self.bands) coverage = Coverage(self.coverage_id, slices, self._range_fields(), self.crs, self._data_type(), self.tiling, metadata) return coverage
def _get_coverage_slices(self, crs, gdal_coverage_converter): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices = [] timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: file_path = tpair.file.get_filepath() # NOTE: don't process any imported file from *.resume.json as it is just waisted time if not self.resumer.is_file_imported(file_path): timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), file_path) if not FileUtil.validate_file_path(file_path): continue valid_coverage_slice = True try: subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file_path)).fill(True) subsets = self._fill_time_axis(tpair, subsets) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file_path, ex) valid_coverage_slice = False if valid_coverage_slice: # Generate local metadata string for current coverage slice self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, tpair.file) local_metadata = gdal_coverage_converter._generate_local_metadata( subsets, self.evaluator_slice) slices.append( Slice(subsets, FileDataProvider(tpair.file), local_metadata)) timer.print_elapsed_time() count += 1 return slices
def _get_slices(self, gdal_dataset): """ Returns the slices for the collection of files given """ files = self.session.get_files() crs = gdal_dataset.get_crs() crs_axes = CRSUtil(crs).get_axes() slices = [] count = 1 for file in files: # print which file is analyzing FileUtil.print_feedback(count, len(files), file.filepath) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill() slices.append(Slice(subsets, FileDataProvider(file))) count += 1 return slices
def to_coverages(self, coverage_slices_dict=None): """ Returns list coverages from all the importing files (gdal|grib|netcdf) :rtype: Array[Coverage] """ crs_axes = CRSUtil(self.crs).get_axes(self.coverage_id) if coverage_slices_dict is None: # Build list of coverage slices from input files coverage_slices_dict = self._create_coverage_slices(crs_axes) global_metadata = None if len(coverage_slices_dict) > 0: for coverage_level, slices in coverage_slices_dict.items(): if len(slices) > 0: first_coverage_slice = slices[0] break # generate coverage extra_metadata from ingredient file based on first input file of first coverage slice. global_metadata = self._generate_global_metadata( first_coverage_slice) # Evaluate all the swe bands's metadata (each file should have same swe bands's metadata), so first file is ok self._evaluate_swe_bands_metadata(self.files[0], self.bands) results = [] base_coverage_id = self.coverage_id for key, value in coverage_slices_dict.items(): slices = coverage_slices_dict[key] if key == "base": coverage = Coverage(base_coverage_id, slices, self._range_fields(), self.crs, self._data_type(), self.tiling, global_metadata) else: # overview coverage (key = overview_index) coverage_id = create_coverage_id_for_overview( self.coverage_id, key) coverage = Coverage(coverage_id, slices, self._range_fields(), self.crs, self._data_type(), self.tiling, global_metadata, base_coverage_id, key) results.append(coverage) return results
def _get_slices(self, crs): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes() slices = [] timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), tpair.file.filepath) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil( tpair.file.get_filepath())).fill() subsets = self._fill_time_axis(tpair, subsets) slices.append(Slice(subsets, FileDataProvider(tpair.file))) count += 1 return slices
def _get_convertors(self): """ Returns a map of coverage id -> GdalToCoverageConverter """ convertors = {} band_data_type = self.DEFAULT_BAND_DATA_TYPE if self.product == self.SLC_PRODUCT: band_data_type = self.SLC_BAND_DATA_TYPE for file in self.session.get_files(): # Check if this file still exists when preparing to import if not FileUtil.validate_file_path(file.get_filepath()): continue # Check if this file belongs to this coverage id modebeam, polarisation = self._get_modebeam_polarisation( file.filepath) cov_id = self._get_coverage_id(self.coverage_id, modebeam, polarisation) # This file already imported in coverage_id.resume.json self.resumer = Resumer(cov_id) if self.resumer.is_file_imported(file.filepath): continue conv = self._get_convertor(convertors, cov_id) file_pair = FilePair(file.filepath, file.filepath) conv.files = [file_pair] crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id) # Different file contains different datetime from its name evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( GdalToCoverageConverter.RECIPE_TYPE, file) conv.data_type = band_data_type slices = conv._create_coverage_slices(crs_axes, evaluator_slice) conv.coverage_slices += slices return convertors
def __init__(self, datetime, dt_format=None, time_crs=None): """ :param str datetime: the datetime value :param str dt_format: the datetime format, if none is given we'll try to guess """ self.init_cache() try: if dt_format is None: self.datetime = arrow.get(datetime) else: self.datetime = arrow.get(datetime, dt_format) if time_crs is None: self.time_crs_code = self.CRS_CODE_ANSI_DATE else: tmp_crs = CRSUtil(time_crs) self.time_crs_code = tmp_crs.get_crs_code() except ParserError as pe: dt_format_err = "auto" if dt_format is None else dt_format raise RuntimeException("Failed to parse the date " + datetime + " using format " + dt_format_err)
def _read(self): """ Reads the metadata from the describe coverage and creates the virtual coverage object :rtype: Coverage """ try: xmlstr = self.description() root = etree.fromstring(xmlstr) crs = self._get_crs(root) crs_axes = CRSUtil(crs).get_axes(self.coverage_id) range_type = self._get_range_types(root) raster_coords = self._get_raster_coords(root) geo_coords = self._get_geo_coords(root) coverage_id = self._get_coverage_id(root) resolutions = self._get_resolutions(root, crs_axes) origin = self._get_origin(root) coverage_axes = self._get_coverage_axes(geo_coords, raster_coords, origin, crs_axes, resolutions) intervals = self._get_intervals(coverage_axes, self.partitioning_scheme) slices = self._get_slices(coverage_axes, intervals) pixel_data_type, generated_file_path = self._get_data_type( slices[0]) coverage = Coverage(coverage_id, slices, range_type, crs, pixel_data_type) self.coverage = coverage except IOError as e: os.remove(generated_file_path) raise RuntimeException( "Could not read the coverage description for url: " + self._get_description_url() + ". Check that the url is accessible and try again. More details: " + str(e)) except XMLSyntaxError as e: os.remove(generated_file_path) raise RuntimeException( "Could not decode the xml description for url " + self._get_description_url() + ". Check that the url is correct and try again. More details: " + str(e))
def to_coverage(self, coverage_slices=None): """ Returns a Coverage from all the importing files (gdal|grib|netcdf) :rtype: Coverage """ crs_axes = CRSUtil(self.crs).get_axes(self.coverage_id) if coverage_slices is None: # Build list of coverage slices from input files coverage_slices = self._create_coverage_slices(crs_axes) global_metadata = None if len(coverage_slices) > 0: first_coverage_slice = coverage_slices[0] # generate coverage extra_metadata from ingredient file based on first input file of first coverage slice. global_metadata = self._generate_global_metadata(first_coverage_slice) # Evaluate all the swe bands's metadata (each file should have same swe bands's metadata), so first file is ok self._evaluate_swe_bands_metadata(self.files[0], self.bands) coverage = Coverage(self.coverage_id, coverage_slices, self._range_fields(), self.crs, self._data_type(), self.tiling, global_metadata) return coverage
def _get_convertors(self): """ Returns a map of coverage id -> GdalToCoverageConverter """ convertors = {} for f in self.session.get_files(): # This one does not contain any information for geo bounds if not FileUtil.validate_file_path(f.get_filepath()): continue gdal_ds = GDALGmlUtil(f.get_filepath()) subdatasets = self._get_subdatasets(gdal_ds, f) gdal_ds.close() level = self._get_level(f.get_filepath()) if len(self.levels) > 0 and level not in self.levels: # skip file, as it's not in the list of levels provided in the ingredients file log.debug("Skipping " + level + " data") continue crs_code = "" evaluator_slice = None for res in self.resolutions: subds_file = self._get_subdataset_file(subdatasets, res) crs_code = self._get_crs_code(subds_file.get_filepath(), crs_code) if len(self.crss) > 0 and crs_code not in self.crss: # skip CRS, it's not in the list of CRSs provided in the ingredients file log.debug("Skipping data with CRS " + crs_code) continue cov_id = self._get_coverage_id(self.coverage_id, crs_code, level, res) conv = self._get_convertor(convertors, cov_id, crs_code, level, res) file_pair = FilePair(subds_file.filepath, f.filepath) conv.files = [file_pair] crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id) if evaluator_slice is None: # This one contains information for geo bounds evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( GdalToCoverageConverter.RECIPE_TYPE, subds_file) # Resolution 10m, 20m and 60m have same data type (UInt16) while TCI has data type (Byte) if res == self.RES_TCI: conv.data_type = "Byte" else: conv.data_type = "UInt16" # Fixed values for 3 axes of Sentinel 2 coverage axis_resolutions = self.RES_DICT[res] slices_dict = conv._create_coverage_slices( crs_axes, evaluator_slice, axis_resolutions) if conv.coverage_slices == {}: conv.coverage_slices = slices_dict else: for key, val in slices_dict.items(): conv.coverage_slices[key] += slices_dict[key] if len(conv.coverage_slices) != 0: first_slice = conv.coverage_slices["base"][0] # This needs one available file to extract metadata later conv.files = [first_slice.data_provider.file] return convertors
def _read_axes(self, crs): """ Returns a list of user axes extracted from the ingredients file :param str crs: the crs of the coverage :rtype: list[UserAxis] """ axes_configurations = self.options['coverage']['slicer']['axes'] user_axes = [] crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id, axes_configurations) default_order = 0 for index, crs_axis in enumerate(crs_axes): exist = False for axis_label, axis_configuration_dicts in axes_configurations.items(): # If axis label configured in ingredient file does not exist in CRS definition, # then "crsOrder" configuration must match with the crs axis order. if crs_axis.label == axis_label \ or ("crsOrder" in axis_configuration_dicts and int(axis_configuration_dicts["crsOrder"]) == index): crs_axes[index].label = axis_label exist = True break if not exist: raise RecipeValidationException( "Could not find a definition for axis '" + crs_axis.label + "' in the axes parameter.") axis = axes_configurations[crs_axis.label] max = axis["max"] if "max" in axis else None if "type" in axis: type = axis["type"] elif crs_axis.is_date_axis(): type = UserAxisType.DATE else: type = UserAxisType.NUMBER order = axis["gridOrder"] if "gridOrder" in axis else default_order irregular = axis["irregular"] if "irregular" in axis else False data_bound = axis["dataBound"] if "dataBound" in axis else True # for irregular axes we consider the resolution 1 / -1 as gmlcov requires resolution for all axis types, # even irregular if "resolution" in axis: resolution = axis["resolution"] else: resolution = 1 default_order += 1 if "statements" in axis: if isinstance(axis["statements"], list): statements = axis["statements"] else: statements = [axis["statements"]] else: statements = [] slice_group_size = None if "sliceGroupSize" in axis: if not irregular: raise RuntimeException("Cannot set 'sliceGroupSize' for regular axis '{}' in ingredient file.".format(crs_axis.label)) else: # Irregular axis with dataBound:false only can use sliceGroupSize (!) value_str = axis["sliceGroupSize"] if "dataBound" not in axis or axis["dataBound"] is True: raise RuntimeException("Option 'sliceGroupSize' can be set only for irregular axis '{}'" " with \"dataBound\": false in ingredient file.".format(crs_axis.label)) try: slice_group_size = float(value_str) if slice_group_size <= 0: raise ValueError except ValueError: raise RuntimeException("Option 'sliceGroupSize' for irregular axis '{}'" " in ingredient file must be positive number. Given '{}'.".format(crs_axis.label, value_str)) if not irregular: user_axes.append( RegularUserAxis(crs_axis.label, resolution, order, axis["min"], max, type, data_bound, statements)) else: # NOTE: irregular axis cannot set any resolution != 1 if int(resolution) != IrregularUserAxis.DEFAULT_RESOLUTION: raise RuntimeException("Cannot set 'resolution' value for irregular axis '{}' in ingredient file." " Given '{}'.".format(crs_axis.label, resolution)) user_axes.append( IrregularUserAxis(crs_axis.label, resolution, order, axis["min"], axis["directPositions"], max, type, data_bound, statements, slice_group_size)) number_of_specified_axes = len(axes_configurations.items()) number_of_crs_axes = len(crs_axes) if number_of_specified_axes != number_of_crs_axes: raise RuntimeException("Number of axes in the coverage CRS ({}) does not match " "the number of axes specified in the ingredients file ({}).".format(number_of_crs_axes, number_of_specified_axes)) return user_axes