def _generate_timeseries_tuples(self, limit=None): """ Generate the timeseries tuples from the original files based on the recipe :rtype: list[TimeFileTuple] """ ret = [] if limit is None: limit = len(self.session.get_files()) time_format = None if 'datetime_format' in self.options['time_parameter']: time_format = self.options['time_parameter']['datetime_format'] if 'metadata_tag' in self.options['time_parameter']: mtag = self.options['time_parameter']['metadata_tag']['tag_name'] for tfile in self.session.get_files(): if len(ret) == limit: break gdal_file = GDALGmlUtil(tfile.get_filepath()) dtutil = DateTimeUtil(gdal_file.get_datetime(mtag), time_format, self.options['time_crs']) ret.append(TimeFileTuple(dtutil, tfile)) elif 'filename' in self.options['time_parameter'] and len(ret) < limit: regex = self.options['time_parameter']['filename']['regex'] group = int(self.options['time_parameter']['filename']['group']) for tfile in self.session.get_files(): if len(ret) == limit: break dtutil = DateTimeUtil(re.search(regex, tfile.filepath).group(group), time_format, self.options['time_crs']) ret.append(TimeFileTuple(dtutil, tfile)) else: raise RecipeValidationException("No method to get the time parameter, you should either choose " "metadata_tag or filename.") return sorted(ret)
def _get_slices(self, crs): # Let's first extract all the axes from our crs crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) # Prepare a list container for our slices slices = [] # Iterate over the files and create a slice for each one for infile in self.session.get_files(): # We need to create the exact position in time and space in which to place this slice # For the space coordinates we can use the GDAL helper to extract it for us # The helper will return a list of subsets based on the crs axes that we extracted # and will fill the coordinates for the ones that it can (the easting and northing axes) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil( infile.get_filepath())).fill() # Now we must fill the time axis as well and indicate the position in time for subset in subsets: # Find the time axis if subset.coverage_axis.axis.crs_axis.is_time_axis(): # Set the time position for it. Our recipe extracts it from a GDAL tag provided by the user # datetime format needs enquoted (e.g: "2015-01") subset.interval.low = '"' + GDALGmlUtil( infile).get_datetime(self.options["time_tag"]) + '"' slices.append(Slice(subsets, FileDataProvider(infile))) return slices
def _get_coverage(self): """ Returns the coverage to be used for the importer """ gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath()) slices = self._get_slices(gdal_dataset) fields = GdalRangeFieldsGenerator(gdal_dataset, self.options['band_names']).get_range_fields() coverage = Coverage(self.session.get_coverage_id(), slices, fields, gdal_dataset.get_crs(), gdal_dataset.get_band_gdal_type(), self.options['tiling']) return coverage
def _generate_timeseries_tuples(self, limit=None): """ Generate the timeseries tuples from the original files based on the recipe. And sort the files in order of time. :rtype: list[TimeFileTuple] """ ret = [] if limit is None: limit = len(self.session.get_files()) time_format = None if 'datetime_format' in self.options['time_parameter']: time_format = self.options['time_parameter']['datetime_format'] if 'metadata_tag' in self.options['time_parameter']: mtag = self.options['time_parameter']['metadata_tag']['tag_name'] for tfile in self.session.get_files(): if len(ret) == limit: break valid_file = True try: gdal_file = GDALGmlUtil(tfile.get_filepath()) except Exception as ex: FileUtil.ignore_coverage_slice_from_file_if_possible( tfile.get_filepath(), ex) valid_file = False if valid_file: dtutil = DateTimeUtil(gdal_file.get_datetime(mtag), time_format, self.options['time_crs']) ret.append(TimeFileTuple(dtutil, tfile)) elif 'filename' in self.options['time_parameter'] and len(ret) < limit: regex = self.options['time_parameter']['filename']['regex'] group = int(self.options['time_parameter']['filename']['group']) for tfile in self.session.get_files(): if len(ret) == limit: break dtutil = DateTimeUtil( re.search(regex, tfile.filepath).group(group), time_format, self.options['time_crs']) ret.append(TimeFileTuple(dtutil, tfile)) else: raise RecipeValidationException( "No method to get the time parameter, you should either choose " "metadata_tag or filename.") # Currently, only sort by datetime to import coverage slices (default is ascending), option: to sort descending if self.options[ "import_order"] == AbstractToCoverageConverter.IMPORT_ORDER_DESCENDING: return sorted(ret, reverse=True) return sorted(ret)
def _get_coverage(self): # Get the crs of one of the images using a GDAL helper class. We are assuming all images have the same CRS gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath()) # Get the crs of the coverage by compounding the two crses crs = CRSUtil.get_compound_crs( [gdal_dataset.get_crs(), self.options['time_crs']]) fields = GdalRangeFieldsGenerator(gdal_dataset).get_range_fields() pixel_type = gdal_dataset.get_band_gdal_type() coverage_id = self.session.get_coverage_id() slices = self._get_slices(crs) return Coverage(coverage_id, slices, fields, crs, pixel_type)
def parse_gdal_global_metadata(file_path): """ Parse the first file of importing gdal files to extract the global metadata for the coverage str file_path: path to first gdal input file :return: dict: global_metadata """ # NOTE: all files should have same global metadata for each file gdal_dataset = GDALGmlUtil(file_path) global_metadata = gdal_dataset.get_metadata() return global_metadata
def _get_coverage(self): """ Returns the coverage to be used for the importer """ gdal_dataset = GDALGmlUtil(self.session.get_files()[0].get_filepath()) slices = self._get_slices(gdal_dataset) fields = GdalRangeFieldsGenerator( gdal_dataset, self.options['band_names']).get_range_fields() coverage = Coverage(self.session.get_coverage_id(), slices, fields, gdal_dataset.get_crs(), gdal_dataset.get_band_gdal_type(), self.options['tiling']) return coverage
def _init_epsg_xy_crs(self): """ From the first file of input file, detect its EPSG code for XY axes """ for file in self.session.get_files(): try: gdal_ds = GDALGmlUtil(file.get_filepath()) self.epsg_xy_crs = gdal_ds.get_crs() break except Exception as e: if ConfigManager.skip == True: pass else: raise e
def _resolve(self, expression, gdal_dataset): """ Resolves the expression in the context of the gdal dataset :param str expression: the expression to resolve :param util.gdal_util.GDALGmlUtil gdal_dataset: the dataset for which to resolve :return: """ if expression.startswith("metadata:"): value = gdal_dataset.get_metadata_tag( expression.replace("metadata:", "")) else: user_bands = [] for band in GDALGmlUtil.get_fields_range_type(gdal_dataset): user_band = UserBand(band.field_name, "", "", "", "", band.nill_values, band.uom_code) user_bands.append(user_band) gdal_dictionary = { "resolutionX": gdal_dataset.get_offset_vectors()[0], "resolutionY": gdal_dataset.get_offset_vectors()[1], "originX": gdal_dataset.get_origin_x(), "originY": gdal_dataset.get_origin_y(), "minX": gdal_dataset.get_extents_x()[0], "maxX": gdal_dataset.get_extents_x()[1], "minY": gdal_dataset.get_extents_y()[0], "maxY": gdal_dataset.get_extents_y()[1], "bands": user_bands } if expression in gdal_dictionary: value = gdal_dictionary[expression] else: raise RuntimeException( "Cannot evaluate the given grib expression {} on the given container." .format(str(expression))) return value
def _get_slices(self, crs): """ Returns the slices for the collection of files given """ files = self.session.get_files() crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices = [] count = 1 for file in files: # NOTE: don't process any imported file from *.resume.json as it is just waisted time if not self.resumer.is_file_imported(file.filepath): timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(files), file.filepath) if not FileUtil.validate_file_path(file.filepath): continue valid_coverage_slice = True try: subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill() except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible(file.get_filepath(), ex) valid_coverage_slice = False if valid_coverage_slice: slices.append(Slice(subsets, FileDataProvider(file))) timer.print_elapsed_time() count += 1 return slices
def _get_coverage(self): """ Returns the coverage to be used for the importer """ gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(self.session.get_files()) crs = gdal_dataset.get_crs() general_recipe = GeneralRecipe(self.session) global_metadata_fields = general_recipe._global_metadata_fields() local_metadata_fields = general_recipe._local_metadata_fields() sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory()) gdal_coverage_converter = GdalToCoverageConverter(self.resumer, self.session.get_default_null_values(), self.recipe_type, sentence_evaluator, self.session.get_coverage_id(), None, self.session.get_files(), crs, None, None, global_metadata_fields, local_metadata_fields, None, None, general_recipe._metadata_type(), None, None) coverage_slices = self._get_coverage_slices(crs, gdal_coverage_converter) fields = GdalRangeFieldsGenerator(gdal_dataset, self.options['band_names']).get_range_fields() global_metadata = None if len(coverage_slices) > 0: global_metadata = gdal_coverage_converter._generate_global_metadata(coverage_slices[0], self.evaluator_slice) coverage = Coverage(self.session.get_coverage_id(), coverage_slices, fields, gdal_dataset.get_crs(), gdal_dataset.get_band_gdal_type(), self.options['tiling'], global_metadata) return coverage
def _get_coverage_slices(self, crs, gdal_coverage_converter): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices_dict = self.create_dict_of_slices(self.session.import_overviews) timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: file = tpair.file file_path = tpair.file.get_filepath() timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), file_path) if not FileUtil.validate_file_path(file_path): continue valid_coverage_slice = True gdal_file = GDALGmlUtil(file.get_filepath()) try: subsets = GdalAxisFiller(crs_axes, gdal_file).fill(True) subsets = self._fill_time_axis(tpair, subsets) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file_path, ex) valid_coverage_slice = False if valid_coverage_slice: # Generate local metadata string for current coverage slice self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, tpair.file) local_metadata = gdal_coverage_converter._generate_local_metadata( subsets, self.evaluator_slice) if self.session.import_overviews_only is False: slices_dict["base"].append( Slice(subsets, FileDataProvider(tpair.file), local_metadata)) # Then, create slices for selected overviews from user for overview_index in self.session.import_overviews: subsets_overview = self.create_subsets_for_overview( subsets, overview_index, gdal_file) slices_dict[str(overview_index)].append( Slice(subsets_overview, FileDataProvider(file), local_metadata)) timer.print_elapsed_time() count += 1 return slices_dict
def _get_coverages(self): """ Returns the list of coverages to be used for the importer """ gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file( self.session.get_files()) crs = CRSUtil.get_compound_crs( [self.options['time_crs'], gdal_dataset.get_crs()]) general_recipe = GeneralRecipe(self.session) global_metadata_fields = general_recipe._global_metadata_fields() local_metadata_fields = general_recipe._local_metadata_fields() sentence_evaluator = SentenceEvaluator(ExpressionEvaluatorFactory()) gdal_coverage_converter = GdalToCoverageConverter( self.resumer, self.session.get_default_null_values(), self.recipe_type, sentence_evaluator, self.session.get_coverage_id(), None, self.session.get_files(), crs, None, None, global_metadata_fields, local_metadata_fields, None, None, general_recipe._metadata_type(), None, None, self.session) coverage_slices_dict = self._get_coverage_slices( crs, gdal_coverage_converter) fields = GdalRangeFieldsGenerator( gdal_dataset, self.options['band_names']).get_range_fields() global_metadata = None if len(coverage_slices_dict["base"]) > 0: global_metadata = gdal_coverage_converter._generate_global_metadata( coverage_slices_dict["base"][0], self.evaluator_slice) results = [] base_coverage_id = self.session.get_coverage_id() for key, value in coverage_slices_dict.items(): if key == "base": # base coverage coverage = Coverage(base_coverage_id, coverage_slices_dict[key], fields, crs, gdal_dataset.get_band_gdal_type(), self.options['tiling'], global_metadata) else: # overview coverage (key = overview_index) coverage_id = create_coverage_id_for_overview( base_coverage_id, key) coverage = Coverage(coverage_id, coverage_slices_dict[key], fields, crs, gdal_dataset.get_band_gdal_type(), self.options['tiling'], global_metadata, base_coverage_id, key) results.append(coverage) return results
def _get_coverage(self): # Get the crs of one of the images using a GDAL helper class. We are assuming all images have the same CRS for file in self.session.get_files(): try: file_path = file.get_filepath() gdal_dataset = GDALGmlUtil(file_path) break except Exception as e: if ConfigManager.skip == True: pass else: raise e # Get the crs of the coverage by compounding the two crses crs = CRSUtil.get_compound_crs([gdal_dataset.get_crs(), self.options['time_crs']]) fields = GdalRangeFieldsGenerator(gdal_dataset).get_range_fields() pixel_type = gdal_dataset.get_band_gdal_type() coverage_id = self.session.get_coverage_id() slices = self._get_slices(crs) return Coverage(coverage_id, slices, fields, crs, pixel_type)
def get_dataset(self): """ Returns the dataset of the file NOTE: gdal cannot open too many files (1989 files with error too many file opens) when getting dataset object and store inside list, so only open dataset when it is needed. :rtype: gdal Dataset """ if self.dataset is None: self.dataset = GDALGmlUtil(self.get_file().filepath) return self.dataset
def _data_type(self): """ Returns the data type for this netcdf dataset :rtype: str """ if len(self.files) < 1: raise RuntimeException("No files to import were specified.") netCDF4 = import_netcdf4() nci = netCDF4.Dataset(self.files[0].get_filepath(), 'r') netcdf_data_type = nci.variables[self.bands[0].identifier].dtype.name return GDALGmlUtil.data_type_to_gdal_type(netcdf_data_type)
def _get_data_type(self, slice): """ Returns the data type of the slice by downloading the slice and trying to guess it with GDAL :param Slice slice: slice :rtype: str """ if isinstance(slice.data_provider, UrlDataProvider): # Do this only for coverages that have more than one axis if len(slice.axis_subsets) > 1: fu = FileUtil() contents = validate_and_read_url(slice.data_provider.get_url()) file_path = fu.write_to_tmp_file(contents, "tif") return GDALGmlUtil(file_path).get_band_gdal_type() return None
def _get_coverage(self): """ Returns the coverage to be used for the importer """ gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file( self.session.get_files()) crs = CRSUtil.get_compound_crs( [self.options['time_crs'], gdal_dataset.get_crs()]) slices = self._get_slices(crs) fields = GdalRangeFieldsGenerator( gdal_dataset, self.options['band_names']).get_range_fields() coverage = Coverage(self.session.get_coverage_id(), slices, fields, crs, gdal_dataset.get_band_gdal_type(), self.options['tiling']) return coverage
def _get_coverage_slices(self, crs, gdal_coverage_converter): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices = [] timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: file_path = tpair.file.get_filepath() # NOTE: don't process any imported file from *.resume.json as it is just waisted time if not self.resumer.is_file_imported(file_path): timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), file_path) if not FileUtil.validate_file_path(file_path): continue valid_coverage_slice = True try: subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file_path)).fill(True) subsets = self._fill_time_axis(tpair, subsets) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file_path, ex) valid_coverage_slice = False if valid_coverage_slice: # Generate local metadata string for current coverage slice self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, tpair.file) local_metadata = gdal_coverage_converter._generate_local_metadata( subsets, self.evaluator_slice) slices.append( Slice(subsets, FileDataProvider(tpair.file), local_metadata)) timer.print_elapsed_time() count += 1 return slices
def get_data_type(self, netcdf_recipe_converter): """ Return the data type of opened dataset :param: AbstractToCoverageConverter netcdf_recipe: Converter for netCDF recipe :return: str data_type """ collected_data_type = [] for band in netcdf_recipe_converter.bands: band_id = band.identifier netcdf_data_type = self.get_dataset().variables[band_id].dtype.name band_data_type = GDALGmlUtil.data_type_to_gdal_type( netcdf_data_type) collected_data_type.append(band_data_type) data_type = ",".join(collected_data_type) return data_type
def _get_slices(self, gdal_dataset): """ Returns the slices for the collection of files given """ files = self.session.get_files() crs = gdal_dataset.get_crs() crs_axes = CRSUtil(crs).get_axes() slices = [] count = 1 for file in files: # print which file is analyzing FileUtil.print_feedback(count, len(files), file.filepath) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill() slices.append(Slice(subsets, FileDataProvider(file))) count += 1 return slices
def _get_slices(self, crs): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes() slices = [] timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), tpair.file.filepath) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil( tpair.file.get_filepath())).fill() subsets = self._fill_time_axis(tpair, subsets) slices.append(Slice(subsets, FileDataProvider(tpair.file))) count += 1 return slices
def _file_band_nil_values(self, index): """ This is used to get the null values (Only 1) from the given band index if one exists when nilValue was not defined in ingredient file :param integer index: the current band index to get the nilValues :rtype: List[RangeTypeNilValue] with only 1 element """ if len(self.files) < 1: raise RuntimeException("No gdal files given for import!") if len(self.default_null_values) > 0: return self.default_null_values # NOTE: all files should have same bands's metadata, so 1 file is ok gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(self.files) # band in gdal starts with 1 gdal_band = gdal_dataset.get_raster_band(index + 1) nil_value = gdal_band.GetNoDataValue() if nil_value is None: return None else: return [nil_value]
def __add_color_palette_table_to_global_metadata(self, metadata_dict, file_path): """ If colorPaletteTable is added in ingredient file, then add it to coverage's global metadata """ supported_recipe = (self.options['coverage']['slicer']['type'] == "gdal") color_palette_table = None if "metadata" in self.options['coverage']: if "colorPaletteTable" in self.options['coverage']['metadata']: value = self.options['coverage']['metadata']['colorPaletteTable'] if value.strip() != "": if value == "auto" and not supported_recipe: raise RecipeValidationException("colorPaletteTable auto is only supported" " in general recipe with slicer's type: gdal.") elif value == "auto": # Get colorPaletteTable automatically from first file gdal_dataset = GDALGmlUtil(file_path) color_palette_table = gdal_dataset.get_color_table() else: # file_path can be relative path or full path file_paths = FileUtil.get_file_paths_by_regex(self.session.get_ingredients_dir_path(), value) if len(file_paths) == 0: raise RecipeValidationException( "Color palette table file does not exist, given: '" + value + "'.") else: file_path = file_paths[0] # Add the content of colorPaletteTable to coverage's metadata with open(file_path, 'r') as file_reader: color_palette_table = file_reader.read() elif supported_recipe: # If colorPaletteTable is not mentioned in the ingredient, automatically fetch it gdal_dataset = GDALGmlUtil(file_path) color_palette_table = gdal_dataset.get_color_table() if color_palette_table is not None: metadata_dict["colorPaletteTable"] = color_palette_table
def _create_coverage_slices(self, crs_axes, calculated_evaluator_slice=None, axis_resolutions=None): """ Returns the slices for the collection of files given :param crs_axes: :rtype: list[Slice] """ from master.recipe.base_recipe import BaseRecipe slices_dict = BaseRecipe.create_dict_of_slices( self.session.import_overviews) count = 1 for file in self.files: timer = Timer() FileUtil.print_feedback(count, len(self.files), file.filepath) # print which file is analyzing if not FileUtil.validate_file_path(file.filepath): continue valid_coverage_slice = True try: if calculated_evaluator_slice is None: # get the evaluator for the current recipe_type (each recipe has different evaluator) self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, file) else: self.evaluator_slice = calculated_evaluator_slice if self.data_type is None: self.data_type = self.evaluator_slice.get_data_type(self) coverage_slice = self._create_coverage_slice( file, crs_axes, self.evaluator_slice, axis_resolutions) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file.get_filepath(), ex) valid_coverage_slice = False if valid_coverage_slice: if self.session.import_overviews_only is False: slices_dict["base"].append(coverage_slice) if self.session.recipe["options"]["coverage"]["slicer"][ "type"] == "gdal": gdal_file = GDALGmlUtil(file.get_filepath()) # Then, create slices for selected overviews from user for overview_index in self.session.import_overviews: axis_subsets_overview = BaseRecipe.create_subsets_for_overview( coverage_slice.axis_subsets, overview_index, gdal_file) coverage_slice_overview = copy.deepcopy(coverage_slice) coverage_slice_overview.axis_subsets = axis_subsets_overview slices_dict[str(overview_index)].append( coverage_slice_overview) timer.print_elapsed_time() count += 1 # Currently, only sort by datetime to import coverage slices (default is ascending) reverse = (self.import_order == self.IMPORT_ORDER_DESCENDING) for key, value in slices_dict.items(): slices_dict[key] = sort_slices_by_datetime(value, reverse) return slices_dict
def _init_epsg_xy_crs(self): """ From the first file of input file, detect its EPSG code for XY axes """ gdal_ds = GDALGmlUtil(self.session.get_files()[0].get_filepath()) self.epsg_xy_crs = gdal_ds.get_crs()
def _get_convertors(self): """ Returns a map of coverage id -> GdalToCoverageConverter """ convertors = {} for f in self.session.get_files(): # This one does not contain any information for geo bounds if not FileUtil.validate_file_path(f.get_filepath()): continue gdal_ds = GDALGmlUtil(f.get_filepath()) subdatasets = self._get_subdatasets(gdal_ds, f) gdal_ds.close() level = self._get_level(f.get_filepath()) if len(self.levels) > 0 and level not in self.levels: # skip file, as it's not in the list of levels provided in the ingredients file log.debug("Skipping " + level + " data") continue crs_code = "" evaluator_slice = None for res in self.resolutions: subds_file = self._get_subdataset_file(subdatasets, res) crs_code = self._get_crs_code(subds_file.get_filepath(), crs_code) if len(self.crss) > 0 and crs_code not in self.crss: # skip CRS, it's not in the list of CRSs provided in the ingredients file log.debug("Skipping data with CRS " + crs_code) continue cov_id = self._get_coverage_id(self.coverage_id, crs_code, level, res) conv = self._get_convertor(convertors, cov_id, crs_code, level, res) file_pair = FilePair(subds_file.filepath, f.filepath) conv.files = [file_pair] crs_axes = CRSUtil(conv.crs).get_axes(self.coverage_id) if evaluator_slice is None: # This one contains information for geo bounds evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( GdalToCoverageConverter.RECIPE_TYPE, subds_file) # Resolution 10m, 20m and 60m have same data type (UInt16) while TCI has data type (Byte) if res == self.RES_TCI: conv.data_type = "Byte" else: conv.data_type = "UInt16" # Fixed values for 3 axes of Sentinel 2 coverage axis_resolutions = self.RES_DICT[res] slices_dict = conv._create_coverage_slices( crs_axes, evaluator_slice, axis_resolutions) if conv.coverage_slices == {}: conv.coverage_slices = slices_dict else: for key, val in slices_dict.items(): conv.coverage_slices[key] += slices_dict[key] if len(conv.coverage_slices) != 0: first_slice = conv.coverage_slices["base"][0] # This needs one available file to extract metadata later conv.files = [first_slice.data_provider.file] return convertors
def _axis_subset(self, crs_axis, gdal_file): """ Returns an axis subset using the given crs axis in the context of the gdal file :param CRSAxis crs_axis: the crs definition of the axis :param File gdal_file: the gdal file :rtype AxisSubset """ user_axis = self._user_axis( self._get_user_axis_by_crs_axis_name(crs_axis.label), GDALEvaluatorSlice(GDALGmlUtil(gdal_file.get_filepath()))) high = user_axis.interval.high if user_axis.interval.high else user_axis.interval.low if isinstance(user_axis, RegularUserAxis): geo_axis = RegularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, user_axis.interval.low, crs_axis) else: # if irregular axis value is fetched from fileName so the coefficient is [0] as slicing if user_axis.directPositions == AbstractToCoverageConverter.DIRECT_POSITIONS_SLICING: user_axis.directPositions = AbstractToCoverageConverter.COEFFICIENT_SLICING geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, user_axis.interval.low, user_axis.directPositions, crs_axis) if not crs_axis.is_easting() and not crs_axis.is_northing(): # GDAL model is 2D so on any axis except x/y we expect to have only one value grid_low = 0 grid_high = 0 else: grid_low = 0 number_of_grid_points = decimal.Decimal(str(user_axis.interval.high)) \ - decimal.Decimal(str(user_axis.interval.low)) # number_of_grid_points = (geo_max - geo_min) / resolution grid_high = grid_low + number_of_grid_points / decimal.Decimal( user_axis.resolution) grid_high = HighPixelAjuster.adjust_high(grid_high) # Negative axis, e.g: Latitude (min <--- max) if user_axis.resolution < 0: grid_high = int(abs(math.floor(grid_high))) else: # Positive axis, e.g: Longitude (min --> max) grid_high = int(abs(math.ceil(grid_high))) # NOTE: Grid Coverage uses the direct intervals as in Rasdaman if self.grid_coverage is False: if grid_high > grid_low: grid_high -= 1 grid_axis = GridAxis(user_axis.order, crs_axis.label, user_axis.resolution, grid_low, grid_high) geo_axis.origin = PointPixelAdjuster.get_origin(user_axis, crs_axis) if user_axis.type == UserAxisType.DATE: self._translate_decimal_to_datetime(user_axis, geo_axis) # NOTE: current, gdal recipe supports only has 2 axes which are "bounded" (i.e: they exist as 2D axes in file) # and 1 or more another axes gotten (i.e: from fileName) which are not "bounded" to create 3D+ coverage. data_bound = crs_axis.is_northing() or crs_axis.is_easting() return AxisSubset( CoverageAxis(geo_axis, grid_axis, data_bound), Interval(user_axis.interval.low, user_axis.interval.high))
def _read_bands(self): """ Returns a user band extracted from the ingredients if specified (required for netCDF/GRIB) :rtype: list[UserBand] """ if "bands" in self.options['coverage']['slicer']: bands = self.options['coverage']['slicer']['bands'] number_of_bands = len(bands) # NOTE: rasdaman supports 1 band grib only to import recipe_type = self.options['coverage']['slicer']['type'] if recipe_type == GRIBToCoverageConverter.RECIPE_TYPE and number_of_bands > 1: raise RuntimeError("Only single band grib files are currently supported. " "Given " + str(number_of_bands) + " bands in ingredient file.") ret_bands = [] i = 0 for band in bands: identifier = self._read_or_empty_string(band, "identifier") if recipe_type == GdalToCoverageConverter.RECIPE_TYPE: # NOTE: for old ingredients with wrong defined "identifier" with band name instead of index 0-based if not identifier.isdigit(): identifier = str(i) ret_bands.append(UserBand( identifier, self._read_or_empty_string(band, "name"), self._read_or_empty_string(band, "description"), self._read_or_empty_string(band, "definition"), self._read_or_empty_string(band, "nilReason"), self._read_or_empty_string(band, "nilValue").split(","), self._read_or_empty_string(band, "uomCode") )) i += 1 return ret_bands else: if self.options['coverage']['slicer']['type'] == GdalToCoverageConverter.RECIPE_TYPE: # If gdal does not specify bands in ingredient file, just fetch all bands from first file for file in self.session.get_files(): try: gdal_util = GDALGmlUtil(file) gdal_fields = gdal_util.get_fields_range_type() ret_bands = [] for field in gdal_fields: ret_bands.append(UserBand( field.field_name, field.field_name, None, None, None, field.nill_values )) break except Exception as e: if ConfigManager.skip == True: pass else: raise e return ret_bands else: raise RuntimeError("'bands' must be specified in ingredient file for netCDF/GRIB recipes.")
def __get_import_overviews(self): """ Get the OVERVIEWs in the ingredients file if user wants to import """ if "options" in self.recipe: if "import_all_overviews" in self.recipe["options"] and "import_overviews" in self.recipe["options"] \ and self.recipe["options"]["import_all_overviews"] is True: raise RuntimeException( "Both settings '{}' or '{}' cannot exist in the ingredients file, " "please specify only one of them.".format( "import_all_overviews", "import_overviews")) self.import_overviews = self.recipe["options"]["import_overviews"] \ if "import_overviews" in self.recipe["options"] else [] if len(self.files) > 0 and ( "import_all_overviews" in self.recipe["options"] or "import_overviews" in self.recipe["options"]): from util.gdal_util import GDALGmlUtil first_input_file_gdal = GDALGmlUtil(self.files[0].filepath) if "import_all_overviews" in self.recipe["options"] \ and bool(self.recipe["options"]["import_all_overviews"]) is True: # import all overviews number_of_overviews = first_input_file_gdal.get_number_of_overviews( ) self.import_overviews = [] if number_of_overviews > 0: self.import_overviews = range(0, number_of_overviews) for overview_index in self.import_overviews: if is_integer(overview_index) is False or int( overview_index) < 0: raise RuntimeException( "'{}' must contain non-negative integers integer values. " "Given: {}".format("import_overviews", overview_index)) elif self.recipe["name"] != "sentinel2": # NOTE: sentinel 2 custom recipe to import .zip file has special format (xml to combine subdatasets (10m, 20m, 60m)) # e.g: SENTINEL2_L2A:/vsizip//home/vagrant/S2A_..._20210601T140140.zip/S2A_..._20210601T140140.SAFE/MTD_MSIL2A.xml:10m:EPSG_32632 overview = first_input_file_gdal.get_overview( overview_index) if overview is None: raise RuntimeException( "Overview index '{}' does not exist in the input file '{}'." .format(overview_index, first_input_file_gdal.gdal_file_path)) from util.gdal_util import GDALGmlUtil # e.g 101140 gdal_version = int(GDALGmlUtil.get_gdal_version()[0:2]) if gdal_version < 20: from osgeo import gdal # e.g: 1.11.4 version = gdal.__version__ raise RuntimeException( "NOTE: Importing overviews is only supported since gdal version 2.0, " "and your system has GDAL version '" + version + "'.")
def _data_type(self): if len(self.files) < 1: raise RuntimeException("No files to import were specified.") # as all files should have same data type, so first file is ok return GDALGmlUtil(self.files[0].get_filepath()).get_band_gdal_type()