def _get_slices(self, crs): """ Returns the slices for the collection of files given """ files = self.session.get_files() crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices = [] count = 1 for file in files: # NOTE: don't process any imported file from *.resume.json as it is just waisted time if not self.resumer.is_file_imported(file.filepath): timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(files), file.filepath) if not FileUtil.validate_file_path(file.filepath): continue valid_coverage_slice = True try: subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill() except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible(file.get_filepath(), ex) valid_coverage_slice = False if valid_coverage_slice: slices.append(Slice(subsets, FileDataProvider(file))) timer.print_elapsed_time() count += 1 return slices
def _get_coverage_slices(self, crs, gdal_coverage_converter): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices_dict = self.create_dict_of_slices(self.session.import_overviews) timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: file = tpair.file file_path = tpair.file.get_filepath() timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), file_path) if not FileUtil.validate_file_path(file_path): continue valid_coverage_slice = True gdal_file = GDALGmlUtil(file.get_filepath()) try: subsets = GdalAxisFiller(crs_axes, gdal_file).fill(True) subsets = self._fill_time_axis(tpair, subsets) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file_path, ex) valid_coverage_slice = False if valid_coverage_slice: # Generate local metadata string for current coverage slice self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, tpair.file) local_metadata = gdal_coverage_converter._generate_local_metadata( subsets, self.evaluator_slice) if self.session.import_overviews_only is False: slices_dict["base"].append( Slice(subsets, FileDataProvider(tpair.file), local_metadata)) # Then, create slices for selected overviews from user for overview_index in self.session.import_overviews: subsets_overview = self.create_subsets_for_overview( subsets, overview_index, gdal_file) slices_dict[str(overview_index)].append( Slice(subsets_overview, FileDataProvider(file), local_metadata)) timer.print_elapsed_time() count += 1 return slices_dict
def _create_coverage_slices(self, crs_axes, calculated_evaluator_slice=None, axis_resolutions=None): """ Returns all the coverage slices for this coverage :param crs_axes: :rtype: list[Slice] """ slices = [] count = 1 for file in self.files: # NOTE: don't process any previously imported file (recorded in *.resume.json) if not self.resumer.is_file_imported(file.filepath): timer = Timer() FileUtil.print_feedback(count, len(self.files), file.filepath) # print which file is analyzing if not FileUtil.validate_file_path(file.filepath): continue valid_coverage_slice = True evaluator_slice = None try: if calculated_evaluator_slice is None: # get the evaluator for the current recipe_type (each recipe has different evaluator) evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice(self.recipe_type, file) else: evaluator_slice = calculated_evaluator_slice if self.data_type is None: self.data_type = evaluator_slice.get_data_type(self) coverage_slice = self._create_coverage_slice(file, crs_axes, evaluator_slice, axis_resolutions) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible(file.get_filepath(), ex) valid_coverage_slice = False if valid_coverage_slice: slices.append(coverage_slice) timer.print_elapsed_time() count += 1 # Currently, only sort by datetime to import coverage slices (default is ascending) reverse = (self.import_order == self.IMPORT_ORDER_DESCENDING) return sort_slices_by_datetime(slices, reverse)
def _get_coverage_slices(self, crs, gdal_coverage_converter): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes(self.session.coverage_id) slices = [] timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: file_path = tpair.file.get_filepath() # NOTE: don't process any imported file from *.resume.json as it is just waisted time if not self.resumer.is_file_imported(file_path): timer = Timer() # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), file_path) if not FileUtil.validate_file_path(file_path): continue valid_coverage_slice = True try: subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file_path)).fill(True) subsets = self._fill_time_axis(tpair, subsets) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file_path, ex) valid_coverage_slice = False if valid_coverage_slice: # Generate local metadata string for current coverage slice self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, tpair.file) local_metadata = gdal_coverage_converter._generate_local_metadata( subsets, self.evaluator_slice) slices.append( Slice(subsets, FileDataProvider(tpair.file), local_metadata)) timer.print_elapsed_time() count += 1 return slices
def _get_slices(self, gdal_dataset): """ Returns the slices for the collection of files given """ files = self.session.get_files() crs = gdal_dataset.get_crs() crs_axes = CRSUtil(crs).get_axes() slices = [] count = 1 for file in files: # print which file is analyzing FileUtil.print_feedback(count, len(files), file.filepath) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil(file.get_filepath())).fill() slices.append(Slice(subsets, FileDataProvider(file))) count += 1 return slices
def _slices(self, crs_axes): """ Returns all the slices for this coverage :param crs_axes: :rtype: list[Slice] """ slices = [] count = 1 for file in self.files: # print which file is analyzing FileUtil.print_feedback(count, len(self.files), file.filepath) slices.append(self._slice(file, crs_axes)) count += 1 # NOTE: we want to sort all the slices by date time axis # to avoid the case the later time slice is added before the sooner time slice sorted_slices = sort_slices_by_datetime(slices) return sorted_slices
def _get_slices(self, crs): """ Returns the slices for the collection of files given """ crs_axes = CRSUtil(crs).get_axes() slices = [] timeseries = self._generate_timeseries_tuples() count = 1 for tpair in timeseries: # print which file is analyzing FileUtil.print_feedback(count, len(timeseries), tpair.file.filepath) subsets = GdalAxisFiller(crs_axes, GDALGmlUtil( tpair.file.get_filepath())).fill() subsets = self._fill_time_axis(tpair, subsets) slices.append(Slice(subsets, FileDataProvider(tpair.file))) count += 1 return slices
def _create_coverage_slices(self, crs_axes, calculated_evaluator_slice=None, axis_resolutions=None): """ Returns the slices for the collection of files given :param crs_axes: :rtype: list[Slice] """ from master.recipe.base_recipe import BaseRecipe slices_dict = BaseRecipe.create_dict_of_slices( self.session.import_overviews) count = 1 for file in self.files: timer = Timer() FileUtil.print_feedback(count, len(self.files), file.filepath) # print which file is analyzing if not FileUtil.validate_file_path(file.filepath): continue valid_coverage_slice = True try: if calculated_evaluator_slice is None: # get the evaluator for the current recipe_type (each recipe has different evaluator) self.evaluator_slice = EvaluatorSliceFactory.get_evaluator_slice( self.recipe_type, file) else: self.evaluator_slice = calculated_evaluator_slice if self.data_type is None: self.data_type = self.evaluator_slice.get_data_type(self) coverage_slice = self._create_coverage_slice( file, crs_axes, self.evaluator_slice, axis_resolutions) except Exception as ex: # If skip: true then just ignore this file from importing, else raise exception FileUtil.ignore_coverage_slice_from_file_if_possible( file.get_filepath(), ex) valid_coverage_slice = False if valid_coverage_slice: if self.session.import_overviews_only is False: slices_dict["base"].append(coverage_slice) if self.session.recipe["options"]["coverage"]["slicer"][ "type"] == "gdal": gdal_file = GDALGmlUtil(file.get_filepath()) # Then, create slices for selected overviews from user for overview_index in self.session.import_overviews: axis_subsets_overview = BaseRecipe.create_subsets_for_overview( coverage_slice.axis_subsets, overview_index, gdal_file) coverage_slice_overview = copy.deepcopy(coverage_slice) coverage_slice_overview.axis_subsets = axis_subsets_overview slices_dict[str(overview_index)].append( coverage_slice_overview) timer.print_elapsed_time() count += 1 # Currently, only sort by datetime to import coverage slices (default is ascending) reverse = (self.import_order == self.IMPORT_ORDER_DESCENDING) for key, value in slices_dict.items(): slices_dict[key] = sort_slices_by_datetime(value, reverse) return slices_dict