def _evaluated_messages(self, grib_file): """ Returns the evaluated_messages for all grib_messages :param String grib_file: path to a grib file :rtype: list[GRIBMessage] """ pygrib = import_pygrib() self.dataset = pygrib.open(grib_file.filepath) evaluated_messages = [] # Message id starts with "1" for i in range(1, self.dataset.messages + 1): grib_message = self.dataset.message(i) axes = [] # Iterate all the axes and evaluate them with message # e.g: Long axis: ${grib:longitudeOfFirstGridPointInDegrees} # Lat axis: ${grib:latitudeOfLastGridPointInDegrees} # Message 1 return: Long: -180, Lat: 90 # Message 2 return: Long: -170, Lat: 80 # ... # Message 20 return: Long: 180, Lat: -90 for user_axis in self.user_axes: # find the crs_axis which are used to evaluate the user_axis (have same name) crs_axis = self._get_crs_axis_by_user_axis_name(user_axis.name) # NOTE: directPositions could be retrieved only when every message evaluated to get values for axis # e.g: message 1 has value: 0, message 3 has value: 2, message 5 has value: 8,...message 20 value: 30 # then, the directPositions of axis is [0, 2, 8,...30] # the syntax to retrieve directions in ingredient file is: ${grib:axis:axis_name} # with axis_name is the name user defined (e.g: AnsiDate?axis-label="time" then axis name is: time) self.evaluator_slice = GribMessageEvaluatorSlice( grib_message, grib_file) evaluated_user_axis = self._user_axis(user_axis, self.evaluator_slice) # When pixelIsPoint:true then it will be adjusted by half pixels for min, max internally (recommended) if self.pixel_is_point is True: PointPixelAdjuster.adjust_axis_bounds_to_continuous_space( evaluated_user_axis, crs_axis) else: # translate the dateTime format to float if evaluated_user_axis.type == UserAxisType.DATE: evaluated_user_axis.interval.low = arrow.get( evaluated_user_axis.interval.low).float_timestamp if evaluated_user_axis.interval.high: evaluated_user_axis.interval.high = arrow.get( evaluated_user_axis.interval.high ).float_timestamp # if low < high, adjust it if evaluated_user_axis.interval.high is not None \ and evaluated_user_axis.interval.low > evaluated_user_axis.interval.high: evaluated_user_axis.interval.low, evaluated_user_axis.interval.high = evaluated_user_axis.interval.high, evaluated_user_axis.interval.low evaluated_user_axis.statements = user_axis.statements axes.append(evaluated_user_axis) evaluated_messages.append(GRIBMessage(i, axes, grib_message)) return evaluated_messages
def _axis_subset(self, crs_axis, nc_file): """ Returns an axis subset using the given crs axis in the context of the nc file :param CRSAxis crs_axis: the crs definition of the axis :param File nc_file: the netcdf file :rtype AxisSubset """ user_axis = self._user_axis(self._get_user_axis_by_crs_axis_name(crs_axis.label), NetcdfEvaluatorSlice(nc_file)) # Normally, without pixelIsPoint:true, in the ingredient needs to +/- 0.5 * resolution for each regular axis # e.g: resolution for axis E is 10000, then # "min": "${netcdf:variable:E:min} - 10000 / 2", # "max": "${netcdf:variable:E:max} + 10000 / 2", # with pixelIsPoint: true, no need to add these values as the service will do it automatically if self.pixel_is_point: PointPixelAdjuster.adjust_axis_bounds_to_continuous_space(user_axis, crs_axis) else: # No adjustment for all regular axes but still need to translate time in datetime to decimal to calculate if user_axis.type == UserAxisType.DATE: user_axis.interval.low = decimal.Decimal(str(arrow.get(user_axis.interval.low).float_timestamp)) if user_axis.interval.high: user_axis.interval.high = decimal.Decimal(str(arrow.get(user_axis.interval.high).float_timestamp)) # if low < high, adjust it if user_axis.interval.high is not None and user_axis.interval.low > user_axis.interval.high: user_axis.interval.low, user_axis.interval.high = user_axis.interval.high, user_axis.interval.low high = user_axis.interval.high if user_axis.interval.high else user_axis.interval.low origin = PointPixelAdjuster.get_origin(user_axis, crs_axis) if isinstance(user_axis, RegularUserAxis): geo_axis = RegularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, origin, crs_axis) else: if user_axis.type == UserAxisType.DATE: if crs_axis.is_uom_day(): coefficients = self._translate_day_date_direct_position_to_coefficients(user_axis.interval.low, user_axis.directPositions) else: coefficients = self._translate_seconds_date_direct_position_to_coefficients(user_axis.interval.low, user_axis.directPositions) else: coefficients = self._translate_number_direct_position_to_coefficients(user_axis.interval.low, user_axis.directPositions) geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, origin, coefficients, crs_axis) grid_low = 0 grid_high = PointPixelAdjuster.get_grid_points(user_axis, crs_axis) # NOTE: Grid Coverage uses the direct intervals as in Rasdaman if self.grid_coverage is False and grid_high > grid_low: grid_high -= 1 grid_axis = GridAxis(user_axis.order, crs_axis.label, user_axis.resolution, grid_low, grid_high) if user_axis.type == UserAxisType.DATE: self._translate_decimal_to_datetime(user_axis, geo_axis) return AxisSubset(CoverageAxis(geo_axis, grid_axis, user_axis.dataBound), Interval(user_axis.interval.low, user_axis.interval.high))
def _axis_subset(self, crs_axis, evaluator_slice, resolution=None): """ Returns an axis subset using the given crs axis in the context of the gdal file :param CRSAxis crs_axis: the crs definition of the axis :param GDALEvaluatorSlice evaluator_slice: the evaluator for GDAL file :param resolution: Known axis resolution, no need to evaluate sentence expression from ingredient file (e.g: Sentinel2 recipe) :rtype AxisSubset """ user_axis = self._user_axis( self._get_user_axis_by_crs_axis_name(crs_axis.label), evaluator_slice) if resolution is not None: user_axis.resolution = resolution high = user_axis.interval.high if user_axis.interval.high is not None else user_axis.interval.low if user_axis.type == UserAxisType.DATE: # it must translate datetime string to float by arrow for calculating later user_axis.interval.low = arrow.get( user_axis.interval.low).float_timestamp if user_axis.interval.high is not None: user_axis.interval.high = arrow.get( user_axis.interval.high).float_timestamp if isinstance(user_axis, RegularUserAxis): geo_axis = RegularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, user_axis.interval.low, crs_axis) else: # Irregular axis (coefficients must be number, not datetime string) if user_axis.type == UserAxisType.DATE: if crs_axis.is_time_day_axis(): coefficients = self._translate_day_date_direct_position_to_coefficients( user_axis.interval.low, user_axis.directPositions) else: coefficients = self._translate_seconds_date_direct_position_to_coefficients( user_axis.interval.low, user_axis.directPositions) else: coefficients = self._translate_number_direct_position_to_coefficients( user_axis.interval.low, user_axis.directPositions) self._update_for_slice_group_size(self.coverage_id, user_axis, crs_axis, coefficients) geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, user_axis.interval.low, coefficients, crs_axis) if not crs_axis.is_x_axis() and not crs_axis.is_y_axis(): # GDAL model is 2D so on any axis except x/y we expect to have only one value grid_low = 0 grid_high = None if user_axis.interval.high is not None: grid_high = 0 else: grid_low = 0 number_of_grid_points = decimal.Decimal(str(user_axis.interval.high)) \ - decimal.Decimal(str(user_axis.interval.low)) # number_of_grid_points = (geo_max - geo_min) / resolution grid_high = grid_low + number_of_grid_points / decimal.Decimal( user_axis.resolution) grid_high = HighPixelAjuster.adjust_high(grid_high) # Negative axis, e.g: Latitude (min <--- max) if user_axis.resolution < 0: grid_high = int(abs(math.floor(grid_high))) else: # Positive axis, e.g: Longitude (min --> max) grid_high = int(abs(math.ceil(grid_high))) # NOTE: Grid Coverage uses the direct intervals as in Rasdaman if self.grid_coverage is False and grid_high is not None: if grid_high > grid_low: grid_high -= 1 grid_axis = GridAxis(user_axis.order, crs_axis.label, user_axis.resolution, grid_low, grid_high) geo_axis.origin = PointPixelAdjuster.get_origin(user_axis, crs_axis) if user_axis.type == UserAxisType.DATE: self._translate_decimal_to_datetime(user_axis, geo_axis) # NOTE: current, gdal recipe supports only has 2 axes which are "bounded" (i.e: they exist as 2D axes in file) # and 1 or more another axes gotten (i.e: from fileName) which are not "bounded" to create 3D+ coverage. data_bound = crs_axis.is_y_axis() or crs_axis.is_x_axis() return AxisSubset( CoverageAxis(geo_axis, grid_axis, data_bound), Interval(user_axis.interval.low, user_axis.interval.high))
def _axis_subset(self, grib_file, evaluated_messages, crs_axis): """ Returns an axis subset using the given crs axis in the context of the grib file :param File grib_file: the current grib file (slice) is evaluated :param List[GirbMessages] evaluated_messages: all Grib messages was evaluated :param CRSAxis crs_axis: the crs definition of the axis :rtype AxisSubset """ # first grib message from grib file, used to extract grib variables only first_grib_message = self.dataset.message(1) # As all the messages contain same axes (but different intervals), so first message is ok to get user_axis first_user_axis = self._get_user_axis_in_evaluated_message( evaluated_messages[0], crs_axis.label) # NOTE: we don't want to change this user_axis belongs to messages, so clone it user_axis = copy.deepcopy(first_user_axis) # Then, we calculate the geo, grid bounds, origin, resolution of this axis for the slice self._set_low_high(evaluated_messages, user_axis) high = user_axis.interval.high if user_axis.interval.high is not None else user_axis.interval.low origin = PointPixelAdjuster.get_origin(user_axis, crs_axis) if isinstance(user_axis, RegularUserAxis): geo_axis = RegularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, origin, crs_axis) else: # after all messages was evaluated, we could get the direct_positions of the axis as in netcdf # then, it can evaluate the grib sentence normally, e.g: ${grib:axis:level} + 5 evaluating_sentence = user_axis.directPositions direct_positions = self._get_axis_values(evaluated_messages, user_axis) # convert all of values in the list to string then it can be evaluated direct_positions = list_util.to_list_string(direct_positions) evaluator_slice = GribMessageEvaluatorSlice( first_grib_message, grib_file, direct_positions) user_axis.directPositions = self.sentence_evaluator.evaluate( evaluating_sentence, evaluator_slice, user_axis.statements) # axis is datetime if user_axis.type == UserAxisType.DATE: if crs_axis.is_time_day_axis(): coefficients = self._translate_day_date_direct_position_to_coefficients( user_axis.interval.low, user_axis.directPositions) else: coefficients = self._translate_seconds_date_direct_position_to_coefficients( user_axis.interval.low, user_axis.directPositions) else: # number axis like Index1D coefficients = self._translate_number_direct_position_to_coefficients( user_axis.interval.low, user_axis.directPositions) self._update_for_slice_group_size(self.coverage_id, user_axis, crs_axis, coefficients) geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, origin, coefficients, crs_axis) grid_low = 0 grid_high = PointPixelAdjuster.get_grid_points(user_axis, crs_axis) # NOTE: Grid Coverage uses the direct intervals as in Rasdaman if self.grid_coverage is False and grid_high > grid_low: grid_high -= 1 grid_axis = GridAxis(user_axis.order, crs_axis.label, user_axis.resolution, grid_low, grid_high) if user_axis.type == UserAxisType.DATE: self._translate_decimal_to_datetime(user_axis, geo_axis) return AxisSubset( CoverageAxis(geo_axis, grid_axis, user_axis.dataBound), Interval(user_axis.interval.low, user_axis.interval.high))
def _axis_subset(self, crs_axis, gdal_file): """ Returns an axis subset using the given crs axis in the context of the gdal file :param CRSAxis crs_axis: the crs definition of the axis :param File gdal_file: the gdal file :rtype AxisSubset """ user_axis = self._user_axis( self._get_user_axis_by_crs_axis_name(crs_axis.label), GDALEvaluatorSlice(GDALGmlUtil(gdal_file.get_filepath()))) high = user_axis.interval.high if user_axis.interval.high else user_axis.interval.low if isinstance(user_axis, RegularUserAxis): geo_axis = RegularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, user_axis.interval.low, crs_axis) else: # if irregular axis value is fetched from fileName so the coefficient is [0] as slicing if user_axis.directPositions == AbstractToCoverageConverter.DIRECT_POSITIONS_SLICING: user_axis.directPositions = AbstractToCoverageConverter.COEFFICIENT_SLICING geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, user_axis.interval.low, user_axis.directPositions, crs_axis) if not crs_axis.is_easting() and not crs_axis.is_northing(): # GDAL model is 2D so on any axis except x/y we expect to have only one value grid_low = 0 grid_high = 0 else: grid_low = 0 number_of_grid_points = decimal.Decimal(str(user_axis.interval.high)) \ - decimal.Decimal(str(user_axis.interval.low)) # number_of_grid_points = (geo_max - geo_min) / resolution grid_high = grid_low + number_of_grid_points / decimal.Decimal( user_axis.resolution) grid_high = HighPixelAjuster.adjust_high(grid_high) # Negative axis, e.g: Latitude (min <--- max) if user_axis.resolution < 0: grid_high = int(abs(math.floor(grid_high))) else: # Positive axis, e.g: Longitude (min --> max) grid_high = int(abs(math.ceil(grid_high))) # NOTE: Grid Coverage uses the direct intervals as in Rasdaman if self.grid_coverage is False: if grid_high > grid_low: grid_high -= 1 grid_axis = GridAxis(user_axis.order, crs_axis.label, user_axis.resolution, grid_low, grid_high) geo_axis.origin = PointPixelAdjuster.get_origin(user_axis, crs_axis) if user_axis.type == UserAxisType.DATE: self._translate_decimal_to_datetime(user_axis, geo_axis) # NOTE: current, gdal recipe supports only has 2 axes which are "bounded" (i.e: they exist as 2D axes in file) # and 1 or more another axes gotten (i.e: from fileName) which are not "bounded" to create 3D+ coverage. data_bound = crs_axis.is_northing() or crs_axis.is_easting() return AxisSubset( CoverageAxis(geo_axis, grid_axis, data_bound), Interval(user_axis.interval.low, user_axis.interval.high))