def get_origin_for_time_axis(user_axis, crs_axis): user_axis.interval.low = arrow.get( user_axis.interval.low).float_timestamp if user_axis.interval.high: user_axis.interval.high = arrow.get( user_axis.interval.high).float_timestamp if isinstance(user_axis, RegularUserAxis): # ansidate, need to calculate with day in seconds if crs_axis.is_time_day_axis: if user_axis.resolution > 0 or user_axis.interval.high is None: # axis goes from low to high, so origin is lowest, with half a pixel shift return decimal.Decimal(str(user_axis.interval.low))\ + decimal.Decimal(0.5) * decimal.Decimal(str(user_axis.resolution)) * DateTimeUtil.DAY_IN_SECONDS else: # axis goes from high to low, so origin is highest, with half pixel shift (resolution is negative) return decimal.Decimal(str(user_axis.interval.high))\ + decimal.Decimal(0.5) * decimal.Decimal(str(user_axis.resolution)) * DateTimeUtil.DAY_IN_SECONDS else: # unix time, already in seconds if user_axis.resolution > 0 or user_axis.interval.high is None: # axis goes from low to high, so origin is lowest, with half a pixel shift return decimal.Decimal(str(user_axis.interval.low))\ + decimal.Decimal(0.5) * decimal.Decimal(str(user_axis.resolution)) else: # axis goes from high to low, so origin is highest, with half pixel shift (resolution is negative) return decimal.Decimal(str(user_axis.interval.high))\ + decimal.Decimal(0.5) * decimal.Decimal(str(user_axis.resolution)) else: # irregular axis, the same but without shift if user_axis.resolution > 0 or user_axis.interval.high is None: return user_axis.interval.low else: return user_axis.interval.high
def adjust_axis_bounds_for_time_axis(user_axis, crs_axis): # convert to timestamp and change the axis type user_axis.interval.low = arrow.get( user_axis.interval.low).float_timestamp if user_axis.interval.high: user_axis.interval.high = arrow.get( user_axis.interval.high).float_timestamp # if low < high, adjust it if user_axis.interval.high is not None and user_axis.interval.low > user_axis.interval.high: user_axis.interval.low, user_axis.interval.high = user_axis.interval.high, user_axis.interval.low # The formula for all regular axes is when "pixelIsPoint": # min = min - 0.5 * resolution # max = max + 0.5 * resolution if isinstance(user_axis, RegularUserAxis): # if axis is time axis if crs_axis.is_time_day_axis(): user_axis.interval.low = decimal.Decimal(str(user_axis.interval.low)) \ - decimal.Decimal(0.5) * decimal.Decimal(str(abs(user_axis.resolution))) * DateTimeUtil.DAY_IN_SECONDS if user_axis.interval.high: user_axis.interval.high = decimal.Decimal(str(user_axis.interval.high)) \ + decimal.Decimal(0.5) * decimal.Decimal(str(abs(user_axis.resolution))) * DateTimeUtil.DAY_IN_SECONDS else: # if axis is normal axis (lat, lon, index1d,...) user_axis.interval.low = decimal.Decimal(str(user_axis.interval.low)) \ - decimal.Decimal(0.5) * decimal.Decimal(str(abs(user_axis.resolution))) if user_axis.interval.high: user_axis.interval.high = decimal.Decimal(str(user_axis.interval.high)) \ + decimal.Decimal(0.5) * decimal.Decimal(str(abs(user_axis.resolution)))
def _get_update_subsets_for_slice(self, slice): """ Returns the given slice's interval as a list of wcst subsets :param slice: the slice for which to generate this :rtype: list[WCSTSubset] """ subsets = [] for axis_subset in slice.axis_subsets: low = axis_subset.interval.low high = axis_subset.interval.high # if ConfigManager.subset_correction and high is not None and low != high and type(low) != str: if ConfigManager.subset_correction and high is not None and low != high and type(low) == str: # Time axis with type = str (e.g: "1970-01-01T02:03:06Z") time_seconds = 1 # AnsiDate (need to change from date to seconds) if axis_subset.coverage_axis.axis.crs_axis.is_time_day_axis(): time_seconds = DateTimeUtil.DAY_IN_SECONDS low = decimal.Decimal(str(arrow.get(low).float_timestamp)) + decimal.Decimal(str(axis_subset.coverage_axis.grid_axis.resolution * time_seconds)) / 2 low = DateTimeUtil.get_datetime_iso(low) if high is not None: high = decimal.Decimal(str(arrow.get(high).float_timestamp)) - decimal.Decimal(str(axis_subset.coverage_axis.grid_axis.resolution * time_seconds)) / 2 high = DateTimeUtil.get_datetime_iso(high) elif ConfigManager.subset_correction and high is not None and low != high and type(low) != str: # regular axes (e.g: latitude, longitude, index1d) low = decimal.Decimal(str(low)) + decimal.Decimal(str(axis_subset.coverage_axis.grid_axis.resolution)) / 2 if high is not None: high = decimal.Decimal(str(high)) - decimal.Decimal(str(axis_subset.coverage_axis.grid_axis.resolution)) / 2 subsets.append(WCSTSubset(axis_subset.coverage_axis.axis.label, low, high)) return subsets
def _evaluated_messages(self, grib_file): """ Returns the evaluated_messages for all grib_messages :param String grib_file: path to a grib file :rtype: list[GRIBMessage] """ pygrib = import_pygrib() self.dataset = pygrib.open(grib_file.filepath) evaluated_messages = [] # Message id starts with "1" for i in range(1, self.dataset.messages + 1): grib_message = self.dataset.message(i) axes = [] # Iterate all the axes and evaluate them with message # e.g: Long axis: ${grib:longitudeOfFirstGridPointInDegrees} # Lat axis: ${grib:latitudeOfLastGridPointInDegrees} # Message 1 return: Long: -180, Lat: 90 # Message 2 return: Long: -170, Lat: 80 # ... # Message 20 return: Long: 180, Lat: -90 for user_axis in self.user_axes: # find the crs_axis which are used to evaluate the user_axis (have same name) crs_axis = self._get_crs_axis_by_user_axis_name(user_axis.name) # NOTE: directPositions could be retrieved only when every message evaluated to get values for axis # e.g: message 1 has value: 0, message 3 has value: 2, message 5 has value: 8,...message 20 value: 30 # then, the directPositions of axis is [0, 2, 8,...30] # the syntax to retrieve directions in ingredient file is: ${grib:axis:axis_name} # with axis_name is the name user defined (e.g: AnsiDate?axis-label="time" then axis name is: time) self.evaluator_slice = GribMessageEvaluatorSlice( grib_message, grib_file) evaluated_user_axis = self._user_axis(user_axis, self.evaluator_slice) # When pixelIsPoint:true then it will be adjusted by half pixels for min, max internally (recommended) if self.pixel_is_point is True: PointPixelAdjuster.adjust_axis_bounds_to_continuous_space( evaluated_user_axis, crs_axis) else: # translate the dateTime format to float if evaluated_user_axis.type == UserAxisType.DATE: evaluated_user_axis.interval.low = arrow.get( evaluated_user_axis.interval.low).float_timestamp if evaluated_user_axis.interval.high: evaluated_user_axis.interval.high = arrow.get( evaluated_user_axis.interval.high ).float_timestamp # if low < high, adjust it if evaluated_user_axis.interval.high is not None \ and evaluated_user_axis.interval.low > evaluated_user_axis.interval.high: evaluated_user_axis.interval.low, evaluated_user_axis.interval.high = evaluated_user_axis.interval.high, evaluated_user_axis.interval.low evaluated_user_axis.statements = user_axis.statements axes.append(evaluated_user_axis) evaluated_messages.append(GRIBMessage(i, axes, grib_message)) return evaluated_messages
def _axis_subset(self, crs_axis, nc_file): """ Returns an axis subset using the given crs axis in the context of the nc file :param CRSAxis crs_axis: the crs definition of the axis :param File nc_file: the netcdf file :rtype AxisSubset """ user_axis = self._user_axis(self._get_user_axis_by_crs_axis_name(crs_axis.label), NetcdfEvaluatorSlice(nc_file)) # Normally, without pixelIsPoint:true, in the ingredient needs to +/- 0.5 * resolution for each regular axis # e.g: resolution for axis E is 10000, then # "min": "${netcdf:variable:E:min} - 10000 / 2", # "max": "${netcdf:variable:E:max} + 10000 / 2", # with pixelIsPoint: true, no need to add these values as the service will do it automatically if self.pixel_is_point: PointPixelAdjuster.adjust_axis_bounds_to_continuous_space(user_axis, crs_axis) else: # No adjustment for all regular axes but still need to translate time in datetime to decimal to calculate if user_axis.type == UserAxisType.DATE: user_axis.interval.low = decimal.Decimal(str(arrow.get(user_axis.interval.low).float_timestamp)) if user_axis.interval.high: user_axis.interval.high = decimal.Decimal(str(arrow.get(user_axis.interval.high).float_timestamp)) # if low < high, adjust it if user_axis.interval.high is not None and user_axis.interval.low > user_axis.interval.high: user_axis.interval.low, user_axis.interval.high = user_axis.interval.high, user_axis.interval.low high = user_axis.interval.high if user_axis.interval.high else user_axis.interval.low origin = PointPixelAdjuster.get_origin(user_axis, crs_axis) if isinstance(user_axis, RegularUserAxis): geo_axis = RegularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, origin, crs_axis) else: if user_axis.type == UserAxisType.DATE: if crs_axis.is_uom_day(): coefficients = self._translate_day_date_direct_position_to_coefficients(user_axis.interval.low, user_axis.directPositions) else: coefficients = self._translate_seconds_date_direct_position_to_coefficients(user_axis.interval.low, user_axis.directPositions) else: coefficients = self._translate_number_direct_position_to_coefficients(user_axis.interval.low, user_axis.directPositions) geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, origin, coefficients, crs_axis) grid_low = 0 grid_high = PointPixelAdjuster.get_grid_points(user_axis, crs_axis) # NOTE: Grid Coverage uses the direct intervals as in Rasdaman if self.grid_coverage is False and grid_high > grid_low: grid_high -= 1 grid_axis = GridAxis(user_axis.order, crs_axis.label, user_axis.resolution, grid_low, grid_high) if user_axis.type == UserAxisType.DATE: self._translate_decimal_to_datetime(user_axis, geo_axis) return AxisSubset(CoverageAxis(geo_axis, grid_axis, user_axis.dataBound), Interval(user_axis.interval.low, user_axis.interval.high))
def _adjust_irregular_axis_geo_lower_bound(self, coverage_id, axis_label, axis_type, is_day_unit, axis_geo_lower_bound, slice_group_size): """ (!) Use only if irregular axis's dataBound is False, then adjust it's geo lower bound by current coverage's axis lower bound. e.g: coverage's axis lower bound is '2015-01-10' and slice_group_size = 5 then irregular axis's lower bound from '2015-01-08' or '2015-01-13' will be adjusted to '2015-01-10'. :param int slice_group_size: a positive integer """ resolution = IrregularUserAxis.DEFAULT_RESOLUTION if axis_label not in self.irregular_axis_geo_lower_bound_dict: # Need to parse it from coverage's DescribeCoverage result cov = CoverageUtil(coverage_id) if cov.exists(): axes_labels = cov.get_axes_labels() # Get current coverage's axis geo lower bound i = axes_labels.index(axis_label) lower_bounds = cov.get_axes_lower_bounds() coverage_geo_lower_bound = lower_bounds[i] # Translate datetime format to seconds if axis_type == UserAxisType.DATE: coverage_geo_lower_bound = arrow.get(coverage_geo_lower_bound).float_timestamp axis_geo_lower_bound = arrow.get(axis_geo_lower_bound).float_timestamp else: # Coverage does not exist, first InsertCoverage request self.irregular_axis_geo_lower_bound_dict[axis_label] = axis_geo_lower_bound return axis_geo_lower_bound else: coverage_geo_lower_bound = self.irregular_axis_geo_lower_bound_dict[axis_label] if is_day_unit: # AnsiDate (8600 seconds / day) slice_group_size = slice_group_size * DateTimeUtil.DAY_IN_SECONDS # e.g: irregular axis's level with current coverage's lower bound is 5, axis's lower bound is 12 # and sliceGroupSize = 5. Result is: (12 - 5) / (1 * 5) = 1 distance = math.floor((axis_geo_lower_bound - coverage_geo_lower_bound) / (resolution * slice_group_size)) # So the adjusted value is 5 + 1 * 1 * 5 = 10 adjusted_bound = coverage_geo_lower_bound + (distance * resolution * slice_group_size) if adjusted_bound < coverage_geo_lower_bound: # Save this one to cache for further processing self.irregular_axis_geo_lower_bound_dict[axis_label] = adjusted_bound return adjusted_bound
def _translate_seconds_date_direct_position_to_coefficients( self, origin, direct_positions): # just translate 1 -> 1 as origin is 0 (e.g: irregular UnixTime) if direct_positions == self.DIRECT_POSITIONS_SLICING: return self.COEFFICIENT_SLICING else: return [(decimal.Decimal(str(arrow.get(x).float_timestamp)) - decimal.Decimal(str(origin))) for x in direct_positions]
def _translate_day_date_direct_position_to_coefficients(self, origin, direct_positions): # coefficients in AnsiDate (day) -> coefficients in UnixTime (seconds) coeff_list = [] # as irregular axis gotten from fileName so it is slicing if direct_positions == self.DIRECT_POSITIONS_SLICING: return self.COEFFICIENT_SLICING else: for coeff in direct_positions: # (current_datetime in seconds - origin in seconds) / 86400 coeff_seconds = ((decimal.Decimal(str(arrow.get(coeff).float_timestamp)) - decimal.Decimal(str(origin))) / decimal.Decimal(DateTimeUtil.DAY_IN_SECONDS)) coeff_list.append(coeff_seconds) return coeff_list
def _axis_subset(self, crs_axis, evaluator_slice, resolution=None): """ Returns an axis subset using the given crs axis in the context of the gdal file :param CRSAxis crs_axis: the crs definition of the axis :param GDALEvaluatorSlice evaluator_slice: the evaluator for GDAL file :param resolution: Known axis resolution, no need to evaluate sentence expression from ingredient file (e.g: Sentinel2 recipe) :rtype AxisSubset """ user_axis = self._user_axis( self._get_user_axis_by_crs_axis_name(crs_axis.label), evaluator_slice) if resolution is not None: user_axis.resolution = resolution high = user_axis.interval.high if user_axis.interval.high is not None else user_axis.interval.low if user_axis.type == UserAxisType.DATE: # it must translate datetime string to float by arrow for calculating later user_axis.interval.low = arrow.get( user_axis.interval.low).float_timestamp if user_axis.interval.high is not None: user_axis.interval.high = arrow.get( user_axis.interval.high).float_timestamp if isinstance(user_axis, RegularUserAxis): geo_axis = RegularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, user_axis.interval.low, crs_axis) else: # Irregular axis (coefficients must be number, not datetime string) if user_axis.type == UserAxisType.DATE: if crs_axis.is_time_day_axis(): coefficients = self._translate_day_date_direct_position_to_coefficients( user_axis.interval.low, user_axis.directPositions) else: coefficients = self._translate_seconds_date_direct_position_to_coefficients( user_axis.interval.low, user_axis.directPositions) else: coefficients = self._translate_number_direct_position_to_coefficients( user_axis.interval.low, user_axis.directPositions) self._update_for_slice_group_size(self.coverage_id, user_axis, crs_axis, coefficients) geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, user_axis.interval.low, coefficients, crs_axis) if not crs_axis.is_x_axis() and not crs_axis.is_y_axis(): # GDAL model is 2D so on any axis except x/y we expect to have only one value grid_low = 0 grid_high = None if user_axis.interval.high is not None: grid_high = 0 else: grid_low = 0 number_of_grid_points = decimal.Decimal(str(user_axis.interval.high)) \ - decimal.Decimal(str(user_axis.interval.low)) # number_of_grid_points = (geo_max - geo_min) / resolution grid_high = grid_low + number_of_grid_points / decimal.Decimal( user_axis.resolution) grid_high = HighPixelAjuster.adjust_high(grid_high) # Negative axis, e.g: Latitude (min <--- max) if user_axis.resolution < 0: grid_high = int(abs(math.floor(grid_high))) else: # Positive axis, e.g: Longitude (min --> max) grid_high = int(abs(math.ceil(grid_high))) # NOTE: Grid Coverage uses the direct intervals as in Rasdaman if self.grid_coverage is False and grid_high is not None: if grid_high > grid_low: grid_high -= 1 grid_axis = GridAxis(user_axis.order, crs_axis.label, user_axis.resolution, grid_low, grid_high) geo_axis.origin = PointPixelAdjuster.get_origin(user_axis, crs_axis) if user_axis.type == UserAxisType.DATE: self._translate_decimal_to_datetime(user_axis, geo_axis) # NOTE: current, gdal recipe supports only has 2 axes which are "bounded" (i.e: they exist as 2D axes in file) # and 1 or more another axes gotten (i.e: from fileName) which are not "bounded" to create 3D+ coverage. data_bound = crs_axis.is_y_axis() or crs_axis.is_x_axis() return AxisSubset( CoverageAxis(geo_axis, grid_axis, data_bound), Interval(user_axis.interval.low, user_axis.interval.high))
def single_datetime(datetime, format): if format: return arrow.get(datetime, format).isoformat() else: return arrow.get(datetime).isoformat()