def get_origin_for_time_axis(user_axis, crs_axis):
user_axis.interval.low = arrow.get(
user_axis.interval.low).float_timestamp
if user_axis.interval.high:
user_axis.interval.high = arrow.get(
user_axis.interval.high).float_timestamp
if isinstance(user_axis, RegularUserAxis):
# ansidate, need to calculate with day in seconds
if crs_axis.is_time_day_axis:
if user_axis.resolution > 0 or user_axis.interval.high is None:
# axis goes from low to high, so origin is lowest, with half a pixel shift
return decimal.Decimal(str(user_axis.interval.low))\
+ decimal.Decimal(0.5) * decimal.Decimal(str(user_axis.resolution)) * DateTimeUtil.DAY_IN_SECONDS
else:
# axis goes from high to low, so origin is highest, with half pixel shift (resolution is negative)
return decimal.Decimal(str(user_axis.interval.high))\
+ decimal.Decimal(0.5) * decimal.Decimal(str(user_axis.resolution)) * DateTimeUtil.DAY_IN_SECONDS
else:
# unix time, already in seconds
if user_axis.resolution > 0 or user_axis.interval.high is None:
# axis goes from low to high, so origin is lowest, with half a pixel shift
return decimal.Decimal(str(user_axis.interval.low))\
+ decimal.Decimal(0.5) * decimal.Decimal(str(user_axis.resolution))
else:
# axis goes from high to low, so origin is highest, with half pixel shift (resolution is negative)
return decimal.Decimal(str(user_axis.interval.high))\
+ decimal.Decimal(0.5) * decimal.Decimal(str(user_axis.resolution))
else:
# irregular axis, the same but without shift
if user_axis.resolution > 0 or user_axis.interval.high is None:
return user_axis.interval.low
else:
return user_axis.interval.high
def adjust_axis_bounds_for_time_axis(user_axis, crs_axis):
# convert to timestamp and change the axis type
user_axis.interval.low = arrow.get(
user_axis.interval.low).float_timestamp
if user_axis.interval.high:
user_axis.interval.high = arrow.get(
user_axis.interval.high).float_timestamp
# if low < high, adjust it
if user_axis.interval.high is not None and user_axis.interval.low > user_axis.interval.high:
user_axis.interval.low, user_axis.interval.high = user_axis.interval.high, user_axis.interval.low
# The formula for all regular axes is when "pixelIsPoint":
# min = min - 0.5 * resolution
# max = max + 0.5 * resolution
if isinstance(user_axis, RegularUserAxis):
# if axis is time axis
if crs_axis.is_time_day_axis():
user_axis.interval.low = decimal.Decimal(str(user_axis.interval.low)) \
- decimal.Decimal(0.5) * decimal.Decimal(str(abs(user_axis.resolution))) * DateTimeUtil.DAY_IN_SECONDS
if user_axis.interval.high:
user_axis.interval.high = decimal.Decimal(str(user_axis.interval.high)) \
+ decimal.Decimal(0.5) * decimal.Decimal(str(abs(user_axis.resolution))) * DateTimeUtil.DAY_IN_SECONDS
else:
# if axis is normal axis (lat, lon, index1d,...)
user_axis.interval.low = decimal.Decimal(str(user_axis.interval.low)) \
- decimal.Decimal(0.5) * decimal.Decimal(str(abs(user_axis.resolution)))
if user_axis.interval.high:
user_axis.interval.high = decimal.Decimal(str(user_axis.interval.high)) \
+ decimal.Decimal(0.5) * decimal.Decimal(str(abs(user_axis.resolution)))
def _get_update_subsets_for_slice(self, slice):
"""
Returns the given slice's interval as a list of wcst subsets
:param slice: the slice for which to generate this
:rtype: list[WCSTSubset]
"""
subsets = []
for axis_subset in slice.axis_subsets:
low = axis_subset.interval.low
high = axis_subset.interval.high
# if ConfigManager.subset_correction and high is not None and low != high and type(low) != str:
if ConfigManager.subset_correction and high is not None and low != high and type(low) == str:
# Time axis with type = str (e.g: "1970-01-01T02:03:06Z")
time_seconds = 1
# AnsiDate (need to change from date to seconds)
if axis_subset.coverage_axis.axis.crs_axis.is_time_day_axis():
time_seconds = DateTimeUtil.DAY_IN_SECONDS
low = decimal.Decimal(str(arrow.get(low).float_timestamp)) + decimal.Decimal(str(axis_subset.coverage_axis.grid_axis.resolution * time_seconds)) / 2
low = DateTimeUtil.get_datetime_iso(low)
if high is not None:
high = decimal.Decimal(str(arrow.get(high).float_timestamp)) - decimal.Decimal(str(axis_subset.coverage_axis.grid_axis.resolution * time_seconds)) / 2
high = DateTimeUtil.get_datetime_iso(high)
elif ConfigManager.subset_correction and high is not None and low != high and type(low) != str:
# regular axes (e.g: latitude, longitude, index1d)
low = decimal.Decimal(str(low)) + decimal.Decimal(str(axis_subset.coverage_axis.grid_axis.resolution)) / 2
if high is not None:
high = decimal.Decimal(str(high)) - decimal.Decimal(str(axis_subset.coverage_axis.grid_axis.resolution)) / 2
subsets.append(WCSTSubset(axis_subset.coverage_axis.axis.label, low, high))
return subsets
def _evaluated_messages(self, grib_file):
"""
Returns the evaluated_messages for all grib_messages
:param String grib_file: path to a grib file
:rtype: list[GRIBMessage]
"""
pygrib = import_pygrib()
self.dataset = pygrib.open(grib_file.filepath)
evaluated_messages = []
# Message id starts with "1"
for i in range(1, self.dataset.messages + 1):
grib_message = self.dataset.message(i)
axes = []
# Iterate all the axes and evaluate them with message
# e.g: Long axis: ${grib:longitudeOfFirstGridPointInDegrees}
# Lat axis: ${grib:latitudeOfLastGridPointInDegrees}
# Message 1 return: Long: -180, Lat: 90
# Message 2 return: Long: -170, Lat: 80
# ...
# Message 20 return: Long: 180, Lat: -90
for user_axis in self.user_axes:
# find the crs_axis which are used to evaluate the user_axis (have same name)
crs_axis = self._get_crs_axis_by_user_axis_name(user_axis.name)
# NOTE: directPositions could be retrieved only when every message evaluated to get values for axis
# e.g: message 1 has value: 0, message 3 has value: 2, message 5 has value: 8,...message 20 value: 30
# then, the directPositions of axis is [0, 2, 8,...30]
# the syntax to retrieve directions in ingredient file is: ${grib:axis:axis_name}
# with axis_name is the name user defined (e.g: AnsiDate?axis-label="time" then axis name is: time)
self.evaluator_slice = GribMessageEvaluatorSlice(
grib_message, grib_file)
evaluated_user_axis = self._user_axis(user_axis,
self.evaluator_slice)
# When pixelIsPoint:true then it will be adjusted by half pixels for min, max internally (recommended)
if self.pixel_is_point is True:
PointPixelAdjuster.adjust_axis_bounds_to_continuous_space(
evaluated_user_axis, crs_axis)
else:
# translate the dateTime format to float
if evaluated_user_axis.type == UserAxisType.DATE:
evaluated_user_axis.interval.low = arrow.get(
evaluated_user_axis.interval.low).float_timestamp
if evaluated_user_axis.interval.high:
evaluated_user_axis.interval.high = arrow.get(
evaluated_user_axis.interval.high
).float_timestamp
# if low < high, adjust it
if evaluated_user_axis.interval.high is not None \
and evaluated_user_axis.interval.low > evaluated_user_axis.interval.high:
evaluated_user_axis.interval.low, evaluated_user_axis.interval.high = evaluated_user_axis.interval.high, evaluated_user_axis.interval.low
evaluated_user_axis.statements = user_axis.statements
axes.append(evaluated_user_axis)
evaluated_messages.append(GRIBMessage(i, axes, grib_message))
return evaluated_messages
def _axis_subset(self, crs_axis, nc_file):
"""
Returns an axis subset using the given crs axis in the context of the nc file
:param CRSAxis crs_axis: the crs definition of the axis
:param File nc_file: the netcdf file
:rtype AxisSubset
"""
user_axis = self._user_axis(self._get_user_axis_by_crs_axis_name(crs_axis.label), NetcdfEvaluatorSlice(nc_file))
# Normally, without pixelIsPoint:true, in the ingredient needs to +/- 0.5 * resolution for each regular axis
# e.g: resolution for axis E is 10000, then
# "min": "${netcdf:variable:E:min} - 10000 / 2",
# "max": "${netcdf:variable:E:max} + 10000 / 2",
# with pixelIsPoint: true, no need to add these values as the service will do it automatically
if self.pixel_is_point:
PointPixelAdjuster.adjust_axis_bounds_to_continuous_space(user_axis, crs_axis)
else:
# No adjustment for all regular axes but still need to translate time in datetime to decimal to calculate
if user_axis.type == UserAxisType.DATE:
user_axis.interval.low = decimal.Decimal(str(arrow.get(user_axis.interval.low).float_timestamp))
if user_axis.interval.high:
user_axis.interval.high = decimal.Decimal(str(arrow.get(user_axis.interval.high).float_timestamp))
# if low < high, adjust it
if user_axis.interval.high is not None and user_axis.interval.low > user_axis.interval.high:
user_axis.interval.low, user_axis.interval.high = user_axis.interval.high, user_axis.interval.low
high = user_axis.interval.high if user_axis.interval.high else user_axis.interval.low
origin = PointPixelAdjuster.get_origin(user_axis, crs_axis)
if isinstance(user_axis, RegularUserAxis):
geo_axis = RegularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, origin, crs_axis)
else:
if user_axis.type == UserAxisType.DATE:
if crs_axis.is_uom_day():
coefficients = self._translate_day_date_direct_position_to_coefficients(user_axis.interval.low,
user_axis.directPositions)
else:
coefficients = self._translate_seconds_date_direct_position_to_coefficients(user_axis.interval.low,
user_axis.directPositions)
else:
coefficients = self._translate_number_direct_position_to_coefficients(user_axis.interval.low,
user_axis.directPositions)
geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, origin, coefficients, crs_axis)
grid_low = 0
grid_high = PointPixelAdjuster.get_grid_points(user_axis, crs_axis)
# NOTE: Grid Coverage uses the direct intervals as in Rasdaman
if self.grid_coverage is False and grid_high > grid_low:
grid_high -= 1
grid_axis = GridAxis(user_axis.order, crs_axis.label, user_axis.resolution, grid_low, grid_high)
if user_axis.type == UserAxisType.DATE:
self._translate_decimal_to_datetime(user_axis, geo_axis)
return AxisSubset(CoverageAxis(geo_axis, grid_axis, user_axis.dataBound),
Interval(user_axis.interval.low, user_axis.interval.high))
def _adjust_irregular_axis_geo_lower_bound(self, coverage_id, axis_label, axis_type, is_day_unit, axis_geo_lower_bound, slice_group_size):
"""
(!) Use only if irregular axis's dataBound is False, then adjust it's geo lower bound by current coverage's axis lower bound.
e.g: coverage's axis lower bound is '2015-01-10' and slice_group_size = 5 then irregular axis's lower bound
from '2015-01-08' or '2015-01-13' will be adjusted to '2015-01-10'.
:param int slice_group_size: a positive integer
"""
resolution = IrregularUserAxis.DEFAULT_RESOLUTION
if axis_label not in self.irregular_axis_geo_lower_bound_dict:
# Need to parse it from coverage's DescribeCoverage result
cov = CoverageUtil(coverage_id)
if cov.exists():
axes_labels = cov.get_axes_labels()
# Get current coverage's axis geo lower bound
i = axes_labels.index(axis_label)
lower_bounds = cov.get_axes_lower_bounds()
coverage_geo_lower_bound = lower_bounds[i]
# Translate datetime format to seconds
if axis_type == UserAxisType.DATE:
coverage_geo_lower_bound = arrow.get(coverage_geo_lower_bound).float_timestamp
axis_geo_lower_bound = arrow.get(axis_geo_lower_bound).float_timestamp
else:
# Coverage does not exist, first InsertCoverage request
self.irregular_axis_geo_lower_bound_dict[axis_label] = axis_geo_lower_bound
return axis_geo_lower_bound
else:
coverage_geo_lower_bound = self.irregular_axis_geo_lower_bound_dict[axis_label]
if is_day_unit:
# AnsiDate (8600 seconds / day)
slice_group_size = slice_group_size * DateTimeUtil.DAY_IN_SECONDS
# e.g: irregular axis's level with current coverage's lower bound is 5, axis's lower bound is 12
# and sliceGroupSize = 5. Result is: (12 - 5) / (1 * 5) = 1
distance = math.floor((axis_geo_lower_bound - coverage_geo_lower_bound) / (resolution * slice_group_size))
# So the adjusted value is 5 + 1 * 1 * 5 = 10
adjusted_bound = coverage_geo_lower_bound + (distance * resolution * slice_group_size)
if adjusted_bound < coverage_geo_lower_bound:
# Save this one to cache for further processing
self.irregular_axis_geo_lower_bound_dict[axis_label] = adjusted_bound
return adjusted_bound
def _translate_seconds_date_direct_position_to_coefficients(
self, origin, direct_positions):
# just translate 1 -> 1 as origin is 0 (e.g: irregular UnixTime)
if direct_positions == self.DIRECT_POSITIONS_SLICING:
return self.COEFFICIENT_SLICING
else:
return [(decimal.Decimal(str(arrow.get(x).float_timestamp)) -
decimal.Decimal(str(origin))) for x in direct_positions]
def _translate_day_date_direct_position_to_coefficients(self, origin, direct_positions):
# coefficients in AnsiDate (day) -> coefficients in UnixTime (seconds)
coeff_list = []
# as irregular axis gotten from fileName so it is slicing
if direct_positions == self.DIRECT_POSITIONS_SLICING:
return self.COEFFICIENT_SLICING
else:
for coeff in direct_positions:
# (current_datetime in seconds - origin in seconds) / 86400
coeff_seconds = ((decimal.Decimal(str(arrow.get(coeff).float_timestamp)) - decimal.Decimal(str(origin)))
/ decimal.Decimal(DateTimeUtil.DAY_IN_SECONDS))
coeff_list.append(coeff_seconds)
return coeff_list
def _axis_subset(self, crs_axis, evaluator_slice, resolution=None):
"""
Returns an axis subset using the given crs axis in the context of the gdal file
:param CRSAxis crs_axis: the crs definition of the axis
:param GDALEvaluatorSlice evaluator_slice: the evaluator for GDAL file
:param resolution: Known axis resolution, no need to evaluate sentence expression from ingredient file (e.g: Sentinel2 recipe)
:rtype AxisSubset
"""
user_axis = self._user_axis(
self._get_user_axis_by_crs_axis_name(crs_axis.label),
evaluator_slice)
if resolution is not None:
user_axis.resolution = resolution
high = user_axis.interval.high if user_axis.interval.high is not None else user_axis.interval.low
if user_axis.type == UserAxisType.DATE:
# it must translate datetime string to float by arrow for calculating later
user_axis.interval.low = arrow.get(
user_axis.interval.low).float_timestamp
if user_axis.interval.high is not None:
user_axis.interval.high = arrow.get(
user_axis.interval.high).float_timestamp
if isinstance(user_axis, RegularUserAxis):
geo_axis = RegularAxis(crs_axis.label, crs_axis.uom,
user_axis.interval.low, high,
user_axis.interval.low, crs_axis)
else:
# Irregular axis (coefficients must be number, not datetime string)
if user_axis.type == UserAxisType.DATE:
if crs_axis.is_time_day_axis():
coefficients = self._translate_day_date_direct_position_to_coefficients(
user_axis.interval.low, user_axis.directPositions)
else:
coefficients = self._translate_seconds_date_direct_position_to_coefficients(
user_axis.interval.low, user_axis.directPositions)
else:
coefficients = self._translate_number_direct_position_to_coefficients(
user_axis.interval.low, user_axis.directPositions)
self._update_for_slice_group_size(self.coverage_id, user_axis,
crs_axis, coefficients)
geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom,
user_axis.interval.low, high,
user_axis.interval.low, coefficients,
crs_axis)
if not crs_axis.is_x_axis() and not crs_axis.is_y_axis():
# GDAL model is 2D so on any axis except x/y we expect to have only one value
grid_low = 0
grid_high = None
if user_axis.interval.high is not None:
grid_high = 0
else:
grid_low = 0
number_of_grid_points = decimal.Decimal(str(user_axis.interval.high)) \
- decimal.Decimal(str(user_axis.interval.low))
# number_of_grid_points = (geo_max - geo_min) / resolution
grid_high = grid_low + number_of_grid_points / decimal.Decimal(
user_axis.resolution)
grid_high = HighPixelAjuster.adjust_high(grid_high)
# Negative axis, e.g: Latitude (min <--- max)
if user_axis.resolution < 0:
grid_high = int(abs(math.floor(grid_high)))
else:
# Positive axis, e.g: Longitude (min --> max)
grid_high = int(abs(math.ceil(grid_high)))
# NOTE: Grid Coverage uses the direct intervals as in Rasdaman
if self.grid_coverage is False and grid_high is not None:
if grid_high > grid_low:
grid_high -= 1
grid_axis = GridAxis(user_axis.order, crs_axis.label,
user_axis.resolution, grid_low, grid_high)
geo_axis.origin = PointPixelAdjuster.get_origin(user_axis, crs_axis)
if user_axis.type == UserAxisType.DATE:
self._translate_decimal_to_datetime(user_axis, geo_axis)
# NOTE: current, gdal recipe supports only has 2 axes which are "bounded" (i.e: they exist as 2D axes in file)
# and 1 or more another axes gotten (i.e: from fileName) which are not "bounded" to create 3D+ coverage.
data_bound = crs_axis.is_y_axis() or crs_axis.is_x_axis()
return AxisSubset(
CoverageAxis(geo_axis, grid_axis, data_bound),
Interval(user_axis.interval.low, user_axis.interval.high))
def single_datetime(datetime, format):
if format:
return arrow.get(datetime, format).isoformat()
else:
return arrow.get(datetime).isoformat()
