def _generate_initial_gml_db(self):
"""
Generates the initial slice in gml for importing using the database method and returns the gml for it
:rtype: File
"""
# Transform the axes domains such that only a point is defined.
# For the first slice we need to import a single point, which will then be updated with the real data
axes_map = OrderedDict()
for axis, grid_axis in self.coverage.get_insert_axes().items():
if axis.coefficient is not None:
assert type(axis.coefficient) == list, "Axis coefficients not of type list."
assert len(axis.coefficient) > 0, "The list of coefficients is empty."
# Get the first coefficient in irregular coverage to create a initial slice
axis = IrregularAxis(axis.label, axis.uomLabel, axis.low, axis.high, axis.origin, [axis.coefficient[0]],
axis.crs_axis)
axes_map[axis] = GridAxis(grid_axis.order, grid_axis.label, grid_axis.resolution, 0, 0)
metadata_provider = MetadataProvider(self.coverage.coverage_id, axes_map,
self.coverage.range_fields, self.coverage.crs, self.coverage.metadata,
self.grid_coverage)
tuple_list = []
# Tuple list for InsertCoverage request should be created from null values if they exist
for range_field in self.coverage.range_fields:
# If band doesn't have null value, default insert value is 0
insert_value = self.DEFAULT_INSERT_VALUE
if (range_field.nilValues is not None) and (len(range_field.nilValues) > 0):
insert_value = strip_trailing_zeros(range_field.nilValues[0].value.split(":")[0])
if insert_value == "":
insert_value = self.DEFAULT_INSERT_VALUE
tuple_list.append(insert_value)
data_provider = TupleListDataProvider(",".join(tuple_list))
mediator = Mediator(metadata_provider, data_provider)
return mediator.get_gml_file() if ConfigManager.mock else mediator.get_gml_str()
def _fill_grid_axis(self):
grid_axis_x = GridAxis(0, "", self.gdal_dataset.get_resolution_x(), 0,
self.gdal_dataset.get_raster_x_size() - 1)
grid_axis_y = GridAxis(1, "", self.gdal_dataset.get_resolution_y(), 0,
self.gdal_dataset.get_raster_y_size() - 1)
for i in range(0, len(self.subsets)):
if self.subsets[i].coverage_axis.axis.crs_axis.is_easting():
grid_axis_x.label = self.subsets[i].coverage_axis.axis.label
self.subsets[i].coverage_axis.grid_axis = grid_axis_x
elif self.subsets[i].coverage_axis.axis.crs_axis.is_northing():
grid_axis_y.label = self.subsets[i].coverage_axis.axis.label
self.subsets[i].coverage_axis.grid_axis = grid_axis_y
else:
self.subsets[i].coverage_axis.grid_axis = GridAxis(i, self.subsets[i].coverage_axis.axis.label, 1, 0, 0)
return self.subsets
def _generate_initial_gml_db(self):
"""
Generates the initial slice in gml for importing using the database method and returns the gml for it
:rtype: File
"""
# Transform the axes domains such that only a point is defined.
# For the first slice we need to import a single point, which will then be updated with the real data
axes_map = OrderedDict()
for axis, grid_axis in self.coverage.get_insert_axes().iteritems():
if axis.coefficient is not None:
assert type(axis.coefficient
) == list, "Axis coefficients not of type list."
assert len(
axis.coefficient) > 0, "The list of coefficients is empty."
# Get the first coefficient in irregular coverage to create a initial slice
axis = IrregularAxis(axis.label, axis.uomLabel, axis.low,
axis.high, axis.origin,
[axis.coefficient[0]], axis.crs_axis)
axes_map[axis] = GridAxis(grid_axis.order, grid_axis.label,
grid_axis.resolution, 0, 0)
metadata_provider = MetadataProvider(
self.coverage.coverage_id, axes_map, self.coverage.range_fields,
self.coverage.crs, self.coverage.metadata, self.grid_coverage)
tuple_list = ",".join(['0'] * len(self.coverage.range_fields))
data_provider = TupleListDataProvider(tuple_list)
file = Mediator(metadata_provider, data_provider).get_gml_file()
return file
def _fill_grid_axis(self, timeseries_recipe=False):
x_order = self.GRID_AXIS_X_ORDER
y_order = self.GRID_AXIS_Y_ORDER
if timeseries_recipe:
x_order = self.GRID_AXIS_X_ORDER_SHIFTED
y_order = self.GRID_AXIS_Y_ORDER_SHIFTED
grid_axis_x = GridAxis(x_order, "",
self.gdal_dataset.get_resolution_x(), 0,
self.gdal_dataset.get_raster_x_size() - 1)
grid_axis_y = GridAxis(y_order, "",
self.gdal_dataset.get_resolution_y(), 0,
self.gdal_dataset.get_raster_y_size() - 1)
for i in range(0, len(self.subsets)):
if self.subsets[i].coverage_axis.axis.crs_axis.is_x_axis():
grid_axis_x.label = self.subsets[i].coverage_axis.axis.label
self.subsets[i].coverage_axis.grid_axis = grid_axis_x
elif self.subsets[i].coverage_axis.axis.crs_axis.is_y_axis():
grid_axis_y.label = self.subsets[i].coverage_axis.axis.label
self.subsets[i].coverage_axis.grid_axis = grid_axis_y
else:
self.subsets[i].coverage_axis.grid_axis = GridAxis(
i, self.subsets[i].coverage_axis.axis.label, 1, 0, 0)
return self.subsets
def _axis_subset(self, crs_axis, nc_file):
"""
Returns an axis subset using the given crs axis in the context of the nc file
:param CRSAxis crs_axis: the crs definition of the axis
:param File nc_file: the netcdf file
:rtype AxisSubset
"""
user_axis = self._user_axis(self._get_user_axis_by_crs_axis_name(crs_axis.label), NetcdfEvaluatorSlice(nc_file))
# Normally, without pixelIsPoint:true, in the ingredient needs to +/- 0.5 * resolution for each regular axis
# e.g: resolution for axis E is 10000, then
# "min": "${netcdf:variable:E:min} - 10000 / 2",
# "max": "${netcdf:variable:E:max} + 10000 / 2",
# with pixelIsPoint: true, no need to add these values as the service will do it automatically
if self.pixel_is_point:
PointPixelAdjuster.adjust_axis_bounds_to_continuous_space(user_axis, crs_axis)
else:
# No adjustment for all regular axes but still need to translate time in datetime to decimal to calculate
if user_axis.type == UserAxisType.DATE:
user_axis.interval.low = decimal.Decimal(str(arrow.get(user_axis.interval.low).float_timestamp))
if user_axis.interval.high:
user_axis.interval.high = decimal.Decimal(str(arrow.get(user_axis.interval.high).float_timestamp))
# if low < high, adjust it
if user_axis.interval.high is not None and user_axis.interval.low > user_axis.interval.high:
user_axis.interval.low, user_axis.interval.high = user_axis.interval.high, user_axis.interval.low
high = user_axis.interval.high if user_axis.interval.high else user_axis.interval.low
origin = PointPixelAdjuster.get_origin(user_axis, crs_axis)
if isinstance(user_axis, RegularUserAxis):
geo_axis = RegularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, origin, crs_axis)
else:
if user_axis.type == UserAxisType.DATE:
if crs_axis.is_uom_day():
coefficients = self._translate_day_date_direct_position_to_coefficients(user_axis.interval.low,
user_axis.directPositions)
else:
coefficients = self._translate_seconds_date_direct_position_to_coefficients(user_axis.interval.low,
user_axis.directPositions)
else:
coefficients = self._translate_number_direct_position_to_coefficients(user_axis.interval.low,
user_axis.directPositions)
geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom, user_axis.interval.low, high, origin, coefficients, crs_axis)
grid_low = 0
grid_high = PointPixelAdjuster.get_grid_points(user_axis, crs_axis)
# NOTE: Grid Coverage uses the direct intervals as in Rasdaman
if self.grid_coverage is False and grid_high > grid_low:
grid_high -= 1
grid_axis = GridAxis(user_axis.order, crs_axis.label, user_axis.resolution, grid_low, grid_high)
if user_axis.type == UserAxisType.DATE:
self._translate_decimal_to_datetime(user_axis, geo_axis)
return AxisSubset(CoverageAxis(geo_axis, grid_axis, user_axis.dataBound),
Interval(user_axis.interval.low, user_axis.interval.high))
def _fill_grid_axis(self):
grid_axis_x = GridAxis(0, "", self.gdal_dataset.get_resolution_x(), 0,
self.gdal_dataset.get_raster_x_size() - 1)
grid_axis_y = GridAxis(1, "", self.gdal_dataset.get_resolution_y(), 0,
self.gdal_dataset.get_raster_y_size() - 1)
for i in range(0, len(self.subsets)):
if self.subsets[i].coverage_axis.axis.crs_axis.is_easting():
grid_axis_x.label = self.subsets[i].coverage_axis.axis.label
self.subsets[i].coverage_axis.grid_axis = grid_axis_x
elif self.subsets[i].coverage_axis.axis.crs_axis.is_northing():
grid_axis_y.label = self.subsets[i].coverage_axis.axis.label
self.subsets[i].coverage_axis.grid_axis = grid_axis_y
else:
self.subsets[i].coverage_axis.grid_axis = GridAxis(
i, self.subsets[i].coverage_axis.axis.label, 1, 0, 0)
return self.subsets
def _get_coverage_axes(self, geo_coords, raster_coords, origin, crs_axes,
resolutions):
"""
Generates the coverage axes for this coverage
:param list[Interval] geo_coords: the geographical coords in the order of the gml
:param list[Interval] raster_coords: the grid coords in the order of the grid (not necessarily the order of the geo axes)
:param list[str] origin: the origin of the coverage
:param list[CRSAxis] crs_axes: a list of the crs axes
:param list[dict] resolutions: a list of triples containing the resolution, coefficient list and position in grid
:rtype: list[CoverageAxis]
"""
axis_index = 0
coverage_axes = []
for crs_axis in crs_axes:
resolution = resolutions[axis_index]
order = resolution['position']
grid_axis = GridAxis(order, crs_axis.label,
resolution['resolution'],
raster_coords[order].low,
raster_coords[order].high)
if resolution['coefficient'] is not None:
geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom,
geo_coords[axis_index].low,
geo_coords[axis_index].high,
origin[axis_index],
resolution['coefficient'], crs_axis)
else:
geo_axis = RegularAxis(crs_axis.label, crs_axis.uom,
geo_coords[axis_index].low,
geo_coords[axis_index].high,
origin[axis_index], crs_axis)
data_bound = crs_axis.is_easting() or crs_axis.is_northing()
coverage_axis = CoverageAxis(geo_axis, grid_axis, data_bound)
coverage_axes.append(coverage_axis)
axis_index += 1
return coverage_axes
def _axis_subset(self, crs_axis, evaluator_slice, resolution=None):
"""
Returns an axis subset using the given crs axis in the context of the gdal file
:param CRSAxis crs_axis: the crs definition of the axis
:param GDALEvaluatorSlice evaluator_slice: the evaluator for GDAL file
:param resolution: Known axis resolution, no need to evaluate sentence expression from ingredient file (e.g: Sentinel2 recipe)
:rtype AxisSubset
"""
user_axis = self._user_axis(
self._get_user_axis_by_crs_axis_name(crs_axis.label),
evaluator_slice)
if resolution is not None:
user_axis.resolution = resolution
high = user_axis.interval.high if user_axis.interval.high is not None else user_axis.interval.low
if user_axis.type == UserAxisType.DATE:
# it must translate datetime string to float by arrow for calculating later
user_axis.interval.low = arrow.get(
user_axis.interval.low).float_timestamp
if user_axis.interval.high is not None:
user_axis.interval.high = arrow.get(
user_axis.interval.high).float_timestamp
if isinstance(user_axis, RegularUserAxis):
geo_axis = RegularAxis(crs_axis.label, crs_axis.uom,
user_axis.interval.low, high,
user_axis.interval.low, crs_axis)
else:
# Irregular axis (coefficients must be number, not datetime string)
if user_axis.type == UserAxisType.DATE:
if crs_axis.is_time_day_axis():
coefficients = self._translate_day_date_direct_position_to_coefficients(
user_axis.interval.low, user_axis.directPositions)
else:
coefficients = self._translate_seconds_date_direct_position_to_coefficients(
user_axis.interval.low, user_axis.directPositions)
else:
coefficients = self._translate_number_direct_position_to_coefficients(
user_axis.interval.low, user_axis.directPositions)
self._update_for_slice_group_size(self.coverage_id, user_axis,
crs_axis, coefficients)
geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom,
user_axis.interval.low, high,
user_axis.interval.low, coefficients,
crs_axis)
if not crs_axis.is_x_axis() and not crs_axis.is_y_axis():
# GDAL model is 2D so on any axis except x/y we expect to have only one value
grid_low = 0
grid_high = None
if user_axis.interval.high is not None:
grid_high = 0
else:
grid_low = 0
number_of_grid_points = decimal.Decimal(str(user_axis.interval.high)) \
- decimal.Decimal(str(user_axis.interval.low))
# number_of_grid_points = (geo_max - geo_min) / resolution
grid_high = grid_low + number_of_grid_points / decimal.Decimal(
user_axis.resolution)
grid_high = HighPixelAjuster.adjust_high(grid_high)
# Negative axis, e.g: Latitude (min <--- max)
if user_axis.resolution < 0:
grid_high = int(abs(math.floor(grid_high)))
else:
# Positive axis, e.g: Longitude (min --> max)
grid_high = int(abs(math.ceil(grid_high)))
# NOTE: Grid Coverage uses the direct intervals as in Rasdaman
if self.grid_coverage is False and grid_high is not None:
if grid_high > grid_low:
grid_high -= 1
grid_axis = GridAxis(user_axis.order, crs_axis.label,
user_axis.resolution, grid_low, grid_high)
geo_axis.origin = PointPixelAdjuster.get_origin(user_axis, crs_axis)
if user_axis.type == UserAxisType.DATE:
self._translate_decimal_to_datetime(user_axis, geo_axis)
# NOTE: current, gdal recipe supports only has 2 axes which are "bounded" (i.e: they exist as 2D axes in file)
# and 1 or more another axes gotten (i.e: from fileName) which are not "bounded" to create 3D+ coverage.
data_bound = crs_axis.is_y_axis() or crs_axis.is_x_axis()
return AxisSubset(
CoverageAxis(geo_axis, grid_axis, data_bound),
Interval(user_axis.interval.low, user_axis.interval.high))
def _axis_subset(self, grib_file, evaluated_messages, crs_axis):
"""
Returns an axis subset using the given crs axis in the context of the grib file
:param File grib_file: the current grib file (slice) is evaluated
:param List[GirbMessages] evaluated_messages: all Grib messages was evaluated
:param CRSAxis crs_axis: the crs definition of the axis
:rtype AxisSubset
"""
# first grib message from grib file, used to extract grib variables only
first_grib_message = self.dataset.message(1)
# As all the messages contain same axes (but different intervals), so first message is ok to get user_axis
first_user_axis = self._get_user_axis_in_evaluated_message(
evaluated_messages[0], crs_axis.label)
# NOTE: we don't want to change this user_axis belongs to messages, so clone it
user_axis = copy.deepcopy(first_user_axis)
# Then, we calculate the geo, grid bounds, origin, resolution of this axis for the slice
self._set_low_high(evaluated_messages, user_axis)
high = user_axis.interval.high if user_axis.interval.high is not None else user_axis.interval.low
origin = PointPixelAdjuster.get_origin(user_axis, crs_axis)
if isinstance(user_axis, RegularUserAxis):
geo_axis = RegularAxis(crs_axis.label, crs_axis.uom,
user_axis.interval.low, high, origin,
crs_axis)
else:
# after all messages was evaluated, we could get the direct_positions of the axis as in netcdf
# then, it can evaluate the grib sentence normally, e.g: ${grib:axis:level} + 5
evaluating_sentence = user_axis.directPositions
direct_positions = self._get_axis_values(evaluated_messages,
user_axis)
# convert all of values in the list to string then it can be evaluated
direct_positions = list_util.to_list_string(direct_positions)
evaluator_slice = GribMessageEvaluatorSlice(
first_grib_message, grib_file, direct_positions)
user_axis.directPositions = self.sentence_evaluator.evaluate(
evaluating_sentence, evaluator_slice, user_axis.statements)
# axis is datetime
if user_axis.type == UserAxisType.DATE:
if crs_axis.is_time_day_axis():
coefficients = self._translate_day_date_direct_position_to_coefficients(
user_axis.interval.low, user_axis.directPositions)
else:
coefficients = self._translate_seconds_date_direct_position_to_coefficients(
user_axis.interval.low, user_axis.directPositions)
else:
# number axis like Index1D
coefficients = self._translate_number_direct_position_to_coefficients(
user_axis.interval.low, user_axis.directPositions)
self._update_for_slice_group_size(self.coverage_id, user_axis,
crs_axis, coefficients)
geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom,
user_axis.interval.low, high, origin,
coefficients, crs_axis)
grid_low = 0
grid_high = PointPixelAdjuster.get_grid_points(user_axis, crs_axis)
# NOTE: Grid Coverage uses the direct intervals as in Rasdaman
if self.grid_coverage is False and grid_high > grid_low:
grid_high -= 1
grid_axis = GridAxis(user_axis.order, crs_axis.label,
user_axis.resolution, grid_low, grid_high)
if user_axis.type == UserAxisType.DATE:
self._translate_decimal_to_datetime(user_axis, geo_axis)
return AxisSubset(
CoverageAxis(geo_axis, grid_axis, user_axis.dataBound),
Interval(user_axis.interval.low, user_axis.interval.high))
def _axis_subset(self, crs_axis, gdal_file):
"""
Returns an axis subset using the given crs axis in the context of the gdal file
:param CRSAxis crs_axis: the crs definition of the axis
:param File gdal_file: the gdal file
:rtype AxisSubset
"""
user_axis = self._user_axis(
self._get_user_axis_by_crs_axis_name(crs_axis.label),
GDALEvaluatorSlice(GDALGmlUtil(gdal_file.get_filepath())))
high = user_axis.interval.high if user_axis.interval.high else user_axis.interval.low
if isinstance(user_axis, RegularUserAxis):
geo_axis = RegularAxis(crs_axis.label, crs_axis.uom,
user_axis.interval.low, high,
user_axis.interval.low, crs_axis)
else:
# if irregular axis value is fetched from fileName so the coefficient is [0] as slicing
if user_axis.directPositions == AbstractToCoverageConverter.DIRECT_POSITIONS_SLICING:
user_axis.directPositions = AbstractToCoverageConverter.COEFFICIENT_SLICING
geo_axis = IrregularAxis(crs_axis.label, crs_axis.uom,
user_axis.interval.low, high,
user_axis.interval.low,
user_axis.directPositions, crs_axis)
if not crs_axis.is_easting() and not crs_axis.is_northing():
# GDAL model is 2D so on any axis except x/y we expect to have only one value
grid_low = 0
grid_high = 0
else:
grid_low = 0
number_of_grid_points = decimal.Decimal(str(user_axis.interval.high)) \
- decimal.Decimal(str(user_axis.interval.low))
# number_of_grid_points = (geo_max - geo_min) / resolution
grid_high = grid_low + number_of_grid_points / decimal.Decimal(
user_axis.resolution)
grid_high = HighPixelAjuster.adjust_high(grid_high)
# Negative axis, e.g: Latitude (min <--- max)
if user_axis.resolution < 0:
grid_high = int(abs(math.floor(grid_high)))
else:
# Positive axis, e.g: Longitude (min --> max)
grid_high = int(abs(math.ceil(grid_high)))
# NOTE: Grid Coverage uses the direct intervals as in Rasdaman
if self.grid_coverage is False:
if grid_high > grid_low:
grid_high -= 1
grid_axis = GridAxis(user_axis.order, crs_axis.label,
user_axis.resolution, grid_low, grid_high)
geo_axis.origin = PointPixelAdjuster.get_origin(user_axis, crs_axis)
if user_axis.type == UserAxisType.DATE:
self._translate_decimal_to_datetime(user_axis, geo_axis)
# NOTE: current, gdal recipe supports only has 2 axes which are "bounded" (i.e: they exist as 2D axes in file)
# and 1 or more another axes gotten (i.e: from fileName) which are not "bounded" to create 3D+ coverage.
data_bound = crs_axis.is_northing() or crs_axis.is_easting()
return AxisSubset(
CoverageAxis(geo_axis, grid_axis, data_bound),
Interval(user_axis.interval.low, user_axis.interval.high))