def test_init_interpolators(self):
# should set method
interp = InterpolationManager("nearest")
assert interp.config[(
"default", )]["interpolators"][0].method == "nearest"
# InterpolationManager init should init all interpolators in the list
interp = InterpolationManager([{
"method": "nearest",
"params": {
"spatial_tolerance": 1
}
}])
assert interp.config[(
"default", )]["interpolators"][0].spatial_tolerance == 1
# should throw TraitErrors defined by Interpolator
with pytest.raises(tl.TraitError):
InterpolationManager([{
"method": "nearest",
"params": {
"spatial_tolerance": "tol"
}
}])
with pytest.raises(AttributeError):
assert interp.config[(
"default", )]["interpolators"][0].myarg == "tol"
def _set_interpolation(self):
"""Update _interpolation property"""
# define interpolator with source coordinates dimensions
if isinstance(self.interpolation, InterpolationManager):
self._interpolation = self.interpolation
else:
self._interpolation = InterpolationManager(self.interpolation)
def test_select_coordinates(self):
reqcoords = Coordinates([[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]],
dims=["lat", "lon", "time", "alt"],
crs="+proj=merc +vunits=m")
srccoords = Coordinates([[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]],
dims=["lat", "lon", "time", "alt"],
crs="+proj=merc +vunits=m")
# create a few dummy interpolators that handle certain dimensions
# (can_select is defined by default to look at dims_supported)
class TimeLat(Interpolator):
methods_supported = ["myinterp"]
dims_supported = ["time", "lat"]
def select_coordinates(self, udims, srccoords, srccoords_idx,
reqcoords):
return srccoords, srccoords_idx
class LatLon(Interpolator):
methods_supported = ["myinterp"]
dims_supported = ["lat", "lon"]
def select_coordinates(self, udims, srccoords, srccoords_idx,
reqcoords):
return srccoords, srccoords_idx
class Lon(Interpolator):
methods_supported = ["myinterp"]
dims_supported = ["lon"]
def select_coordinates(self, udims, srccoords, srccoords_idx,
reqcoords):
return srccoords, srccoords_idx
# set up a strange interpolation definition
# we want to interpolate (lat, lon) first, then after (time, alt)
interp = InterpolationManager([
{
"method": "myinterp",
"interpolators": [LatLon, TimeLat],
"dims": ["lat", "lon"]
},
{
"method": "myinterp",
"interpolators": [TimeLat, Lon],
"dims": ["time", "alt"]
},
])
coords, cidx = interp.select_coordinates(srccoords, reqcoords)
assert len(coords) == len(srccoords)
assert len(coords["lat"]) == len(srccoords["lat"])
assert cidx == tuple([slice(0, None)] * 4)
def test_str_definition(self):
# should throw an error if string input is not one of the INTERPOLATION_METHODS
with pytest.raises(InterpolationException):
InterpolationManager("test")
interp = InterpolationManager("nearest")
assert interp.config[("default", )]
assert isinstance(interp.config[("default", )], dict)
assert interp.config[("default", )]["method"] == "nearest"
assert isinstance(interp.config[("default", )]["interpolators"][0],
Interpolator)
def test_nearest_preview_select(self):
# test straight ahead functionality
reqcoords = Coordinates([[-0.5, 1.5, 3.5], [0.5, 2.5, 4.5]],
dims=["lat", "lon"])
srccoords = Coordinates([[0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5]],
dims=["lat", "lon"])
interp = InterpolationManager("nearest_preview")
coords, cidx = interp.select_coordinates(srccoords, reqcoords)
assert len(coords) == len(srccoords) == len(cidx)
assert len(coords["lat"]) == len(reqcoords["lat"])
assert len(coords["lon"]) == len(reqcoords["lon"])
assert np.all(coords["lat"].coordinates == np.array([0, 2, 4]))
# test when selection is applied serially
# this is equivalent to above
reqcoords = Coordinates([[-0.5, 1.5, 3.5], [0.5, 2.5, 4.5]],
dims=["lat", "lon"])
srccoords = Coordinates([[0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5]],
dims=["lat", "lon"])
interp = InterpolationManager([{
"method": "nearest_preview",
"dims": ["lat"]
}, {
"method": "nearest_preview",
"dims": ["lon"]
}])
coords, cidx = interp.select_coordinates(srccoords, reqcoords)
def test_dict_definition(self):
# should handle a default definition without any dimensions
interp = InterpolationManager({
"method": "nearest",
"params": {
"spatial_tolerance": 1
}
})
assert isinstance(interp.config[("default", )], dict)
assert interp.config[("default", )]["method"] == "nearest"
assert isinstance(interp.config[("default", )]["interpolators"][0],
Interpolator)
assert interp.config[("default", )]["params"] == {
"spatial_tolerance": 1
}
# handle string methods
interp = InterpolationManager({
"method": "nearest",
"dims": ["lat", "lon"]
})
print(interp.config)
assert isinstance(interp.config[("lat", "lon")], dict)
assert interp.config[("lat", "lon")]["method"] == "nearest"
assert isinstance(
interp.config[list(interp.config.keys())[-1]]["interpolators"][0],
Interpolator)
assert interp.config[list(interp.config.keys())[-1]]["params"] == {}
# handle dict methods
# should throw an error if method is not in dict
with pytest.raises(InterpolationException):
InterpolationManager([{"test": "test", "dims": ["lat", "lon"]}])
# should throw an error if method is not a string
with pytest.raises(InterpolationException):
InterpolationManager([{"method": 5, "dims": ["lat", "lon"]}])
# should throw an error if method is not one of the INTERPOLATION_METHODS and no interpolators defined
with pytest.raises(InterpolationException):
InterpolationManager([{
"method": "myinter",
"dims": ["lat", "lon"]
}])
# should throw an error if params is not a dict
with pytest.raises(TypeError):
InterpolationManager([{
"method": "nearest",
"dims": ["lat", "lon"],
"params": "test"
}])
# should throw an error if interpolators is not a list
with pytest.raises(TypeError):
InterpolationManager([{
"method": "nearest",
"interpolators": "test",
"dims": ["lat", "lon"]
}])
# should throw an error if interpolators are not Interpolator classes
with pytest.raises(TypeError):
InterpolationManager([{
"method": "nearest",
"interpolators": [NearestNeighbor, "test"],
"dims": ["lat", "lon"]
}])
# should throw an error if dimension is defined twice
with pytest.raises(InterpolationException):
InterpolationManager([{
"method": "nearest",
"dims": ["lat", "lon"]
}, {
"method": "bilinear",
"dims": ["lat"]
}])
# should throw an error if dimension is not a list
with pytest.raises(TypeError):
InterpolationManager([{"method": "nearest", "dims": "lat"}])
# should handle standard INTEPROLATION_SHORTCUTS
interp = InterpolationManager([{
"method": "nearest",
"dims": ["lat", "lon"]
}])
assert isinstance(interp.config[("lat", "lon")], dict)
assert interp.config[("lat", "lon")]["method"] == "nearest"
assert isinstance(interp.config[("lat", "lon")]["interpolators"][0],
Interpolator)
assert interp.config[("lat", "lon")]["params"] == {}
# should not allow custom methods if interpolators can't support
with pytest.raises(InterpolatorException):
interp = InterpolationManager([{
"method":
"myinter",
"interpolators": [NearestNeighbor, NearestPreview],
"dims": ["lat", "lon"]
}])
# should allow custom methods if interpolators can support
class MyInterp(Interpolator):
methods_supported = ["myinter"]
interp = InterpolationManager([{
"method": "myinter",
"interpolators": [MyInterp],
"dims": ["lat", "lon"]
}])
assert interp.config[("lat", "lon")]["method"] == "myinter"
assert isinstance(interp.config[("lat", "lon")]["interpolators"][0],
MyInterp)
# should allow params to be set
interp = InterpolationManager([{
"method": "myinter",
"interpolators": [MyInterp],
"params": {
"spatial_tolerance": 5
},
"dims": ["lat", "lon"],
}])
assert interp.config[("lat", "lon")]["params"] == {
"spatial_tolerance": 5
}
# set default equal to empty tuple
interp = InterpolationManager([{
"method": "bilinear",
"dims": ["lat"]
}])
assert interp.config[list(
interp.config.keys())[-1]]["method"] == INTERPOLATION_DEFAULT
# use default with override if not all dimensions are supplied
interp = InterpolationManager([{
"method": "bilinear",
"dims": ["lat"]
}, "nearest"])
assert interp.config[list(
interp.config.keys())[-1]]["method"] == "nearest"
# make sure default is always the last key in the ordered config dict
interp = InterpolationManager(
["nearest", {
"method": "bilinear",
"dims": ["lat"]
}])
assert list(interp.config.keys())[-1] == ("default", )
# should sort the dims keys
interp = InterpolationManager(
["nearest", {
"method": "bilinear",
"dims": ["lon", "lat"]
}])
assert interp.config[("lat", "lon")]["method"] == "bilinear"
def test_interpolator_init_type(self):
"""test constructor"""
# should throw an error if definition is not str, dict, or Interpolator
with pytest.raises(TypeError):
InterpolationManager(5)
def test_interpolate(self):
class TestInterp(Interpolator):
dims_supported = ["lat", "lon"]
methods_supported = ["myinterp"]
def can_interpolate(self, udims, src, req):
return udims
def interpolate(self, udims, source_coordinates, source_data,
eval_coordinates, output_data):
output_data = source_data
return output_data
# test basic functionality
reqcoords = Coordinates([[-0.5, 1.5, 3.5], [0.5, 2.5, 4.5]],
dims=["lat", "lon"])
srccoords = Coordinates([[0, 2, 4], [0, 3, 4]], dims=["lat", "lon"])
srcdata = UnitsDataArray(
np.random.rand(3, 3),
coords=[srccoords[c].coordinates for c in srccoords],
dims=srccoords.dims)
outdata = UnitsDataArray(
np.zeros(srcdata.shape),
coords=[reqcoords[c].coordinates for c in reqcoords],
dims=reqcoords.dims)
interp = InterpolationManager({
"method": "myinterp",
"interpolators": [TestInterp],
"dims": ["lat", "lon"]
})
outdata = interp.interpolate(srccoords, srcdata, reqcoords, outdata)
assert np.all(outdata == srcdata)
# test if data is size 1
class TestFakeInterp(Interpolator):
dims_supported = ["lat"]
def interpolate(self, udims, source_coordinates, source_data,
eval_coordinates, output_data):
return None
reqcoords = Coordinates([[1]], dims=["lat"])
srccoords = Coordinates([[1]], dims=["lat"])
srcdata = UnitsDataArray(
np.random.rand(1),
coords=[srccoords[c].coordinates for c in srccoords],
dims=srccoords.dims)
outdata = UnitsDataArray(
np.zeros(srcdata.shape),
coords=[reqcoords[c].coordinates for c in reqcoords],
dims=reqcoords.dims)
interp = InterpolationManager({
"method": "myinterp",
"interpolators": [TestFakeInterp],
"dims": ["lat", "lon"]
})
outdata = interp.interpolate(srccoords, srcdata, reqcoords, outdata)
assert np.all(outdata == srcdata)
def test_select_interpolator_queue(self):
reqcoords = Coordinates([[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]],
dims=["lat", "lon", "time", "alt"])
srccoords = Coordinates([[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]],
dims=["lat", "lon", "time", "alt"])
# create a few dummy interpolators that handle certain dimensions
# (can_select is defined by default to look at dims_supported)
class TimeLat(Interpolator):
methods_supported = ["myinterp"]
dims_supported = ["time", "lat"]
def can_select(self, udims, source_coordinates, eval_coordinates):
return self._filter_udims_supported(udims)
def can_interpolate(self, udims, source_coordinates,
eval_coordinates):
return self._filter_udims_supported(udims)
class LatLon(Interpolator):
methods_supported = ["myinterp"]
dims_supported = ["lat", "lon"]
def can_select(self, udims, source_coordinates, eval_coordinates):
return self._filter_udims_supported(udims)
def can_interpolate(self, udims, source_coordinates,
eval_coordinates):
return self._filter_udims_supported(udims)
class Lon(Interpolator):
methods_supported = ["myinterp"]
dims_supported = ["lon"]
def can_select(self, udims, source_coordinates, eval_coordinates):
return self._filter_udims_supported(udims)
def can_interpolate(self, udims, source_coordinates,
eval_coordinates):
return self._filter_udims_supported(udims)
# set up a strange interpolation definition
# we want to interpolate (lat, lon) first, then after (time, alt)
interp = InterpolationManager([
{
"method": "myinterp",
"interpolators": [LatLon, TimeLat],
"dims": ["lat", "lon"]
},
{
"method": "myinterp",
"interpolators": [TimeLat, Lon],
"dims": ["time", "alt"]
},
])
# default = 'nearest', which will return NearestPreview for can_select
interpolator_queue = interp._select_interpolator_queue(
srccoords, reqcoords, "can_select")
assert isinstance(interpolator_queue, OrderedDict)
assert isinstance(interpolator_queue[("lat", "lon")], LatLon)
assert ("time", "alt") not in interpolator_queue and (
"alt", "time") not in interpolator_queue
# should throw an error if strict is set and not all dimensions can be handled
with pytest.raises(InterpolationException):
interp_copy = deepcopy(interp)
interpolator_queue = interp_copy._select_interpolator_queue(
srccoords, reqcoords, "can_select", strict=True)
# default = Nearest, which can handle all dims for can_interpolate
interpolator_queue = interp._select_interpolator_queue(
srccoords, reqcoords, "can_interpolate")
assert isinstance(interpolator_queue, OrderedDict)
assert isinstance(interpolator_queue[("lat", "lon")], LatLon)
class Interpolate(Node):
"""Node to used to interpolate from self.source.coordinates to the user-specified, evaluated coordinates.
Parameters
----------
source : Any
The source node which will be interpolated
interpolation : str, dict, optional
Interpolation definition for the data source.
By default, the interpolation method is set to ``'nearest'`` for all dimensions.
If input is a string, it must match one of the interpolation shortcuts defined in
:attr:`podpac.data.INTERPOLATION_SHORTCUTS`. The interpolation method associated
with this string will be applied to all dimensions at the same time.
If input is a dict or list of dict, the dict or dict elements must adhere to the following format:
The key ``'method'`` defining the interpolation method name.
If the interpolation method is not one of :attr:`podpac.data.INTERPOLATION_SHORTCUTS`, a
second key ``'interpolators'`` must be defined with a list of
:class:`podpac.interpolators.Interpolator` classes to use in order of uages.
The dictionary may contain an option ``'params'`` key which contains a dict of parameters to pass along to
the :class:`podpac.interpolators.Interpolator` classes associated with the interpolation method.
The dict may contain the key ``'dims'`` which specifies dimension names (i.e. ``'time'`` or ``('lat', 'lon')`` ).
If the dictionary does not contain a key for all unstacked dimensions of the source coordinates, the
:attr:`podpac.data.INTERPOLATION_DEFAULT` value will be used.
All dimension keys must be unstacked even if the underlying coordinate dimensions are stacked.
Any extra dimensions included but not found in the source coordinates will be ignored.
The dict may contain a key ``'params'`` that can be used to configure the :class:`podpac.interpolators.Interpolator` classes associated with the interpolation method.
If input is a :class:`podpac.data.Interpolation` class, this Interpolation
class will be used without modification.
cache_output : bool
Should the node's output be cached? If not provided or None, uses default based on
settings["CACHE_DATASOURCE_OUTPUT_DEFAULT"]. If True, outputs will be cached and retrieved from cache. If False,
outputs will not be cached OR retrieved from cache (even if they exist in cache).
Examples
-----
# To use bilinear interpolation for [lat,lon] a specific interpolator for [time], and the default for [alt], use:
>>> interp_node = Interpolation(
source=some_node,
interpolation=interpolation = [
{
'method': 'bilinear',
'dims': ['lat', 'lon']
},
{
'method': [podpac.interpolators.NearestNeighbor],
'dims': ['time']
}
]
)
"""
source = NodeTrait(allow_none=True).tag(attr=True)
_source_xr = tl.Instance(UnitsDataArray, allow_none=True) # This is needed for the Interpolation Mixin
interpolation = InterpolationTrait().tag(attr=True)
cache_output = tl.Bool()
# privates
_interpolation = tl.Instance(InterpolationManager)
_coordinates = tl.Instance(Coordinates, allow_none=True, default_value=None, read_only=True)
_requested_source_coordinates = tl.Instance(Coordinates)
_requested_source_coordinates_index = tl.Tuple()
_requested_source_data = tl.Instance(UnitsDataArray)
_evaluated_coordinates = tl.Instance(Coordinates)
# this adds a more helpful error message if user happens to try an inspect _interpolation before evaluate
@tl.default("_interpolation")
def _default_interpolation(self):
self._set_interpolation()
return self._interpolation
@tl.default("cache_output")
def _cache_output_default(self):
return settings["CACHE_NODE_OUTPUT_DEFAULT"]
# ------------------------------------------------------------------------------------------------------------------
# Properties
# ------------------------------------------------------------------------------------------------------------------
@property
def interpolation_class(self):
"""Get the interpolation class currently set for this data source.
The DataSource ``interpolation`` property is used to define the
:class:`podpac.data.InterpolationManager` class that will handle interpolation for requested coordinates.
Returns
-------
:class:`podpac.data.InterpolationManager`
InterpolationManager class defined by DataSource `interpolation` definition
"""
return self._interpolation
@property
def interpolators(self):
"""Return the interpolators selected for the previous node evaluation interpolation.
If the node has not been evaluated, or if interpolation was not necessary, this will return
an empty OrderedDict
Returns
-------
OrderedDict
Key are tuple of unstacked dimensions, the value is the interpolator used to interpolate these dimensions
"""
if self._interpolation._last_interpolator_queue is not None:
return self._interpolation._last_interpolator_queue
else:
return OrderedDict()
def _set_interpolation(self):
"""Update _interpolation property"""
# define interpolator with source coordinates dimensions
if isinstance(self.interpolation, InterpolationManager):
self._interpolation = self.interpolation
else:
self._interpolation = InterpolationManager(self.interpolation)
def _eval(self, coordinates, output=None, _selector=None):
"""Evaluates this node using the supplied coordinates.
The coordinates are mapped to the requested coordinates, interpolated if necessary, and set to
`_requested_source_coordinates` with associated index `_requested_source_coordinates_index`. The requested
source coordinates and index are passed to `get_data()` returning the source data at the
coordinatesset to `_requested_source_data`. Finally `_requested_source_data` is interpolated
using the `interpolate` method and set to the `output` attribute of the node.
Parameters
----------
coordinates : :class:`podpac.Coordinates`
{requested_coordinates}
An exception is raised if the requested coordinates are missing dimensions in the DataSource.
Extra dimensions in the requested coordinates are dropped.
output : :class:`podpac.UnitsDataArray`, optional
{eval_output}
_selector :
{eval_selector}
Returns
-------
{eval_return}
Raises
------
ValueError
Cannot evaluate these coordinates
"""
_logger.debug("Evaluating {} data source".format(self.__class__.__name__))
# store requested coordinates for debugging
if settings["DEBUG"]:
self._original_requested_coordinates = coordinates
# store input coordinates to evaluated coordinates
self._evaluated_coordinates = deepcopy(coordinates)
# reset interpolation
self._set_interpolation()
selector = self._interpolation.select_coordinates
source_out = self._source_eval(self._evaluated_coordinates, selector)
source_coords = Coordinates.from_xarray(source_out.coords, crs=source_out.crs)
# Drop extra coordinates
extra_dims = [d for d in coordinates.udims if d not in source_coords.udims]
coordinates = coordinates.drop(extra_dims)
# Transform so that interpolation happens on the source data coordinate system
if source_coords.crs.lower() != coordinates.crs.lower():
coordinates = coordinates.transform(source_coords.crs)
if output is None:
if "output" in source_out.dims:
self.set_trait("outputs", source_out.coords["output"].data.tolist())
output = self.create_output_array(coordinates)
if source_out.size == 0: # short cut
return output
# interpolate data into output
output = self._interpolation.interpolate(source_coords, source_out, coordinates, output)
# if requested crs is differented than coordinates,
# fabricate a new output with the original coordinates and new values
if self._evaluated_coordinates.crs != coordinates.crs:
output = self.create_output_array(self._evaluated_coordinates.drop(extra_dims), data=output[:].values)
# save output to private for debugging
if settings["DEBUG"]:
self._output = output
self._source_xr = source_out
return output
def _source_eval(self, coordinates, selector, output=None):
if isinstance(self._source_xr, UnitsDataArray):
return self._source_xr
else:
return self.source.eval(coordinates, output=output, _selector=selector)
def find_coordinates(self):
"""
Get the available coordinates for the Node. For a DataSource, this is just the coordinates.
Returns
-------
coords_list : list
singleton list containing the coordinates (Coordinates object)
"""
return self.source.find_coordinates()