def test_interpolate_irregular_arbitrary_2dims(self):
""" irregular interpolation """
# Note, this test also tests the looper helper
# try >2 dims
source = np.random.rand(5, 5, 3)
coords_src = Coordinates(
[clinspace(0, 10, 5),
clinspace(0, 10, 5), [2, 3, 5]],
dims=["lat", "lon", "time"])
coords_dst = Coordinates(
[clinspace(1, 11, 5),
clinspace(1, 11, 5), [2, 3, 4]],
dims=["lat", "lon", "time"])
node = MockArrayDataSource(
data=source,
coordinates=coords_src,
interpolation=[{
"method": "nearest",
"interpolators": [ScipyGrid]
}, {
"method": "linear",
"dims": ["time"]
}],
)
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst["lat"].coordinates)
assert np.all(output.lon.values == coords_dst["lon"].coordinates)
assert np.all(output.time.values == coords_dst["time"].coordinates)
def test_interpolate_irregular_lat_lon(self):
""" irregular interpolation """
source = np.random.rand(5, 5)
coords_src = Coordinates(
[clinspace(0, 10, 5), clinspace(0, 10, 5)], dims=["lat", "lon"])
coords_dst = Coordinates([[[0, 2, 4, 6, 8, 10], [0, 2, 4, 5, 6, 10]]],
dims=["lat_lon"])
node = MockArrayDataSource(data=source,
coordinates=coords_src,
interpolation={
"method": "nearest",
"interpolators": [ScipyGrid]
})
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert "lat_lon" in output.dims
np.testing.assert_array_equal(output["lat"].values,
coords_dst["lat"].coordinates)
np.testing.assert_array_equal(output["lon"].values,
coords_dst["lon"].coordinates)
assert output.values[0] == source[0, 0]
assert output.values[1] == source[1, 1]
assert output.values[-1] == source[-1, -1]
def test_respect_bounds(self):
source = np.random.rand(5)
coords_src = Coordinates([[1, 2, 3, 4, 5]], ["alt"])
coords_dst = Coordinates([[-0.5, 1.1, 2.6]], ["alt"])
node = MockArrayDataSource(
data=source,
coordinates=coords_src,
interpolation={
"method": "nearest",
"interpolators": [NearestNeighbor],
"params": {
"respect_bounds": False
},
},
)
output = node.eval(coords_dst)
np.testing.assert_array_equal(output.data, source[[0, 0, 2]])
node = MockArrayDataSource(
data=source,
coordinates=coords_src,
interpolation={
"method": "nearest",
"interpolators": [NearestNeighbor],
"params": {
"respect_bounds": True
}
},
)
output = node.eval(coords_dst)
np.testing.assert_array_equal(output.data[1:], source[[0, 2]])
assert np.isnan(output.data[0])
def test_slice_index(self):
selector = Selector("nearest")
src = Coordinates([[0, 1, 2, 3, 4, 5]], dims=["lat"])
# uniform
req = Coordinates([[2, 4]], dims=["lat"])
c, ci = selector.select(src, req, index_type="slice")
assert isinstance(ci[0], slice)
assert c == src[ci]
# non uniform
req = Coordinates([[1, 2, 4]], dims=["lat"])
c, ci = selector.select(src, req, index_type="slice")
assert isinstance(ci[0], slice)
assert c == src[ci]
# empty
req = Coordinates([[10]], dims=["lat"])
c, ci = selector.select(src, req, index_type="slice")
assert isinstance(ci[0], slice)
assert c == src[ci]
# singleton
req = Coordinates([[2]], dims=["lat"])
c, ci = selector.select(src, req, index_type="slice")
assert isinstance(ci[0], slice)
assert c == src[ci]
def test_interpolate_scipy_point(self):
""" interpolate point data to nearest neighbor with various coords_dst"""
source = np.random.rand(6)
coords_src = Coordinates(
[[[0, 2, 4, 6, 8, 10], [0, 2, 4, 5, 6, 10]]], dims=['lat_lon'])
coords_dst = Coordinates([[[1, 2, 3, 4, 5], [1, 2, 3, 4, 5]]],
dims=['lat_lon'])
node = MockArrayDataSource(source=source,
native_coordinates=coords_src,
interpolation={
'method': 'nearest',
'interpolators': [ScipyPoint]
})
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(
output.lat_lon.values == coords_dst.coords['lat_lon'])
assert output.values[0] == source[0]
assert output.values[-1] == source[3]
coords_dst = Coordinates([[1, 2, 3, 4, 5], [1, 2, 3, 4, 5]],
dims=['lat', 'lon'])
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst.coords['lat'])
assert output.values[0, 0] == source[0]
assert output.values[-1, -1] == source[3]
def test_interpolate_scipy_point(self):
""" interpolate point data to nearest neighbor with various coords_dst"""
source = np.random.rand(6)
coords_src = Coordinates([[[0, 2, 4, 6, 8, 10], [0, 2, 4, 5, 6, 10]]],
dims=["lat_lon"])
coords_dst = Coordinates([[[1, 2, 3, 4, 5], [1, 2, 3, 4, 5]]],
dims=["lat_lon"])
node = MockArrayDataSource(data=source,
coordinates=coords_src,
interpolation={
"method": "nearest",
"interpolators": [ScipyPoint]
})
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert "lat_lon" in output.dims
np.testing.assert_array_equal(output.lat.values,
coords_dst["lat"].coordinates)
np.testing.assert_array_equal(output.lon.values,
coords_dst["lon"].coordinates)
assert output.values[0] == source[0]
assert output.values[-1] == source[3]
coords_dst = Coordinates([[1, 2, 3, 4, 5], [1, 2, 3, 4, 5]],
dims=["lat", "lon"])
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
np.testing.assert_array_equal(output.lat.values,
coords_dst["lat"].coordinates)
assert output.values[0, 0] == source[0]
assert output.values[-1, -1] == source[3]
def test_evaluate_extra_dims(self):
# drop extra unstacked dimension
class MyDataSource(DataSource):
coordinates = Coordinates([1, 11], dims=["lat", "lon"])
def get_data(self, coordinates, coordinates_index):
return self.create_output_array(coordinates)
node = MyDataSource()
coords = Coordinates([1, 11, "2018-01-01"],
dims=["lat", "lon", "time"])
output = node.eval(coords)
assert output.dims == ("lat", "lon") # time dropped
# drop extra stacked dimension if none of its dimensions are needed
class MyDataSource(DataSource):
coordinates = Coordinates(["2018-01-01"], dims=["time"])
def get_data(self, coordinates, coordinates_index):
return self.create_output_array(coordinates)
node = MyDataSource()
coords = Coordinates([[1, 11], "2018-01-01"], dims=["lat_lon", "time"])
output = node.eval(coords)
assert output.dims == ("time", ) # lat_lon dropped
def test_interpolate_irregular_arbitrary_2dims(self):
""" irregular interpolation """
# try >2 dims
source = np.random.rand(5, 5, 3)
coords_src = Coordinates(
[clinspace(0, 10, 5),
clinspace(0, 10, 5), [2, 3, 5]],
dims=['lat', 'lon', 'time'])
coords_dst = Coordinates(
[clinspace(1, 11, 5),
clinspace(1, 11, 5), [2, 3, 5]],
dims=['lat', 'lon', 'time'])
node = MockArrayDataSource(source=source,
native_coordinates=coords_src,
interpolation={
'method': 'nearest',
'interpolators': [ScipyGrid]
})
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst.coords['lat'])
assert np.all(output.lon.values == coords_dst.coords['lon'])
assert np.all(output.time.values == coords_dst.coords['time'])
def test_spatial_tolerance(self):
# unstacked 1D
source = np.random.rand(5)
coords_src = Coordinates([np.linspace(0, 10, 5)], dims=['lat'])
node = MockArrayDataSource(source=source,
native_coordinates=coords_src,
interpolation={
'default': {
'method': 'nearest',
'params': {
'spatial_tolerance': 1.1
}
}
})
coords_dst = Coordinates([[1, 1.2, 1.5, 5, 9]], dims=['lat'])
output = node.eval(coords_dst)
print(output)
print(source)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst.coords['lat'])
assert output.values[0] == source[0] and \
np.isnan(output.values[1]) and \
output.values[2] == source[1]
def test_time_tolerance(self):
# unstacked 1D
source = np.random.rand(5, 5)
coords_src = Coordinates([
np.linspace(0, 10, 5),
clinspace('2018-01-01', '2018-01-09', 5)
],
dims=['lat', 'time'])
node = MockArrayDataSource(source=source,
native_coordinates=coords_src,
interpolation={
'default': {
'method': 'nearest',
'params': {
'spatial_tolerance':
1.1,
'time_tolerance':
np.timedelta64(1, 'D')
}
}
})
coords_dst = Coordinates(
[[1, 1.2, 1.5, 5, 9],
clinspace('2018-01-01', '2018-01-09', 3)],
dims=['lat', 'time'])
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst.coords['lat'])
assert output.values[0, 0] == source[0, 0] and output.values[0, 1] == source[0, 2] and \
np.isnan(output.values[1, 0]) and np.isnan(output.values[1, 1]) and \
output.values[2, 0] == source[1, 0] and output.values[2, 1] == source[1, 2]
def test_time_tolerance(self):
# unstacked 1D
source = np.random.rand(5, 5)
coords_src = Coordinates(
[np.linspace(0, 10, 5),
clinspace("2018-01-01", "2018-01-09", 5)],
dims=["lat", "time"])
node = MockArrayDataSource(
data=source,
coordinates=coords_src,
interpolation={
"method": "nearest",
"params": {
"spatial_tolerance": 1.1,
"time_tolerance": np.timedelta64(1, "D")
},
},
)
coords_dst = Coordinates([[1, 1.2, 1.5, 5, 9],
clinspace("2018-01-01", "2018-01-09", 3)],
dims=["lat", "time"])
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst["lat"].coordinates)
assert (output.values[0, 0] == source[0, 0]
and output.values[0, 1] == source[0, 2]
and np.isnan(output.values[1, 0])
and np.isnan(output.values[1, 1])
and output.values[2, 0] == source[1, 0]
and output.values[2, 1] == source[1, 2])
def test_interpolate(self):
class TestInterp(Interpolator):
dims_supported = ['lat', 'lon']
def interpolate(self, udims, source_coordinates, source_data,
eval_coordinates, output_data):
output_data = source_data
return output_data
# test basic functionality
reqcoords = Coordinates([[-.5, 1.5, 3.5], [.5, 2.5, 4.5]],
dims=['lat', 'lon'])
srccoords = Coordinates([[0, 2, 4], [0, 3, 4]], dims=['lat', 'lon'])
srcdata = UnitsDataArray(
np.random.rand(3, 3),
coords=[srccoords[c].coordinates for c in srccoords],
dims=srccoords.dims)
outdata = UnitsDataArray(
np.zeros(srcdata.shape),
coords=[reqcoords[c].coordinates for c in reqcoords],
dims=reqcoords.dims)
interp = Interpolation({
('lat', 'lon'): {
'method': 'test',
'interpolators': [TestInterp]
}
})
outdata = interp.interpolate(srccoords, srcdata, reqcoords, outdata)
assert np.all(outdata == srcdata)
# test if data is size 1
class TestFakeInterp(Interpolator):
dims_supported = ['lat']
def interpolate(self, udims, source_coordinates, source_data,
eval_coordinates, output_data):
return None
reqcoords = Coordinates([[1]], dims=['lat'])
srccoords = Coordinates([[1]], dims=['lat'])
srcdata = UnitsDataArray(
np.random.rand(1),
coords=[srccoords[c].coordinates for c in srccoords],
dims=srccoords.dims)
outdata = UnitsDataArray(
np.zeros(srcdata.shape),
coords=[reqcoords[c].coordinates for c in reqcoords],
dims=reqcoords.dims)
interp = Interpolation({
('lat', 'lon'): {
'method': 'test',
'interpolators': [TestFakeInterp]
}
})
outdata = interp.interpolate(srccoords, srcdata, reqcoords, outdata)
assert np.all(outdata == srcdata)
def test_nearest_selector_negative_time_step(self):
selector = Selector("nearest")
request1 = Coordinates([clinspace("2020-01-01", "2020-01-11", 11)],
["time"])
request2 = Coordinates([clinspace("2020-01-11", "2020-01-01", 11)],
["time"])
source1 = Coordinates(
[clinspace("2020-01-22T00", "2020-01-01T00", 126)], ["time"])
source2 = Coordinates(
[clinspace("2020-01-01T00", "2020-01-22T00", 126)], ["time"])
c11, ci11 = selector.select(source1, request1)
assert len(c11["time"]) == 11
assert len(ci11[0]) == 11
c12, ci12 = selector.select(source1, request2)
assert len(c12["time"]) == 11
assert len(ci12[0]) == 11
c21, ci21 = selector.select(source2, request1)
assert len(c21["time"]) == 11
assert len(ci21[0]) == 11
c22, ci22 = selector.select(source2, request2)
assert len(c22["time"]) == 11
assert len(ci22[0]) == 11
np.testing.assert_equal(ci11[0], ci12[0])
np.testing.assert_equal(ci21[0], ci22[0])
def test_nearest_preview_select(self):
# test straight ahead functionality
reqcoords = Coordinates([[-0.5, 1.5, 3.5], [0.5, 2.5, 4.5]],
dims=["lat", "lon"])
srccoords = Coordinates([[0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5]],
dims=["lat", "lon"])
interp = InterpolationManager("nearest_preview")
coords, cidx = interp.select_coordinates(srccoords, reqcoords)
assert len(coords) == len(srccoords) == len(cidx)
assert len(coords["lat"]) == len(reqcoords["lat"])
assert len(coords["lon"]) == len(reqcoords["lon"])
assert np.all(coords["lat"].coordinates == np.array([0, 2, 4]))
# test when selection is applied serially
# this is equivalent to above
reqcoords = Coordinates([[-0.5, 1.5, 3.5], [0.5, 2.5, 4.5]],
dims=["lat", "lon"])
srccoords = Coordinates([[0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5]],
dims=["lat", "lon"])
interp = InterpolationManager([{
"method": "nearest_preview",
"dims": ["lat"]
}, {
"method": "nearest_preview",
"dims": ["lon"]
}])
coords, cidx = interp.select_coordinates(srccoords, reqcoords)
def test_linear_1D_issue411and413(self):
data = [0, 1, 2]
raw_coords = data.copy()
raw_e_coords = [0, 0.5, 1, 1.5, 2]
for dim in ["lat", "lon", "alt", "time"]:
ec = Coordinates([raw_e_coords], [dim])
arrb = ArrayBase(source=data, coordinates=Coordinates([raw_coords], [dim]))
node = Interpolate(source=arrb, interpolation="linear")
o = node.eval(ec)
np.testing.assert_array_equal(o.data, raw_e_coords, err_msg="dim {} failed to interpolate".format(dim))
# Do time interpolation explicitly
raw_coords = ["2020-11-01", "2020-11-03", "2020-11-05"]
raw_et_coords = ["2020-11-01", "2020-11-02", "2020-11-03", "2020-11-04", "2020-11-05"]
ec = Coordinates([raw_et_coords], ["time"])
arrb = ArrayBase(source=data, coordinates=Coordinates([raw_coords], ["time"]))
node = Interpolate(source=arrb, interpolation="linear")
o = node.eval(ec)
np.testing.assert_array_equal(
o.data, raw_e_coords, err_msg="dim time failed to interpolate with datetime64 coords"
)
class TestInterpolation(object):
s1 = ArrayBase(
source=np.random.rand(9, 15),
coordinates=Coordinates([np.linspace(0, 8, 9), np.linspace(0, 14, 15)], ["lat", "lon"]),
)
s2 = ArrayBase(
source=np.random.rand(9, 15),
coordinates=Coordinates([np.linspace(9, 17, 9), np.linspace(0, 14, 15)], ["lat", "lon"]),
)
interp = Interpolate(source=s1, interpolation="nearest")
coords = Coordinates([np.linspace(0, 8, 17), np.linspace(0, 14, 29)], ["lat", "lon"])
coords2 = Coordinates([np.linspace(0, 17, 18), np.linspace(0, 14, 15)], ["lat", "lon"])
coords2c = Coordinates([np.linspace(0.1, 16.8, 5), np.linspace(0.1, 13.8, 3)], ["lat", "lon"])
def test_basic_interpolation(self):
# This JUST tests the interface, tests for the actual value of the interpolation is left
# to the test_interpolation_manager.py file
o = self.interp.eval(self.coords)
assert o.shape == (17, 29)
def test_interpolation_definition(self):
node = Node.from_json(self.interp.json)
o1 = node.eval(self.coords)
o2 = self.interp.eval(self.coords)
np.testing.assert_array_equal(o1.data, o2.data)
assert node.json == self.interp.json
def test_compositor_chain(self):
dc = DataCompositor(sources=[self.s2, self.s1])
node = Interpolate(source=dc, interpolation="nearest")
o = node.eval(self.coords2)
np.testing.assert_array_equal(o.data, np.concatenate([self.s1.source, self.s2.source], axis=0))
def test_interpolate_rasterio(self):
""" regular interpolation using rasterio"""
assert rasterio is not None
source = np.arange(0, 15)
source.resize((3, 5))
coords_src = Coordinates(
[clinspace(0, 10, 3), clinspace(0, 10, 5)], dims=["lat", "lon"])
coords_dst = Coordinates(
[clinspace(1, 11, 3), clinspace(1, 11, 5)], dims=["lat", "lon"])
# try one specific rasterio case to measure output
node = MockArrayDataSource(data=source,
coordinates=coords_src,
interpolation={
"method": "min",
"interpolators": [Rasterio]
})
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst["lat"].coordinates)
assert output.data[0, 3] == 3.0
assert output.data[0, 4] == 4.0
node = MockArrayDataSource(data=source,
coordinates=coords_src,
interpolation={
"method": "max",
"interpolators": [Rasterio]
})
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst["lat"].coordinates)
assert output.data[0, 3] == 9.0
assert output.data[0, 4] == 9.0
# TODO boundary should be able to use a default
node = MockArrayDataSource(
data=source,
coordinates=coords_src,
interpolation={
"method": "bilinear",
"interpolators": [Rasterio]
},
boundary={
"lat": 2.5,
"lon": 1.25
},
)
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst["lat"].coordinates)
np.testing.assert_allclose(
output, [[1.4, 2.4, 3.4, 4.4, 5.0], [6.4, 7.4, 8.4, 9.4, 10.0],
[10.4, 11.4, 12.4, 13.4, 14.0]])
def test_select_coordinates(self):
reqcoords = Coordinates([[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]],
dims=["lat", "lon", "time", "alt"],
crs="+proj=merc +vunits=m")
srccoords = Coordinates([[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]],
dims=["lat", "lon", "time", "alt"],
crs="+proj=merc +vunits=m")
# create a few dummy interpolators that handle certain dimensions
# (can_select is defined by default to look at dims_supported)
class TimeLat(Interpolator):
methods_supported = ["myinterp"]
dims_supported = ["time", "lat"]
def select_coordinates(self, udims, srccoords, srccoords_idx,
reqcoords):
return srccoords, srccoords_idx
class LatLon(Interpolator):
methods_supported = ["myinterp"]
dims_supported = ["lat", "lon"]
def select_coordinates(self, udims, srccoords, srccoords_idx,
reqcoords):
return srccoords, srccoords_idx
class Lon(Interpolator):
methods_supported = ["myinterp"]
dims_supported = ["lon"]
def select_coordinates(self, udims, srccoords, srccoords_idx,
reqcoords):
return srccoords, srccoords_idx
# set up a strange interpolation definition
# we want to interpolate (lat, lon) first, then after (time, alt)
interp = InterpolationManager([
{
"method": "myinterp",
"interpolators": [LatLon, TimeLat],
"dims": ["lat", "lon"]
},
{
"method": "myinterp",
"interpolators": [TimeLat, Lon],
"dims": ["time", "alt"]
},
])
coords, cidx = interp.select_coordinates(srccoords, reqcoords)
assert len(coords) == len(srccoords)
assert len(coords["lat"]) == len(srccoords["lat"])
assert cidx == tuple([slice(0, None)] * 4)
def test_ignored_interpolation_params_issue340(self, caplog):
node = Array(
source=[0, 1, 2],
coordinates=Coordinates([[0, 2, 1]], dims=["time"]),
interpolation={"method": "nearest", "params": {"fake_param": 1.1, "spatial_tolerance": 1}},
)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
node.eval(Coordinates([[0.5, 1.5]], ["time"]))
assert "interpolation parameter 'fake_param' was ignored" in caplog.text
assert "interpolation parameter 'spatial_tolerance' was ignored" not in caplog.text
def test_set_coordinates(self):
node = MockDataSource()
node.set_coordinates(Coordinates([]))
assert node.coordinates == Coordinates([])
assert node.coordinates != node.get_coordinates()
# don't overwrite
node = MockDataSource()
node.coordinates
node.set_coordinates(Coordinates([]))
assert node.coordinates != Coordinates([])
assert node.coordinates == node.get_coordinates()
def test_interpolate_scipy_grid(self):
source = np.arange(0, 25)
source.resize((5, 5))
coords_src = Coordinates(
[clinspace(0, 10, 5), clinspace(0, 10, 5)], dims=["lat", "lon"])
coords_dst = Coordinates(
[clinspace(1, 11, 5), clinspace(1, 11, 5)], dims=["lat", "lon"])
# try one specific rasterio case to measure output
node = MockArrayDataSource(data=source,
coordinates=coords_src,
interpolation={
"method": "nearest",
"interpolators": [ScipyGrid]
})
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst["lat"].coordinates)
print(output)
assert output.data[0, 0] == 0.0
assert output.data[0, 3] == 3.0
assert output.data[1, 3] == 8.0
assert np.isnan(output.data[0,
4]) # TODO: how to handle outside bounds
node = MockArrayDataSource(data=source,
coordinates=coords_src,
interpolation={
"method": "cubic_spline",
"interpolators": [ScipyGrid]
})
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst["lat"].coordinates)
assert int(output.data[0, 0]) == 2
assert int(output.data[2, 4]) == 16
node = MockArrayDataSource(data=source,
coordinates=coords_src,
interpolation={
"method": "bilinear",
"interpolators": [ScipyGrid]
})
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst["lat"].coordinates)
assert int(output.data[0, 0]) == 2
assert int(output.data[3, 3]) == 20
assert np.isnan(output.data[4,
4]) # TODO: how to handle outside bounds
def test_spatial_tolerance(self):
# unstacked 1D
source = np.random.rand(5)
coords_src = Coordinates([np.linspace(0, 10, 5)], dims=["lat"])
node = MockArrayDataSource(
data=source,
coordinates=coords_src,
interpolation={
"method": "nearest",
"params": {
"spatial_tolerance": 1.1
}
},
)
coords_dst = Coordinates([[1, 1.2, 1.5, 5, 9]], dims=["lat"])
output = node.eval(coords_dst)
print(output)
print(source)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst["lat"].coordinates)
assert output.values[0] == source[0] and np.isnan(
output.values[1]) and output.values[2] == source[1]
# stacked 1D
source = np.random.rand(5)
coords_src = Coordinates(
[[np.linspace(0, 10, 5),
np.linspace(0, 10, 5)]],
dims=[["lat", "lon"]])
node = MockArrayDataSource(
data=source,
coordinates=coords_src,
interpolation={
"method": "nearest",
"params": {
"spatial_tolerance": 1.1
}
},
)
coords_dst = Coordinates([[[1, 1.2, 1.5, 5, 9], [1, 1.2, 1.5, 5, 9]]],
dims=[["lat", "lon"]])
output = node.eval(coords_dst)
print(output)
print(source)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst["lat"].coordinates)
assert output.values[0] == source[0] and np.isnan(
output.values[1]) and output.values[2] == source[1]
def test_interpolate_alt(self):
""" for now alt uses nearest neighbor """
source = np.random.rand(5)
coords_src = Coordinates([clinspace(0, 10, 5)], dims=['alt'])
coords_dst = Coordinates([clinspace(1, 11, 5)], dims=['alt'])
node = MockArrayDataSource(source=source,
native_coordinates=coords_src)
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.alt.values == coords_dst.coords['alt'])
def test_interpolate_scipy_grid(self):
source = np.arange(0, 25)
source.resize((5, 5))
coords_src = Coordinates(
[clinspace(0, 10, 5), clinspace(0, 10, 5)],
dims=['lat', 'lon'])
coords_dst = Coordinates(
[clinspace(1, 11, 5), clinspace(1, 11, 5)],
dims=['lat', 'lon'])
# try one specific rasterio case to measure output
node = MockArrayDataSource(source=source,
native_coordinates=coords_src)
node.interpolation = {
'method': 'nearest',
'interpolators': [ScipyGrid]
}
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst.coords['lat'])
print(output)
assert output.data[0, 0] == 0.
assert output.data[0, 3] == 3.
assert output.data[1, 3] == 8.
assert np.isnan(
output.data[0, 4]) # TODO: how to handle outside bounds
node.interpolation = {
'method': 'cubic_spline',
'interpolators': [ScipyGrid]
}
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst.coords['lat'])
assert int(output.data[0, 0]) == 2
assert int(output.data[2, 4]) == 16
node.interpolation = {
'method': 'bilinear',
'interpolators': [ScipyGrid]
}
output = node.eval(coords_dst)
assert isinstance(output, UnitsDataArray)
assert np.all(output.lat.values == coords_dst.coords['lat'])
assert int(output.data[0, 0]) == 2
assert int(output.data[3, 3]) == 20
assert np.isnan(
output.data[4, 4]) # TODO: how to handle outside bounds
def test_select_coordinates(self):
reqcoords = Coordinates([[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]],
dims=['lat', 'lon', 'time', 'alt'])
srccoords = Coordinates([[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]],
dims=['lat', 'lon', 'time', 'alt'])
# create a few dummy interpolators that handle certain dimensions
# (can_select is defined by default to look at dims_supported)
class TimeLat(Interpolator):
dims_supported = ['time', 'lat']
def select_coordinates(self, udims, srccoords, srccoords_idx,
reqcoords):
return srccoords, srccoords_idx
class LatLon(Interpolator):
dims_supported = ['lat', 'lon']
def select_coordinates(self, udims, srccoords, srccoords_idx,
reqcoords):
return srccoords, srccoords_idx
class Lon(Interpolator):
dims_supported = ['lon']
def select_coordinates(self, udims, srccoords, srccoords_idx,
reqcoords):
return srccoords, srccoords_idx
# set up a strange interpolation definition
# we want to interpolate (lat, lon) first, then after (time, alt)
interp = Interpolation({
('lat', 'lon'): {
'method': 'myinterp',
'interpolators': [LatLon, TimeLat]
},
('time', 'alt'): {
'method': 'myinterp',
'interpolators': [TimeLat, Lon]
}
})
coords, cidx = interp.select_coordinates(srccoords, [], reqcoords)
assert len(coords) == len(srccoords)
assert len(coords['lat']) == len(srccoords['lat'])
assert cidx == ()
def _get_source_coordinates(self, requested_coordinates):
# get available time coordinates
# TODO do these two checks during node initialization
available_coordinates = self.coordinates_source.find_coordinates()
if len(available_coordinates) != 1:
raise ValueError(
"Cannot evaluate this node; too many available coordinates")
avail_coords = available_coordinates[0]
if 'time' not in avail_coords.udims:
raise ValueError(
"GroupReduce coordinates source node must be time-dependent")
# intersect grouped time coordinates using groupby DatetimeAccessor
avail_time = xr.DataArray(avail_coords.coords['time'])
eval_time = xr.DataArray(requested_coordinates.coords['time'])
N = getattr(avail_time.dt, self.groupby)
E = getattr(eval_time.dt, self.groupby)
native_time_mask = np.in1d(N, E)
# use requested spatial coordinates and filtered available times
coords = Coordinates(time=avail_time.data[native_time_mask],
lat=requested_coordinates['lat'],
lon=requested_coordinates['lon'],
order=('time', 'lat', 'lon'))
return coords
def test_parallel_process_zarr_async_starti(self):
# Can't use tempfile.TemporaryDirectory because multiple processess need access to dir
tmpdir = os.path.join(tempfile.gettempdir(),
"test_parallel_process_zarr_async_starti.zarr")
node = Process(source=CoordData(coord_name="time")) # , block=False)
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node_p = ParallelAsyncOutputZarr(source=node,
number_of_workers=5,
chunks={"time": 2},
fill_output=False,
zarr_file=tmpdir,
start_i=1)
o_zarr = node_p.eval(coords)
# print(o_zarr.info)
time.sleep(0.01)
np.testing.assert_array_equal([np.nan, np.nan, 3, 4, 5],
o_zarr["data"][:])
node_p = ParallelAsyncOutputZarr(source=node,
number_of_workers=5,
chunks={"time": 2},
fill_output=False,
zarr_file=tmpdir,
start_i=0)
o_zarr = node_p.eval(coords)
np.testing.assert_array_equal([1, 2, 3, 4, 5], o_zarr["data"][:])
shutil.rmtree(tmpdir)
def test_eval_multiple(self):
coords = Coordinates([0, 10], dims=["lat", "lon"])
z = Zarr(source=self.path, data_key=["a", "b"])
out = z.eval(coords)
assert out.dims == ("lat", "lon", "output")
np.testing.assert_array_equal(out["output"], ["a", "b"])
assert out.sel(output="a")[0, 0] == 0.0
assert out.sel(output="b")[0, 0] == 1.0
# single output key
z = Zarr(source=self.path, data_key=["a"])
out = z.eval(coords)
assert out.dims == ("lat", "lon", "output")
np.testing.assert_array_equal(out["output"], ["a"])
assert out.sel(output="a")[0, 0] == 0.0
# alternate output names
z = Zarr(source=self.path, data_key=["a", "b"], outputs=["A", "B"])
out = z.eval(coords)
assert out.dims == ("lat", "lon", "output")
np.testing.assert_array_equal(out["output"], ["A", "B"])
assert out.sel(output="A")[0, 0] == 0.0
assert out.sel(output="B")[0, 0] == 1.0
# default
z = Zarr(source=self.path)
out = z.eval(coords)
assert out.dims == ("lat", "lon", "output")
np.testing.assert_array_equal(out["output"], ["a", "b"])
assert out.sel(output="a")[0, 0] == 0.0
assert out.sel(output="b")[0, 0] == 1.0
def make_coordinate_combinations(lat=None, lon=None, alt=None, time=None):
''' Generates every combination of stacked and unstacked coordinates podpac expects to handle
Parameters
-----------
lat: podpac.core.coordinates.Coordinates1d, optional
1D coordinate object used to create the Coordinate objects that contain the latitude dimension. By default uses:
UniformCoord(start=0, stop=2, size=3)
lon: podpac.core.coordinates.Coordinates1d, optional
Same as above but for longitude. By default uses:
UniformCoord(start=2, stop=6, size=3)
alt: podpac.core.coordinates.Coordinates1d, optional
Same as above but for longitude. By default uses:
UniformCoord(start=6, stop=12, size=3)
time: podpac.core.coordinates.Coordinates1d, optional
Same as above but for longitude. By default uses:
UniformCoord(start='2018-01-01T00:00:00', stop='2018-03-01T00:00:00', size=3)
Returns
-------
OrderedDict:
Dictionary of all the podpac.Core.Coordinate objects podpac expects to handle. The dictionary keys is a tuple of
coordinate dimensions, and the values are the actual Coordinate objects.
Notes
------
When custom lat, lon, alt, and time 1D coordinates are given, only those with the same number of coordinates are
stacked together. For example, if lat, lon, alt, and time have sizes 3, 4, 5, and 6, respectively, no stacked
coordinates are created. Also, no exception or warning is thrown for this case.
'''
# make the 1D coordinates
if lat is None:
lat = ArrayCoordinates1d([0, 1, 2], name='lat')
if lon is None:
lon = ArrayCoordinates1d([2, 4, 6], name='lon')
if alt is None:
alt = ArrayCoordinates1d([6, 9, 12], name='alt')
if time is None:
time = ArrayCoordinates1d(['2018-01-01', '2018-02-01', '2018-03-01'],
name='time')
d = dict([('lat', lat), ('lon', lon), ('alt', alt), ('tim', time)])
dims_list = get_dims_list()
# make the stacked coordinates
for dim in [dim for dims in dims_list for dim in dims if '_' in dim]:
cs = [d[k] for k in dim.split('_')]
if any(c.size != cs[0].size for c in cs):
continue # can't stack these
d[dim] = StackedCoordinates(cs)
# make the ND coordinates
coord_collection = OrderedDict()
for dims in dims_list:
if any(dim not in d for dim in dims):
continue
coord_collection[dims] = Coordinates([d[dim] for dim in dims])
return coord_collection
def _remove_nans(self, source_data, source_coordinates):
index = np.array(np.isnan(source_data), bool)
if not np.any(index):
return source_data, source_coordinates
data = source_data.data[~index]
coords = np.meshgrid(*[
source_coordinates[d.split("_")[0]].coordinates
for d in source_coordinates.dims
],
indexing="ij")
repeat_shape = coords[0].shape
coords = [c[~index] for c in coords]
final_dims = [d.split("_")[0] for d in source_coordinates.dims]
# Add back in any stacked coordinates
for i, d in enumerate(source_coordinates.dims):
dims = d.split("_")
if len(dims) == 1:
continue
reshape = np.ones(len(coords), int)
reshape[i] = -1
repeats = list(repeat_shape)
repeats[i] = 1
for dd in dims[1:]:
crds = source_coordinates[dd].coordinates.reshape(*reshape)
for j, r in enumerate(repeats):
crds = crds.repeat(r, axis=j)
coords.append(crds[~index])
final_dims.append(dd)
return data, Coordinates([coords], dims=[final_dims])