def test_coordinates(self):
node = Array(source=self.data, coordinates=self.coordinates)
assert node.coordinates
node = Array(source=self.data)
with pytest.raises(tl.TraitError):
node.coordinates
def test_composite_multithreaded(self):
with podpac.settings:
podpac.settings["MULTITHREADING"] = True
podpac.settings["N_THREADS"] = 8
acoords = podpac.Coordinates([[-1, 0, 1], [10, 20, 30]],
dims=["lat", "lon"])
asource = np.ones(acoords.shape)
asource[0, :] = np.nan
a = Array(source=asource, coordinates=acoords)
bcoords = podpac.Coordinates([[0, 1, 2, 3], [10, 20, 30, 40]],
dims=["lat", "lon"])
bsource = np.zeros(bcoords.shape)
bsource[:, 0] = np.nan
b = Array(source=bsource, coordinates=bcoords)
coords = podpac.Coordinates([[0, 1, 2], [10, 20, 30, 40, 50]],
dims=["lat", "lon"])
node = OrderedCompositor(sources=[a, b], interpolation="bilinear")
expected = np.array([[1.0, 1.0, 1.0, 0.0, np.nan],
[1.0, 1.0, 1.0, 0.0, np.nan],
[np.nan, np.nan, 0.0, 0.0, np.nan]])
np.testing.assert_allclose(node.eval(coords),
expected,
equal_nan=True)
node = OrderedCompositor(sources=[b, a], interpolation="bilinear")
expected = np.array([[1.0, 1.0, 0.0, 0.0, np.nan],
[1.0, 1.0, 0.0, 0.0, np.nan],
[np.nan, np.nan, 0.0, 0.0, np.nan]])
np.testing.assert_allclose(node.eval(coords),
expected,
equal_nan=True)
def test_get_data(self):
""" defined get_data function"""
node = Array(source=self.data, coordinates=self.coordinates)
output = node.eval(self.coordinates)
assert isinstance(output, UnitsDataArray)
assert output.values[0, 0] == self.data[0, 0]
assert output.values[4, 5] == self.data[4, 5]
def test_source_interpolation(self):
"""test get data from reprojected source"""
# no source_interpolation
source = Array(source=self.data,
coordinates=self.coordinates,
interpolation="nearest")
with pytest.warns(DeprecationWarning):
node = ReprojectedSource(
source=source,
reprojected_coordinates=self.reprojected_coordinates)
assert source.interpolation == "nearest"
assert node.source.interpolation == "nearest"
assert node.eval_source.interpolation == "nearest"
assert node.eval_source.coordinates == source.coordinates
np.testing.assert_array_equal(node.eval_source.source, source.source)
# matching source_interpolation
source = Array(source=self.data,
coordinates=self.coordinates,
interpolation="nearest")
with pytest.warns(DeprecationWarning):
node = ReprojectedSource(
source=source,
reprojected_coordinates=self.reprojected_coordinates,
source_interpolation="nearest")
assert source.interpolation == "nearest"
assert node.source.interpolation == "nearest"
assert node.eval_source.interpolation == "nearest"
assert node.eval_source.coordinates == source.coordinates
np.testing.assert_array_equal(node.eval_source.source, source.source)
# non-matching source_interpolation
source = Array(source=self.data,
coordinates=self.coordinates,
interpolation="nearest")
with pytest.warns(DeprecationWarning):
node = ReprojectedSource(
source=source,
reprojected_coordinates=self.reprojected_coordinates,
source_interpolation="bilinear")
assert source.interpolation == "nearest"
assert node.source.interpolation == "nearest"
assert node.eval_source.interpolation == "bilinear"
assert node.eval_source.coordinates == source.coordinates
np.testing.assert_array_equal(node.eval_source.source, source.source)
# no source.interpolation to set (trigger logger warning)
source = Node()
with pytest.warns(DeprecationWarning):
node = ReprojectedSource(
source=source,
reprojected_coordinates=self.reprojected_coordinates,
source_interpolation="bilinear")
def test_composite_into_result(self):
coords = podpac.Coordinates([[0, 1], [10, 20, 30]],
dims=["lat", "lon"])
a = Array(source=np.ones(coords.shape), coordinates=coords)
b = Array(source=np.zeros(coords.shape), coordinates=coords)
node = OrderedCompositor(sources=[a, b], interpolation="bilinear")
result = node.create_output_array(coords,
data=np.random.random(coords.shape))
output = node.eval(coords, output=result)
np.testing.assert_array_equal(output, a.source)
np.testing.assert_array_equal(result, a.source)
def test_ignored_interpolation_params_issue340(self, caplog):
node = Array(
source=[0, 1, 2],
coordinates=Coordinates([[0, 2, 1]], dims=["time"]),
interpolation={"method": "nearest", "params": {"fake_param": 1.1, "spatial_tolerance": 1}},
)
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
node.eval(Coordinates([[0.5, 1.5]], ["time"]))
assert "interpolation parameter 'fake_param' was ignored" in caplog.text
assert "interpolation parameter 'spatial_tolerance' was ignored" not in caplog.text
def test_composite_short_circuit(self):
with podpac.settings:
podpac.settings["MULTITHREADING"] = False
podpac.settings["DEBUG"] = True
coords = podpac.Coordinates([[0, 1], [10, 20, 30]],
dims=["lat", "lon"])
a = Array(source=np.ones(coords.shape), coordinates=coords)
b = Array(source=np.zeros(coords.shape), coordinates=coords)
node = OrderedCompositor(sources=[a, b], interpolation="bilinear")
output = node.eval(coords)
np.testing.assert_array_equal(output, a.source)
assert node._eval_sources[0]._output is not None
assert node._eval_sources[1]._output is None
def test_multiple_outputs(self):
class MyAlgorithm(Algorithm):
x = NodeTrait().tag(attr=True)
y = NodeTrait().tag(attr=True)
outputs = ["sum", "prod", "diff"]
def algorithm(self, inputs):
sum_ = inputs["x"] + inputs["y"]
prod = inputs["x"] * inputs["y"]
diff = inputs["x"] - inputs["y"]
return np.stack([sum_, prod, diff], -1)
coords = podpac.Coordinates([[0, 1, 2], [10, 20]], dims=["lat", "lon"])
x = Arange()
y = Array(source=np.full(coords.shape, 2), coordinates=coords)
xout = np.arange(6).reshape(3, 2)
# all outputs
node = MyAlgorithm(x=x, y=y)
result = node.eval(coords)
assert result.dims == ("lat", "lon", "output")
np.testing.assert_array_equal(result["output"],
["sum", "prod", "diff"])
np.testing.assert_array_equal(result.sel(output="sum"), xout + 2)
np.testing.assert_array_equal(result.sel(output="prod"), xout * 2)
np.testing.assert_array_equal(result.sel(output="diff"), xout - 2)
# extract an output
node = MyAlgorithm(x=x, y=y, output="prod")
result = node.eval(coords)
assert result.dims == ("lat", "lon")
np.testing.assert_array_equal(result, xout * 2)
def test_get_data(self):
"""test get data from reprojected source"""
source = Array(source=self.data, coordinates=self.coordinates)
with pytest.warns(DeprecationWarning):
node = ReprojectedSource(
source=source, reprojected_coordinates=source.coordinates)
output = node.eval(node.coordinates)
def test_composite_extra_dims(self):
with podpac.settings:
podpac.settings["MULTITHREADING"] = False
coords = podpac.Coordinates([[0, 1], [10, 20, 30]],
dims=["lat", "lon"])
a = Array(source=np.ones(coords.shape), coordinates=coords)
extra = podpac.Coordinates(
[coords["lat"], coords["lon"], "2020-01-01"],
dims=["lat", "lon", "time"])
# dims not provided, eval fails with extra dims
node = OrderedCompositor(sources=[a], interpolation="bilinear")
np.testing.assert_array_equal(node.eval(coords), a.source)
with pytest.raises(
podpac.NodeException,
match="Cannot evaluate compositor with requested dims"):
node.eval(extra)
# dims provided, remove extra dims
node = OrderedCompositor(sources=[a],
dims=["lat", "lon"],
interpolation="bilinear")
np.testing.assert_array_equal(node.eval(coords), a.source)
np.testing.assert_array_equal(node.eval(extra), a.source)
def test_composite_short_circuit_multithreaded(self):
with podpac.settings:
podpac.settings["MULTITHREADING"] = True
podpac.settings["N_THREADS"] = 8
podpac.settings["DEBUG"] = True
coords = podpac.Coordinates([[0, 1], [10, 20, 30]],
dims=["lat", "lon"])
n_threads_before = podpac.core.managers.multi_threading.thread_manager._n_threads_used
a = Array(source=np.ones(coords.shape), coordinates=coords)
b = Array(source=np.zeros(coords.shape), coordinates=coords)
node = OrderedCompositor(sources=[a, b], interpolation="bilinear")
output = node.eval(coords)
np.testing.assert_array_equal(output, a.source)
assert node._multi_threaded == True
assert podpac.core.managers.multi_threading.thread_manager._n_threads_used == n_threads_before
def test_year_substitution_multiple_outputs(self):
multi = Array(source=np.random.random(COORDS.shape + (2, )),
coordinates=COORDS,
outputs=["a", "b"])
node = YearSubstituteCoordinates(source=multi, year="2018")
o = node.eval(COORDS)
assert o.time.dt.year.data[0] == 2018
assert not np.array_equal(o["time"], COORDS["time"].coordinates)
def test_find_coordinates(self):
class MyAlgorithm(BaseAlgorithm):
x = NodeTrait().tag(attr=True)
y = NodeTrait().tag(attr=True)
node = MyAlgorithm(
x=Array(coordinates=podpac.Coordinates([[0, 1, 2], [10, 20]],
dims=["lat", "lon"])),
y=Array(coordinates=podpac.Coordinates([[0, 1, 2], [110, 120]],
dims=["lat", "lon"])),
)
l = node.find_coordinates()
assert isinstance(l, list)
assert len(l) == 2
assert node.x.coordinates in l
assert node.y.coordinates in l
def test_interpolation_mixin(self):
class InterpArray(InterpolationMixin, ArrayBase):
pass
data = np.random.rand(4, 5)
native_coords = Coordinates([np.linspace(0, 3, 4), np.linspace(0, 4, 5)], ["lat", "lon"])
coords = Coordinates([np.linspace(0, 3, 7), np.linspace(0, 4, 9)], ["lat", "lon"])
iarr_src = InterpArray(source=data, coordinates=native_coords, interpolation="bilinear")
arr_src = Array(source=data, coordinates=native_coords, interpolation="bilinear")
arrb_src = ArrayBase(source=data, coordinates=native_coords)
iaso = iarr_src.eval(coords)
aso = arr_src.eval(coords)
abso = arrb_src.eval(coords)
np.testing.assert_array_equal(iaso.data, aso.data)
np.testing.assert_array_equal(abso.data, data)
def test_stacked_coords_with_partial_dims_issue123(self):
node = Array(
source=[0, 1, 2],
coordinates=Coordinates(
[[[0, 2, 1], [10, 12, 11], ["2018-01-01", "2018-01-02", "2018-01-03"]]], dims=["lat_lon_time"]
),
interpolation="nearest",
)
# unstacked or and stacked requests without time
o1 = node.eval(Coordinates([[0.5, 1.5], [10.5, 11.5]], dims=["lat", "lon"]))
o2 = node.eval(Coordinates([[[0.5, 1.5], [10.5, 11.5]]], dims=["lat_lon"]))
assert_array_equal(o1.data, [[0, 2], [2, 1]])
assert_array_equal(o2.data, [0, 1])
# request without lat or lon
o3 = node.eval(Coordinates(["2018-01-01"], dims=["time"]))
assert o3.data[0] == 0
class TestUnaryAlgorithm(object):
source = Array(coordinates=podpac.Coordinates([[0, 1, 2], [10, 20]],
dims=["lat", "lon"]))
def test_outputs(self):
node = UnaryAlgorithm(source=self.source)
assert node.outputs == None
node = UnaryAlgorithm(source=Array(outputs=["a", "b"]))
assert node.outputs == ["a", "b"]
def test_composite_stacked_unstacked(self):
anative = podpac.Coordinates(
[podpac.clinspace((0, 1), (1, 2), size=3)], dims=["lat_lon"])
bnative = podpac.Coordinates(
[podpac.clinspace(-2, 3, 3),
podpac.clinspace(-1, 4, 3)],
dims=["lat", "lon"])
a = Array(source=np.random.rand(3), coordinates=anative)
b = Array(source=np.random.rand(3, 3) + 2, coordinates=bnative)
coords = podpac.Coordinates(
[podpac.clinspace(-3, 4, 32),
podpac.clinspace(-2, 5, 32)],
dims=["lat", "lon"])
node = OrderedCompositor(sources=[a, b], interpolation="nearest")
o = node.eval(coords)
# Check that both data sources are being used in the interpolation
assert np.any(o.data >= 2)
assert np.any(o.data <= 1)
def setup_module():
global coords, source, data, multisource, bdata
coords = podpac.Coordinates(
[
podpac.clinspace(0, 1, 10),
podpac.clinspace(0, 1, 10),
podpac.crange("2018-01-01", "2018-01-10", "1,D")
],
dims=["lat", "lon", "time"],
)
a = np.random.random(coords.shape)
a[3, 0, 0] = np.nan
a[0, 3, 0] = np.nan
a[0, 0, 3] = np.nan
source = Array(source=a, coordinates=coords)
data = source.eval(coords)
ab = np.stack([a, 2 * a], -1)
multisource = Array(source=ab, coordinates=coords, outputs=["a", "b"])
bdata = 2 * data
def test_year_substitution_missing_coords(self):
source = Array(
source=[[1, 2, 3], [4, 5, 6]],
coordinates=podpac.Coordinates([
podpac.crange("2018-01-01", "2018-01-02", "1,D"),
podpac.clinspace(45, 66, 3)
],
dims=["time", "lat"]),
)
node = YearSubstituteCoordinates(source=source, year="2018")
o = node.eval(COORDS)
assert o.time.dt.year.data[0] == 2018
assert o["time"].data != xr.DataArray(COORDS["time"].coordinates).data
def test_year_substitution_missing_coords_orig_coords(self):
source = Array(
source=[[1, 2, 3], [4, 5, 6]],
coordinates=podpac.Coordinates([
podpac.crange("2018-01-01", "2018-01-02", "1,D"),
podpac.clinspace(45, 66, 3)
],
dims=["time", "lat"]),
)
node = YearSubstituteCoordinates(source=source,
year="2018",
substitute_eval_coords=True)
o = node.eval(COORDS)
assert o.time.dt.year.data[0] == 2017
np.testing.assert_array_equal(o["time"], COORDS["time"].coordinates)
def test_eval_extra_dims(self):
coords = COORDS.drop("time")
a = Array(source=np.ones(coords.shape), coordinates=coords)
b = Array(source=np.zeros(coords.shape), coordinates=coords)
# no dims provided, evaluation fails with extra requested dims
node = MockComposite(sources=[a, b])
np.testing.assert_array_equal(node.eval(coords), a.source)
with pytest.raises(
podpac.NodeException,
match="Cannot evaluate compositor with requested dims"):
node.eval(COORDS)
# dims provided, evaluation should succeed with extra requested dims
node = MockComposite(sources=[a, b], dims=["lat", "lon"])
np.testing.assert_array_equal(node.eval(coords), a.source)
np.testing.assert_array_equal(node.eval(COORDS), a.source)
# drop stacked dimensions if none of its dimensions are needed
c = podpac.Coordinates(
[COORDS["lat"], COORDS["lon"], [COORDS["time"], [10, 20]]],
dims=["lat", "lon", "time_alt"])
np.testing.assert_array_equal(node.eval(c), a.source)
# TODO
# but don't drop stacked dimensions if any of its dimensions are needed
# c = podpac.Coordinates([[COORDS['lat'], COORDS['lon'], np.arange(COORDS['lat'].size)]], dims=['lat_lon_alt'])
# np.testing.assert_array_equal(node.eval(c), np.ones(COORDS['lat'].size))
# dims can also be specified by the node
class MockComposite2(MockComposite):
dims = ["lat", "lon"]
node = MockComposite2(sources=[a, b])
np.testing.assert_array_equal(node.eval(coords), a.source)
np.testing.assert_array_equal(node.eval(COORDS), a.source)
def test_coordinates(self):
"""test coordinates"""
# source has no coordinates, just use reprojected_coordinates
node = ReprojectedSource(
source=Node(),
reprojected_coordinates=self.reprojected_coordinates)
assert node.coordinates == self.reprojected_coordinates
# source has coordinates
source = Array(coordinates=self.coordinates)
node = ReprojectedSource(
source=source,
reprojected_coordinates=self.reprojected_coordinates)
assert node.coordinates == self.reprojected_coordinates
def test_coordinates(self):
"""test coordinates"""
# source has no coordinates, just use reprojected_coordinates
with pytest.warns(DeprecationWarning):
node = ReprojectedSource(
source=Node(),
reprojected_coordinates=self.reprojected_coordinates)
assert node.coordinates == self.reprojected_coordinates
# source has coordinates
source = Array(coordinates=self.coordinates)
with pytest.warns(DeprecationWarning):
node = ReprojectedSource(
source=source,
reprojected_coordinates=self.reprojected_coordinates)
assert node.coordinates == self.reprojected_coordinates
def test_get_data_multiple(self):
data = np.random.rand(11, 11, 2)
node = Array(source=data,
coordinates=self.coordinates,
outputs=["a", "b"])
output = node.eval(self.coordinates)
assert isinstance(output, UnitsDataArray)
assert output.dims == ("lat", "lon", "output")
np.testing.assert_array_equal(output["output"], ["a", "b"])
np.testing.assert_array_equal(output.sel(output="a"), data[:, :, 0])
np.testing.assert_array_equal(output.sel(output="b"), data[:, :, 1])
node = Array(source=data,
coordinates=self.coordinates,
outputs=["a", "b"],
output="b")
output = node.eval(self.coordinates)
assert isinstance(output, UnitsDataArray)
assert output.dims == ("lat", "lon")
np.testing.assert_array_equal(output, data[:, :, 1])
from podpac.core.compositor.compositor import BaseCompositor
COORDS = podpac.Coordinates(
[
podpac.clinspace(45, 0, 16),
podpac.clinspace(-70, -65, 16),
podpac.clinspace(0, 1, 2)
],
dims=["lat", "lon", "time"],
)
LON, LAT, TIME = np.meshgrid(COORDS["lon"].coordinates,
COORDS["lat"].coordinates,
COORDS["time"].coordinates)
ARRAY_LAT = Array(source=LAT.astype(float),
coordinates=COORDS,
interpolation="bilinear")
ARRAY_LON = Array(source=LON.astype(float),
coordinates=COORDS,
interpolation="bilinear")
ARRAY_TIME = Array(source=TIME.astype(float),
coordinates=COORDS,
interpolation="bilinear")
MULTI_0_XY = Array(source=np.full(COORDS.shape + (2, ), 0),
coordinates=COORDS,
outputs=["x", "y"])
MULTI_1_XY = Array(source=np.full(COORDS.shape + (2, ), 1),
coordinates=COORDS,
outputs=["x", "y"])
MULTI_4_YX = Array(source=np.full(COORDS.shape + (2, ), 4),
def test_invalid_data(self):
with pytest.raises(ValueError,
match="Array 'source' data must be numerical"):
node = Array(source=["a", "b"], coordinates=self.coordinates)
def test_outputs(self):
node = UnaryAlgorithm(source=self.source)
assert node.outputs == None
node = UnaryAlgorithm(source=Array(outputs=["a", "b"]))
assert node.outputs == ["a", "b"]
def test_no_cache(self):
node = Array()
assert len(node.cache_ctrl._cache_stores) == 0
import podpac
from podpac.core.data.array_source import Array
from podpac.core.compositor.ordered_compositor import OrderedCompositor
COORDS = podpac.Coordinates(
[
podpac.clinspace(45, 0, 16),
podpac.clinspace(-70, -65, 16),
podpac.clinspace(0, 1, 2)
],
dims=["lat", "lon", "time"],
)
MULTI_0_XY = Array(source=np.full(COORDS.shape + (2, ), 0),
coordinates=COORDS,
outputs=["x", "y"])
MULTI_1_XY = Array(source=np.full(COORDS.shape + (2, ), 1),
coordinates=COORDS,
outputs=["x", "y"])
MULTI_4_YX = Array(source=np.full(COORDS.shape + (2, ), 4),
coordinates=COORDS,
outputs=["y", "x"])
MULTI_2_X = Array(source=np.full(COORDS.shape + (1, ), 2),
coordinates=COORDS,
outputs=["x"])
MULTI_3_Z = Array(source=np.full(COORDS.shape + (1, ), 3),
coordinates=COORDS,
outputs=["z"])
def test_get_data(self):
"""test get data from reprojected source"""
source = Array(source=self.data, coordinates=self.coordinates)
node = ReprojectedSource(source=source,
reprojected_coordinates=source.coordinates)
output = node.eval(node.coordinates)
