def test_f(self):
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node = Arange()
q = Queue()
_f(node.json, coords.json, q, {})
o = q.get()
np.testing.assert_array_equal(o, node.eval(coords))
def test_mask_defaults_bool_op(self):
coords = podpac.Coordinates(
[podpac.clinspace(0, 1, 4),
podpac.clinspace(0, 1, 3)],
dims=["lat", "lon"])
sine_node = Arange()
a = sine_node.eval(coords).copy()
# Less than
node = Mask(source=sine_node, mask=sine_node, bool_op="<")
output = node.eval(coords)
b = a.copy()
b.data[a.data < 1] = np.nan
np.testing.assert_allclose(output, b)
# Less than equal
node = Mask(source=sine_node, mask=sine_node, bool_op="<=")
output = node.eval(coords)
b = a.copy()
b.data[a.data <= 1] = np.nan
np.testing.assert_allclose(output, b)
# Greater than
node = Mask(source=sine_node, mask=sine_node, bool_op=">")
output = node.eval(coords)
b = a.copy()
b.data[a.data > 1] = np.nan
np.testing.assert_allclose(output, b)
# Greater than equal
node = Mask(source=sine_node, mask=sine_node, bool_op=">=")
output = node.eval(coords)
b = a.copy()
b.data[a.data >= 1] = np.nan
np.testing.assert_allclose(output, b)
def test_f_fmt(self):
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node = Arange()
q = Queue()
_f(node.json, coords.json, q, {"format": "dict", "format_kwargs": {}})
o = q.get()
np.testing.assert_array_equal(o["data"],
node.eval(coords).to_dict()["data"])
def test_mp_results_the_same(self):
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node = Arange()
o_sp = node.eval(coords)
node_mp = Process(source=node)
o_mp = node_mp.eval(coords)
np.testing.assert_array_equal(o_sp.data, o_mp.data)
def test_mp_results_the_same_set_output(self):
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node = Arange()
o_sp = node.eval(coords)
output = o_sp.copy()
output[:] = np.nan
node_mp = Process(source=node)
o_mp = node_mp.eval(coords, output=output)
np.testing.assert_array_equal(o_sp, output)
def test_masked_val(self):
coords = podpac.Coordinates(
[podpac.clinspace(0, 1, 4),
podpac.clinspace(0, 1, 3)],
dims=["lat", "lon"])
sine_node = Arange()
a = sine_node.eval(coords).copy()
a.data[a.data == 1] = -9999
node = Mask(source=sine_node, mask=sine_node, masked_val=-9999)
output = node.eval(coords)
np.testing.assert_allclose(output, a)
def test_mask_defaults(self):
coords = podpac.Coordinates(
[podpac.crange(-90, 90, 1.0),
podpac.crange(-180, 180, 1.0)],
dims=["lat", "lon"])
sine_node = Arange()
a = sine_node.eval(coords).copy()
a.data[a.data == 1] = np.nan
node = Mask(source=sine_node, mask=sine_node)
output = node.eval(coords)
np.testing.assert_allclose(output, a)
def test_multiple_outputs(self):
class MyAlgorithm(Algorithm):
x = NodeTrait().tag(attr=True)
y = NodeTrait().tag(attr=True)
outputs = ["sum", "prod", "diff"]
def algorithm(self, inputs):
sum_ = inputs["x"] + inputs["y"]
prod = inputs["x"] * inputs["y"]
diff = inputs["x"] - inputs["y"]
return np.stack([sum_, prod, diff], -1)
coords = podpac.Coordinates([[0, 1, 2], [10, 20]], dims=["lat", "lon"])
x = Arange()
y = Array(source=np.full(coords.shape, 2), coordinates=coords)
xout = np.arange(6).reshape(3, 2)
# all outputs
node = MyAlgorithm(x=x, y=y)
result = node.eval(coords)
assert result.dims == ("lat", "lon", "output")
np.testing.assert_array_equal(result["output"],
["sum", "prod", "diff"])
np.testing.assert_array_equal(result.sel(output="sum"), xout + 2)
np.testing.assert_array_equal(result.sel(output="prod"), xout * 2)
np.testing.assert_array_equal(result.sel(output="diff"), xout - 2)
# extract an output
node = MyAlgorithm(x=x, y=y, output="prod")
result = node.eval(coords)
assert result.dims == ("lat", "lon")
np.testing.assert_array_equal(result, xout * 2)
def test_doy_window2_mean_rescale_max_min(self):
with podpac.settings:
podpac.settings.set_unsafe_eval(True)
coords = podpac.coordinates.concat([
podpac.Coordinates(
[podpac.crange("1999-12-29", "2000-01-03", "1,D",
"time")]),
podpac.Coordinates(
[podpac.crange("2001-12-30", "2002-01-02", "1,D",
"time")]),
])
node = Arange()
node_max = Arithmetic(source=node, eqn="(source < 5) + source")
node_min = Arithmetic(source=node, eqn="-1*(source < 5) + source")
nodedoywindow_s = FM(
source=node,
window=2,
cache_output=False,
force_eval=True,
scale_max=node_max,
scale_min=node_min,
rescale=False,
)
o_s = nodedoywindow_s.eval(coords)
np.testing.assert_array_almost_equal([0.5] * o_s.size, o_s)
def test_year_substitution_orig_coords(self):
node = YearSubstituteCoordinates(source=Arange(),
year="2018",
substitute_eval_coords=True)
o = node.eval(COORDS)
assert o.time.dt.year.data[0] == xr.DataArray(
COORDS["time"].coordinates).dt.year.data[0]
np.testing.assert_array_equal(o["time"], COORDS["time"].coordinates)
def test_outputs(self):
node = Combine(a=Arange(), b=Arange(), c=Arange())
assert set(node.outputs) == set(["a", "b", "c"])
node = Combine(a=Arange(),
b=Arange(),
c=Arange(),
outputs=["o1", "o2", "o3"])
assert set(node.outputs) == set(["o1", "o2", "o3"])
def test_multi_threading(self):
coords = podpac.Coordinates([[1, 2, 3]], ["lat"])
with podpac.settings:
podpac.settings.set_unsafe_eval(True)
podpac.settings["CACHE_NODE_OUTPUT_DEFAULT"] = False
podpac.settings["DEFAULT_CACHE"] = []
podpac.settings["RAM_CACHE_ENABLED"] = False
node1 = Arithmetic(A=Arange(), B=Arange(), eqn="A+B")
node2 = Arithmetic(A=node1, B=Arange(), eqn="A+B")
# multithreaded
podpac.settings["MULTITHREADING"] = True
podpac.settings["N_THREADS"] = 8
omt = node2.eval(coords)
# single threaded
podpac.settings["MULTITHREADING"] = False
ost = node2.eval(coords)
np.testing.assert_array_equal(omt, ost)
def test_doy_window2(self):
coords = podpac.coordinates.concat([
podpac.Coordinates(
[podpac.crange("1999-12-29", "2000-01-03", "1,D", "time")]),
podpac.Coordinates(
[podpac.crange("2001-12-30", "2002-01-02", "1,D", "time")]),
])
node = Arange()
nodedoywindow = F(source=node,
window=2,
cache_output=False,
force_eval=True)
o = nodedoywindow.eval(coords)
np.testing.assert_array_equal(o, [6, 5, 3, 3, 5, 6])
def test_init(self):
a = SinCoords()
b = Arange()
with podpac.settings:
podpac.settings.set_unsafe_eval(True)
node = Generic(code="import numpy as np\noutput = np.minimum(a,b)",
a=a,
b=b)
podpac.settings.set_unsafe_eval(False)
with pytest.warns(UserWarning, match="Insecure evaluation"):
node = Generic(
code="import numpy as np\noutput = np.minimum(a,b)",
a=a,
b=b)
def test_eval(self):
coords = podpac.Coordinates([[0, 1, 2], [10, 20]], dims=["lat", "lon"])
node = Combine(a=Arange(), b=Arange(), c=Arange())
output = node.eval(coords)
assert output.dims == ("lat", "lon", "output")
assert set(output["output"].data) == set(["a", "b", "c"])
np.testing.assert_array_equal(output.sel(output="a"),
Arange().eval(coords))
np.testing.assert_array_equal(output.sel(output="b"),
Arange().eval(coords))
np.testing.assert_array_equal(output.sel(output="c"),
Arange().eval(coords))
def test_evaluate(self):
with podpac.settings:
podpac.settings.set_unsafe_eval(True)
coords = podpac.Coordinates(
[podpac.crange(-90, 90, 1.0),
podpac.crange(-180, 180, 1.0)],
dims=["lat", "lon"])
a = SinCoords()
b = Arange()
node = Generic(code="import numpy as np\noutput = np.minimum(a,b)",
a=a,
b=b)
output = node.eval(coords)
a = node.eval(coords)
b = node.eval(coords)
np.testing.assert_allclose(output, np.minimum(a, b))
def test_evaluate_not_allowed(self):
with podpac.settings:
podpac.settings.set_unsafe_eval(False)
coords = podpac.Coordinates(
[podpac.crange(-90, 90, 1.0),
podpac.crange(-180, 180, 1.0)],
dims=["lat", "lon"])
a = SinCoords()
b = Arange()
with pytest.warns(UserWarning, match="Insecure evaluation"):
node = Generic(
code="import numpy as np\noutput = np.minimum(a,b)",
a=a,
b=b)
with pytest.raises(PermissionError):
node.eval(coords)
def test_multi_threading_stress_nthreads(self):
coords = podpac.Coordinates([np.linspace(0, 1, 4)], ["lat"])
A = Arithmetic(A=Arange(), eqn="A**0")
B = Arithmetic(A=Arange(), eqn="A**1")
C = Arithmetic(A=Arange(), eqn="A**2")
D = Arithmetic(A=Arange(), eqn="A**3")
E = Arithmetic(A=Arange(), eqn="A**4")
F = Arithmetic(A=Arange(), eqn="A**5")
node2 = Arithmetic(A=A, B=B, C=C, D=D, E=E, F=F, eqn="A+B+C+D+E+F")
node3 = Arithmetic(A=A,
B=B,
C=C,
D=D,
E=E,
F=F,
G=node2,
eqn="A+B+C+D+E+F+G")
with podpac.settings:
podpac.settings["MULTITHREADING"] = True
podpac.settings["N_THREADS"] = 8
podpac.settings["CACHE_NODE_OUTPUT_DEFAULT"] = False
podpac.settings["DEFAULT_CACHE"] = []
podpac.settings["RAM_CACHE_ENABLED"] = False
podpac.settings.set_unsafe_eval(True)
omt = node3.eval(coords)
assert node3._multi_threaded
assert not node2._multi_threaded
with podpac.settings:
podpac.settings["MULTITHREADING"] = True
podpac.settings[
"N_THREADS"] = 9 # 2 threads available after first 7
podpac.settings["CACHE_NODE_OUTPUT_DEFAULT"] = False
podpac.settings["DEFAULT_CACHE"] = []
podpac.settings["RAM_CACHE_ENABLED"] = False
podpac.settings.set_unsafe_eval(True)
omt = node3.eval(coords)
assert node3._multi_threaded
assert node2._multi_threaded
def test_doy_window2_mean_rescale_float(self):
coords = podpac.coordinates.concat([
podpac.Coordinates(
[podpac.crange("1999-12-29", "2000-01-03", "1,D", "time")]),
podpac.Coordinates(
[podpac.crange("2001-12-30", "2002-01-02", "1,D", "time")]),
])
node = Arange()
nodedoywindow = FM(source=node,
window=2,
cache_output=False,
force_eval=True)
o = nodedoywindow.eval(coords)
nodedoywindow_s = FM(source=node,
window=2,
cache_output=False,
force_eval=True,
scale_float=[0, coords.size],
rescale=True)
o_s = nodedoywindow_s.eval(coords)
np.testing.assert_array_almost_equal(o, o_s)
def test_multi_threading_cache_race(self):
coords = podpac.Coordinates([np.linspace(0, 1, 1024)], ["lat"])
with podpac.settings:
podpac.settings["MULTITHREADING"] = True
podpac.settings["N_THREADS"] = 3
podpac.settings["CACHE_NODE_OUTPUT_DEFAULT"] = True
podpac.settings["DEFAULT_CACHE"] = ["ram"]
podpac.settings["RAM_CACHE_ENABLED"] = True
podpac.settings.set_unsafe_eval(True)
A = Arithmetic(A=Arange(), eqn="A**2")
B = Arithmetic(A=Arange(), eqn="A**2")
C = Arithmetic(A=Arange(), eqn="A**2")
D = Arithmetic(A=Arange(), eqn="A**2")
E = Arithmetic(A=Arange(), eqn="A**2")
F = Arithmetic(A=Arange(), eqn="A**2")
node2 = Arithmetic(A=A, B=B, C=C, D=D, E=E, F=F, eqn="A+B+C+D+E+F")
om = node2.eval(coords)
from_cache = [n._from_cache for n in node2.inputs.values()]
assert sum(from_cache) > 0
def test_in_place(self):
coords = podpac.Coordinates(
[podpac.clinspace(0, 1, 4),
podpac.clinspace(0, 1, 3)],
dims=["lat", "lon"])
sine_node = Arange()
node = Mask(source=sine_node, mask=sine_node, in_place=True)
output = node.eval(coords)
a = sine_node.eval(coords)
# In-place editing doesn't seem to work here
# np.testing.assert_allclose(output, node.source._output)
coords = podpac.Coordinates(
[podpac.clinspace(0, 1, 4),
podpac.clinspace(0, 2, 3)],
dims=["lat", "lon"])
sine_node = Arange()
node = Mask(source=sine_node, mask=sine_node, in_place=False)
output = node.eval(coords)
a = sine_node.eval(coords)
assert not np.all(a == output)
def get_data(self, coordinates, slc):
node = Arange()
return node.eval(coordinates)
def test_year_substitution(self):
node = YearSubstituteCoordinates(source=Arange(), year="2018")
o = node.eval(COORDS)
assert o.time.dt.year.data[0] == 2018
assert not np.array_equal(o["time"], COORDS["time"].coordinates)
def test_interpolation_warning(self):
node = ReprojectedSource(source=Arange(),
reprojected_coordinates=self.coordinates)
output = node.eval(node.coordinates)
def test_time_expansion(self):
node = ExpandCoordinates(source=Arange(), time=("-5,D", "0,D", "1,D"))
o = node.eval(COORDS)
def test_spatial_expansion(self):
node = ExpandCoordinates(source=Arange(), lat=(-1, 1, 0.1))
o = node.eval(COORDS)
def test_no_expansion(self):
node = SelectCoordinates(source=Arange())
o = node.eval(COORDS)
def test_time_selection(self):
node = SelectCoordinates(source=Arange(),
time=("2017-08-01", "2017-09-30", "1,D"))
o = node.eval(COORDS)
def test_Arange(self):
coords = podpac.Coordinates([[0, 1, 2], [0, 1, 2, 3, 4]],
dims=["lat", "lon"])
node = Arange()
output = node.eval(coords)
assert output.shape == coords.shape
def test_spatial_selection(self):
node = SelectCoordinates(source=Arange(), lat=(46, 56, 1))
o = node.eval(COORDS)