def test_get_source_data(self):
a = ArrayRaw(source=np.arange(5) + 100,
coordinates=podpac.Coordinates([[0, 1, 2, 3, 4]],
dims=["lat"]))
b = ArrayRaw(source=np.arange(5) + 200,
coordinates=podpac.Coordinates([[5, 6, 7, 8, 9]],
dims=["lat"]))
c = ArrayRaw(source=np.arange(5) + 300,
coordinates=podpac.Coordinates([[10, 11, 12, 13, 14]],
dims=["lat"]))
node = TileCompositorRaw(sources=[a, b, c])
data = node.get_source_data()
np.testing.assert_array_equal(data["lat"], np.arange(15))
np.testing.assert_array_equal(
data, np.hstack([source.source for source in node.sources]))
# with bounds
data = node.get_source_data({"lat": (2.5, 6.5)})
np.testing.assert_array_equal(data["lat"], [3, 4, 5, 6])
np.testing.assert_array_equal(data, [103, 104, 200, 201])
# error
with podpac.settings:
podpac.settings.set_unsafe_eval(True)
d = podpac.algorithm.Arithmetic(eqn="a+2", a=a)
node = TileCompositorRaw(sources=[a, b, c, d])
with pytest.raises(ValueError,
match="Cannot get composited source data"):
node.get_source_data()
def test_select_sources_intersection(self):
source_coords = podpac.Coordinates([[0, 10]], ["time"])
node = BaseCompositor(sources=[DataSource(),
DataSource()],
source_coordinates=source_coords)
# select all
selected = node.select_sources(source_coords)
assert len(selected) == 2
assert selected[0] == node.sources[0]
assert selected[1] == node.sources[1]
# select first
c = podpac.Coordinates(
[podpac.clinspace(0, 1, 10),
podpac.clinspace(0, 1, 11), 0], ["lat", "lon", "time"])
selected = node.select_sources(c)
assert len(selected) == 1
assert selected[0] == node.sources[0]
# select second
c = podpac.Coordinates(
[podpac.clinspace(0, 1, 10),
podpac.clinspace(0, 1, 11), 10], ["lat", "lon", "time"])
selected = node.select_sources(c)
assert len(selected) == 1
assert selected[0] == node.sources[1]
# select none
c = podpac.Coordinates(
[podpac.clinspace(0, 1, 10),
podpac.clinspace(0, 1, 11), 100], ["lat", "lon", "time"])
selected = node.select_sources(c)
assert len(selected) == 0
def test_doy_window2_mean_rescale_max_min(self):
with podpac.settings:
podpac.settings.set_unsafe_eval(True)
coords = podpac.coordinates.concat([
podpac.Coordinates(
[podpac.crange("1999-12-29", "2000-01-03", "1,D",
"time")]),
podpac.Coordinates(
[podpac.crange("2001-12-30", "2002-01-02", "1,D",
"time")]),
])
node = Arange()
node_max = Arithmetic(source=node, eqn="(source < 5) + source")
node_min = Arithmetic(source=node, eqn="-1*(source < 5) + source")
nodedoywindow_s = FM(
source=node,
window=2,
cache_output=False,
force_eval=True,
scale_max=node_max,
scale_min=node_min,
rescale=False,
)
o_s = nodedoywindow_s.eval(coords)
np.testing.assert_array_almost_equal([0.5] * o_s.size, o_s)
def setup_class(cls):
cls.c1 = podpac.Coordinates(
[podpac.clinspace((0, 0), (1, 1), 10), [0, 1, 2]],
dims=['lat_lon', 'time'])
cls.c2 = podpac.Coordinates(
[podpac.clinspace((0.5, 0.1), (1.5, 1.1), 15)], dims=['lat_lon'])
cls.crds = [cls.c1, cls.c2]
def test_composite_extra_dims(self):
with podpac.settings:
podpac.settings["MULTITHREADING"] = False
coords = podpac.Coordinates([[0, 1], [10, 20, 30]],
dims=["lat", "lon"])
a = Array(source=np.ones(coords.shape), coordinates=coords)
extra = podpac.Coordinates(
[coords["lat"], coords["lon"], "2020-01-01"],
dims=["lat", "lon", "time"])
# dims not provided, eval fails with extra dims
node = OrderedCompositor(sources=[a], interpolation="bilinear")
np.testing.assert_array_equal(node.eval(coords), a.source)
with pytest.raises(
podpac.NodeException,
match="Cannot evaluate compositor with requested dims"):
node.eval(extra)
# dims provided, remove extra dims
node = OrderedCompositor(sources=[a],
dims=["lat", "lon"],
interpolation="bilinear")
np.testing.assert_array_equal(node.eval(coords), a.source)
np.testing.assert_array_equal(node.eval(extra), a.source)
def test_silent_nearest_neighbor_interp_bug_issue412(self):
node = podpac.data.Array(
source=[0, 1, 2],
coordinates=podpac.Coordinates([[1, 5, 9]], dims=["lat"]),
interpolation=[{
"method": "bilinear",
"dims": ["lat"],
"interpolators": [ScipyGrid]
}],
)
with pytest.raises(InterpolationException, match="can't be handled"):
o = node.eval(
podpac.Coordinates([podpac.crange(1, 9, 1)], dims=["lat"]))
node = podpac.data.Array(
source=[0, 1, 2],
coordinates=podpac.Coordinates([[1, 5, 9]], dims=["lat"]),
interpolation=[{
"method": "bilinear",
"dims": ["lat"]
}],
)
o = node.eval(
podpac.Coordinates([podpac.crange(1, 9, 1)], dims=["lat"]))
assert_array_equal(o.data, np.linspace(0, 2, 9))
def test_composite_multithreaded(self):
with podpac.settings:
podpac.settings["MULTITHREADING"] = True
podpac.settings["N_THREADS"] = 8
acoords = podpac.Coordinates([[-1, 0, 1], [10, 20, 30]],
dims=["lat", "lon"])
asource = np.ones(acoords.shape)
asource[0, :] = np.nan
a = Array(source=asource, coordinates=acoords)
bcoords = podpac.Coordinates([[0, 1, 2, 3], [10, 20, 30, 40]],
dims=["lat", "lon"])
bsource = np.zeros(bcoords.shape)
bsource[:, 0] = np.nan
b = Array(source=bsource, coordinates=bcoords)
coords = podpac.Coordinates([[0, 1, 2], [10, 20, 30, 40, 50]],
dims=["lat", "lon"])
node = OrderedCompositor(sources=[a, b], interpolation="bilinear")
expected = np.array([[1.0, 1.0, 1.0, 0.0, np.nan],
[1.0, 1.0, 1.0, 0.0, np.nan],
[np.nan, np.nan, 0.0, 0.0, np.nan]])
np.testing.assert_allclose(node.eval(coords),
expected,
equal_nan=True)
node = OrderedCompositor(sources=[b, a], interpolation="bilinear")
expected = np.array([[1.0, 1.0, 0.0, 0.0, np.nan],
[1.0, 1.0, 0.0, 0.0, np.nan],
[np.nan, np.nan, 0.0, 0.0, np.nan]])
np.testing.assert_allclose(node.eval(coords),
expected,
equal_nan=True)
def test_eval_not_implemented(self):
node = Node()
with pytest.raises(NotImplementedError):
node.eval(podpac.Coordinates([]))
with pytest.raises(NotImplementedError):
node.eval(podpac.Coordinates([]), output=None)
def test_source_coordinates(self):
# none (default)
node = BaseCompositor(sources=[ARRAY_LAT, ARRAY_LON, ARRAY_TIME])
assert node.source_coordinates is None
# unstacked
node = BaseCompositor(
sources=[podpac.algorithm.Arange(),
podpac.algorithm.SinCoords()],
source_coordinates=podpac.Coordinates([[0, 1]], dims=["time"]),
)
# stacked
node = BaseCompositor(
sources=[podpac.algorithm.Arange(),
podpac.algorithm.SinCoords()],
source_coordinates=podpac.Coordinates([[[0, 1], [10, 20]]],
dims=["time_alt"]),
)
# invalid size
with pytest.raises(
ValueError,
match=
"Invalid source_coordinates, source and source_coordinates size mismatch"
):
node = BaseCompositor(
sources=[
podpac.algorithm.Arange(),
podpac.algorithm.SinCoords()
],
source_coordinates=podpac.Coordinates([[0, 1, 2]],
dims=["time"]),
)
with pytest.raises(
ValueError,
match=
"Invalid source_coordinates, source and source_coordinates size mismatch"
):
node = BaseCompositor(
sources=[
podpac.algorithm.Arange(),
podpac.algorithm.SinCoords()
],
source_coordinates=podpac.Coordinates([[0, 1, 2]],
dims=["time"]),
)
# invalid ndims
with pytest.raises(ValueError, match="Invalid source_coordinates"):
node = BaseCompositor(
sources=[
podpac.algorithm.Arange(),
podpac.algorithm.SinCoords()
],
source_coordinates=podpac.Coordinates([[0, 1], [10, 20]],
dims=["time", "alt"]),
)
def setup_class(cls):
class MyNode(Node):
pass
cls.node = MyNode(cache_type='disk')
cls.node.rem_cache(key='*', coordinates='*')
cls.coords = podpac.Coordinates([0, 0], dims=['lat', 'lon'])
cls.coords2 = podpac.Coordinates([1, 1], dims=['lat', 'lon'])
def find_coordinates(self):
return [
podpac.Coordinates([[0, 1, 2], [0, 10, 20]],
dims=["lat", "lon"],
crs="EPSG:2193"),
podpac.Coordinates([[3, 4], [30, 40]],
dims=["lat", "lon"],
crs="EPSG:2193"),
]
def setup_class(cls):
cls._ram_cache_enabled = podpac.settings["RAM_CACHE_ENABLED"]
podpac.settings["RAM_CACHE_ENABLED"] = True
class MyNode(Node):
pass
cls.node = MyNode(cache_ctrl=CacheCtrl([RamCacheStore()]))
cls.node.rem_cache(key="*", coordinates="*")
cls.coords = podpac.Coordinates([0, 0], dims=["lat", "lon"])
cls.coords2 = podpac.Coordinates([1, 1], dims=["lat", "lon"])
def test_get_data_time_interpolation(self):
# exact lat/lon, interpolated times
c1 = podpac.Coordinates(["2018-01-01T01:00:00", [1, 2], [1, 2]],
dims=["time", "lat", "lon"])
c2 = podpac.Coordinates(["2018-01-02T23:00:00", [1, 2], [1, 2]],
dims=["time", "lat", "lon"])
c3 = podpac.Coordinates(["2018-01-03T01:00:00", [1, 2], [1, 2]],
dims=["time", "lat", "lon"])
node = MockTileCompositor(interpolation="nearest")
np.testing.assert_array_equal(node.eval(c1), [[[5, 6], [9, 10]]])
np.testing.assert_array_equal(node.eval(c2),
[[[185, 186], [189, 190]]])
np.testing.assert_array_equal(node.eval(c3),
[[[185, 186], [189, 190]]])
def test_selection_crs(self):
base = podpac.core.data.array_source.ArrayRaw(
source=[0, 1, 2],
coordinates=podpac.Coordinates(
[[1, 5, 9]],
dims=["time"],
crs="+proj=longlat +datum=WGS84 +no_defs +vunits=m"),
)
node = podpac.interpolators.Interpolate(source=base,
interpolation="linear")
tocrds = podpac.Coordinates([podpac.crange(1, 9, 1, "time")],
crs="EPSG:4326")
o = node.eval(tocrds)
assert o.crs == tocrds.crs
assert_array_equal(o.data, np.linspace(0, 2, 9))
def get_coordinates(self):
# lookup available dates and use pre-fetched lat and lon bounds
time = podpac.Coordinates(
[[_parse_modis_date(date) for date in self.available_dates]],
dims=["time"],
crs=CRS)
return podpac.coordinates.merge_dims([time, self.tile_coordinates])
def get_coordinates(self):
return podpac.Coordinates(
[["2018-01-01", "2018-01-03", "2018-01-05"],
podpac.clinspace(0, 11, 12),
podpac.clinspace(0, 11, 12)],
dims=["time", "lat", "lon"],
)
def get_coordinates(self):
# use pre-fetched coordinate bounds (instead of loading from the dataset)
spatial_coords = get_tile_coordinates(self.horizontal, self.vertical)
time_coords = podpac.Coordinates([_parse_modis_date(self.date)],
["time"],
crs=spatial_coords.crs)
return podpac.coordinates.merge_dims([spatial_coords, time_coords])
def get_tile_coordinates(h, v):
""" use pre-fetched lat and lon bounds to get coordinates for a single tile """
lat_start, lat_stop = SINUSOIDAL_VERTICAL[v]
lon_start, lon_stop = SINUSOIDAL_HORIZONTAL[h]
lat = podpac.clinspace(lat_start, lat_stop, 2400, name="lat")
lon = podpac.clinspace(lon_start, lon_stop, 2400, name="lon")
return podpac.Coordinates([lat, lon], crs=CRS)
def setup_method(self, method):
self.coord_src = podpac.Coordinates([
podpac.clinspace(45, 0, 16),
podpac.clinspace(-70, -65, 16),
podpac.clinspace(0, 1, 2)
],
dims=['lat', 'lon', 'time'])
LON, LAT, TIME = np.meshgrid(self.coord_src['lon'].coordinates,
self.coord_src['lat'].coordinates,
self.coord_src['time'].coordinates)
self.latSource = LAT
self.lonSource = LON
self.timeSource = TIME
self.nasLat = Array(source=LAT.astype(float),
native_coordinates=self.coord_src,
interpolation='bilinear')
self.nasLon = Array(source=LON.astype(float),
native_coordinates=self.coord_src,
interpolation='bilinear')
self.nasTime = Array(source=TIME.astype(float),
native_coordinates=self.coord_src,
interpolation='bilinear')
self.sources = np.array([self.nasLat, self.nasLon, self.nasTime])
def test_get_bounds(self):
node = podpac.data.Array(
source=np.ones((3, 4)),
coordinates=podpac.Coordinates([range(3), range(4)], ["lat", "lon"], crs="EPSG:2193"),
)
with podpac.settings:
podpac.settings["DEFAULT_CRS"] = "EPSG:4326"
# specify crs
bounds, crs = node.get_bounds(crs="EPSG:3857")
assert bounds == {
"lat": (-13291827.558247399, -13291815.707967814),
"lon": (9231489.26794932, 9231497.142754894),
}
assert crs == "EPSG:3857"
# native/source crs
bounds, crs = node.get_bounds(crs="source")
assert bounds == {"lat": (0, 2), "lon": (0, 3)}
assert crs == "EPSG:2193"
# default crs
bounds, crs = node.get_bounds()
assert bounds == {
"lat": (-75.81365382984804, -75.81362774074242),
"lon": (82.92787904584206, 82.92794978642414),
}
assert crs == "EPSG:4326"
def test_multiple_outputs(self):
class MyAlgorithm(Algorithm):
x = NodeTrait().tag(attr=True)
y = NodeTrait().tag(attr=True)
outputs = ["sum", "prod", "diff"]
def algorithm(self, inputs):
sum_ = inputs["x"] + inputs["y"]
prod = inputs["x"] * inputs["y"]
diff = inputs["x"] - inputs["y"]
return np.stack([sum_, prod, diff], -1)
coords = podpac.Coordinates([[0, 1, 2], [10, 20]], dims=["lat", "lon"])
x = Arange()
y = Array(source=np.full(coords.shape, 2), coordinates=coords)
xout = np.arange(6).reshape(3, 2)
# all outputs
node = MyAlgorithm(x=x, y=y)
result = node.eval(coords)
assert result.dims == ("lat", "lon", "output")
np.testing.assert_array_equal(result["output"],
["sum", "prod", "diff"])
np.testing.assert_array_equal(result.sel(output="sum"), xout + 2)
np.testing.assert_array_equal(result.sel(output="prod"), xout * 2)
np.testing.assert_array_equal(result.sel(output="diff"), xout - 2)
# extract an output
node = MyAlgorithm(x=x, y=y, output="prod")
result = node.eval(coords)
assert result.dims == ("lat", "lon")
np.testing.assert_array_equal(result, xout * 2)
def test_extract_output(self):
coords = podpac.Coordinates([[0, 1, 2, 3], [0, 1]],
dims=["lat", "lon"])
class MyNode1(Node):
outputs = ["a", "b", "c"]
def _eval(self, coordinates, output=None, selector=None):
return self.create_output_array(coordinates)
# don't extract when no output field is requested
node = MyNode1()
out = node.eval(coords)
assert out.shape == (4, 2, 3)
# do extract when an output field is requested
node = MyNode1(output="b")
out = node.eval(coords)
assert out.shape == (4, 2)
# should still work if the node has already extracted it
class MyNode2(Node):
outputs = ["a", "b", "c"]
def _eval(self, coordinates, output=None, selector=None):
out = self.create_output_array(coordinates)
return out.sel(output=self.output)
node = MyNode2(output="b")
out = node.eval(coords)
assert out.shape == (4, 2)
def test_evaluate_transpose(self):
class MyNode(Node):
def _eval(self, coordinates, output=None, selector=None):
coords = coordinates.transpose("lat", "lon")
data = np.arange(coords.size).reshape(coords.shape)
a = self.create_output_array(coords, data=data)
if output is None:
output = a
else:
output[:] = a.transpose(*output.dims)
return output
coords = podpac.Coordinates([[0, 1, 2, 3], [0, 1]],
dims=["lat", "lon"])
node = MyNode()
o1 = node.eval(coords)
o2 = node.eval(coords.transpose("lon", "lat"))
# returned output should match the requested coordinates and data should be transposed
assert o1.dims == ("lat", "lon")
assert o2.dims == ("lon", "lat")
np.testing.assert_array_equal(o2.transpose("lat", "lon").data, o1.data)
# with transposed output
o3 = node.create_output_array(coords.transpose("lon", "lat"))
o4 = node.eval(coords, output=o3)
assert o3.dims == ("lon", "lat") # stay the same
assert o4.dims == ("lat", "lon") # match requested coordinates
np.testing.assert_equal(o3.transpose("lat", "lon").data, o4.data)
def get_native_coordinates(self):
return podpac.Coordinates([
podpac.crange('2010-01-01', '2018-01-01', '4,h'),
podpac.clinspace(-180, 180, 6),
podpac.clinspace(-80, -70, 6)
],
dims=['time', 'lat', 'lon'])
def get_coordinates(self):
# get spatial coordinates from rasterio over s3
spatial_coordinates = super(SatUtilsSource, self).get_coordinates()
time = podpac.Coordinates([self.date],
dims=["time"],
crs=spatial_coordinates.crs)
return podpac.coordinates.merge_dims([spatial_coordinates, time])
def test_interpolation(self):
a = ArrayRaw(source=np.arange(5) + 100,
coordinates=podpac.Coordinates([[0, 1, 2, 3, 4]],
dims=["lat"]))
b = ArrayRaw(source=np.arange(5) + 200,
coordinates=podpac.Coordinates([[5, 6, 7, 8, 9]],
dims=["lat"]))
c = ArrayRaw(source=np.arange(5) + 300,
coordinates=podpac.Coordinates([[10, 11, 12, 13, 14]],
dims=["lat"]))
node = TileCompositor(sources=[a, b, c], interpolation="bilinear")
output = node.eval(podpac.Coordinates([[3.5, 4.5, 5.5]], dims=["lat"]))
np.testing.assert_array_equal(output["lat"], [3.5, 4.5, 5.5])
np.testing.assert_array_equal(output, [103.5, 152.0, 200.5])
def test_doy_window2(self):
coords = podpac.coordinates.concat([
podpac.Coordinates(
[podpac.crange("1999-12-29", "2000-01-03", "1,D", "time")]),
podpac.Coordinates(
[podpac.crange("2001-12-30", "2002-01-02", "1,D", "time")]),
])
node = Arange()
nodedoywindow = F(source=node,
window=2,
cache_output=False,
force_eval=True)
o = nodedoywindow.eval(coords)
np.testing.assert_array_equal(o, [6, 5, 3, 3, 5, 6])
def test_composition(self):
a = ArrayRaw(source=np.arange(5) + 100,
coordinates=podpac.Coordinates([[0, 1, 2, 3, 4]],
dims=["lat"]))
b = ArrayRaw(source=np.arange(5) + 200,
coordinates=podpac.Coordinates([[5, 6, 7, 8, 9]],
dims=["lat"]))
c = ArrayRaw(source=np.arange(5) + 300,
coordinates=podpac.Coordinates([[10, 11, 12, 13, 14]],
dims=["lat"]))
node = TileCompositorRaw(sources=[a, b, c])
output = node.eval(podpac.Coordinates([[3.5, 4.5, 5.5]], dims=["lat"]))
np.testing.assert_array_equal(output["lat"], [3, 4, 5, 6])
np.testing.assert_array_equal(output, [103, 104, 200, 201])
def test_eval_missing_dim(self):
c = podpac.Coordinates([COORDS["lat"]])
node = MockWCSBase()
with pytest.raises(ValueError,
match="Cannot evaluate these coordinates"):
output = node.eval(c)
def test_eval_point(self):
node = OGRRaw(source=self.source,
driver=self.driver,
layer=self.layer,
attribute=self.attribute)
coords = podpac.Coordinates([43.7, -72.3], dims=["lat", "lon"])
output = node.eval(coords)