def test_doy_window2_mean_rescale_max_min(self):
with podpac.settings:
podpac.settings.set_unsafe_eval(True)
coords = podpac.coordinates.concat([
podpac.Coordinates(
[podpac.crange("1999-12-29", "2000-01-03", "1,D",
"time")]),
podpac.Coordinates(
[podpac.crange("2001-12-30", "2002-01-02", "1,D",
"time")]),
])
node = Arange()
node_max = Arithmetic(source=node, eqn="(source < 5) + source")
node_min = Arithmetic(source=node, eqn="-1*(source < 5) + source")
nodedoywindow_s = FM(
source=node,
window=2,
cache_output=False,
force_eval=True,
scale_max=node_max,
scale_min=node_min,
rescale=False,
)
o_s = nodedoywindow_s.eval(coords)
np.testing.assert_array_almost_equal([0.5] * o_s.size, o_s)
def test_silent_nearest_neighbor_interp_bug_issue412(self):
node = podpac.data.Array(
source=[0, 1, 2],
coordinates=podpac.Coordinates([[1, 5, 9]], dims=["lat"]),
interpolation=[{
"method": "bilinear",
"dims": ["lat"],
"interpolators": [ScipyGrid]
}],
)
with pytest.raises(InterpolationException, match="can't be handled"):
o = node.eval(
podpac.Coordinates([podpac.crange(1, 9, 1)], dims=["lat"]))
node = podpac.data.Array(
source=[0, 1, 2],
coordinates=podpac.Coordinates([[1, 5, 9]], dims=["lat"]),
interpolation=[{
"method": "bilinear",
"dims": ["lat"]
}],
)
o = node.eval(
podpac.Coordinates([podpac.crange(1, 9, 1)], dims=["lat"]))
assert_array_equal(o.data, np.linspace(0, 2, 9))
def test_SinCoords(self):
coords = podpac.Coordinates([
podpac.crange(-90, 90, 1.0),
podpac.crange("2018-01-01", "2018-01-30", "1,D")
],
dims=["lat", "time"])
node = SinCoords()
output = node.eval(coords)
assert output.shape == coords.shape
def test_SinCoords(self):
coords = podpac.Coordinates([
podpac.crange(-90, 90, 1.0),
podpac.crange('2018-01-01', '2018-01-30', '1,D')
],
dims=['lat', 'time'])
node = SinCoords()
output = node.eval(coords)
assert output.shape == coords.shape
def test_mask_defaults(self):
coords = podpac.Coordinates(
[podpac.crange(-90, 90, 1.0),
podpac.crange(-180, 180, 1.0)],
dims=["lat", "lon"])
sine_node = Arange()
a = sine_node.eval(coords).copy()
a.data[a.data == 1] = np.nan
node = Mask(source=sine_node, mask=sine_node)
output = node.eval(coords)
np.testing.assert_allclose(output, a)
def test_Arithmetic(self):
coords = podpac.Coordinates(
[podpac.crange(-90, 90, 1.0),
podpac.crange(-180, 180, 1.0)],
dims=['lat', 'lon'])
sine_node = SinCoords()
node = Arithmetic(A=sine_node,
B=sine_node,
eqn='2*abs(A) - B + {offset}',
params={'offset': 1})
output = node.eval(coords)
a = sine_node.eval(coords)
b = sine_node.eval(coords)
np.testing.assert_allclose(output, 2 * abs(a) - b + 1)
def get_native_coordinates(self):
return podpac.Coordinates([
podpac.crange('2010-01-01', '2018-01-01', '4,h'),
podpac.clinspace(-180, 180, 6),
podpac.clinspace(-80, -70, 6)
],
dims=['time', 'lat', 'lon'])
def test_doy_window2(self):
coords = podpac.coordinates.concat([
podpac.Coordinates(
[podpac.crange("1999-12-29", "2000-01-03", "1,D", "time")]),
podpac.Coordinates(
[podpac.crange("2001-12-30", "2002-01-02", "1,D", "time")]),
])
node = Arange()
nodedoywindow = F(source=node,
window=2,
cache_output=False,
force_eval=True)
o = nodedoywindow.eval(coords)
np.testing.assert_array_equal(o, [6, 5, 3, 3, 5, 6])
def test_eval(self):
lat = clinspace(45, 66, 30, name="lat")
lon = clinspace(-80, 70, 40, name="lon")
time = crange("2017-09-01", "2017-10-31", "1,D", name="time")
kernel1d = [1, 2, 1]
kernel2d = [[1, 2, 1]]
kernel3d = [[[1, 2, 1]]]
node1d = Convolution(source=Arange(), kernel=kernel1d, kernel_dims=["time"])
node2d = Convolution(source=Arange(), kernel=kernel2d, kernel_dims=["lat", "lon"])
node3d = Convolution(source=Arange(), kernel=kernel3d, kernel_dims=["lon", "lat", "time"])
o = node1d.eval(Coordinates([time]))
o = node2d.eval(Coordinates([lat, lon]))
o = node3d.eval(Coordinates([lat, lon, time]))
with pytest.raises(
ValueError, match="Kernel dims must contain all of the dimensions in source but not all of "
):
node2d.eval(Coordinates([lat, lon, time]))
with pytest.raises(
ValueError, match="Kernel dims must contain all of the dimensions in source but not all of "
):
node2d.eval(Coordinates([lat, time]))
def test_evaluate_not_allowed(self):
with podpac.settings:
podpac.settings.set_unsafe_eval(False)
coords = podpac.Coordinates(
[podpac.crange(-90, 90, 1.0),
podpac.crange(-180, 180, 1.0)],
dims=["lat", "lon"])
sine_node = SinCoords()
with pytest.warns(UserWarning, match="Insecure evaluation"):
node = Arithmetic(A=sine_node,
B=sine_node,
eqn="2*abs(A) - B + {offset}",
params={"offset": 1})
with pytest.raises(PermissionError):
node.eval(coords)
def test_evaluate(self):
with podpac.settings:
podpac.settings.set_unsafe_eval(True)
coords = podpac.Coordinates(
[podpac.crange(-90, 90, 1.0),
podpac.crange(-180, 180, 1.0)],
dims=["lat", "lon"])
sine_node = SinCoords()
node = Arithmetic(A=sine_node,
B=sine_node,
eqn="2*abs(A) - B + {offset}",
params={"offset": 1})
output = node.eval(coords)
a = sine_node.eval(coords)
b = sine_node.eval(coords)
np.testing.assert_allclose(output, 2 * abs(a) - b + 1)
def test_evaluate(self):
with podpac.settings:
podpac.settings.set_unsafe_eval(True)
coords = podpac.Coordinates(
[podpac.crange(-90, 90, 1.0),
podpac.crange(-180, 180, 1.0)],
dims=["lat", "lon"])
a = SinCoords()
b = Arange()
node = Generic(code="import numpy as np\noutput = np.minimum(a,b)",
a=a,
b=b)
output = node.eval(coords)
a = node.eval(coords)
b = node.eval(coords)
np.testing.assert_allclose(output, np.minimum(a, b))
def test_evaluate_not_allowed(self):
with podpac.settings:
podpac.settings.set_unsafe_eval(False)
coords = podpac.Coordinates(
[podpac.crange(-90, 90, 1.0),
podpac.crange(-180, 180, 1.0)],
dims=["lat", "lon"])
a = SinCoords()
b = Arange()
with pytest.warns(UserWarning, match="Insecure evaluation"):
node = Generic(
code="import numpy as np\noutput = np.minimum(a,b)",
a=a,
b=b)
with pytest.raises(PermissionError):
node.eval(coords)
def setup_module():
global coords, source, data
coords = podpac.Coordinates(
[podpac.clinspace(0, 1, 10), podpac.clinspace(0, 1, 10), podpac.crange('2018-01-01', '2018-01-10', '1,D')],
dims=['lat', 'lon', 'time'])
a = np.random.random(coords.shape)
a[3, 0, 0] = np.nan
a[0, 3, 0] = np.nan
a[0, 0, 3] = np.nan
source = Array(source=a, native_coordinates=coords)
data = source.eval(coords)
def get_native_coordinates(self):
"""Summary
Returns
-------
TYPE
Description
"""
try:
return self.load_cached_obj('native.coordinates')
except:
pass
ds = self.dataset
times = self.get_available_dates()
lons = podpac.crange(ds['lon'][0], ds['lon'][-1], ds['lon'][1] - ds['lon'][0])
lats = podpac.crange(ds['lat'][0], ds['lat'][-1], ds['lat'][1] - ds['lat'][0])
coords = podpac.Coordinates([times, lats, lons], dims=['time', 'lat', 'lon'])
self.cache_obj(coords, 'native.coordinates')
return coords
class MyDataSource(DataSource):
coordinates = podpac.Coordinates(
[
podpac.crange("2010-01-01", "2018-01-01", "4,h"),
podpac.clinspace(-180, 180, 6),
podpac.clinspace(-80, -70, 6),
],
dims=["time", "lat", "lon"],
)
def get_data(self, coordinates, slc):
node = Arange()
return node.eval(coordinates)
def test_year_substitution_missing_coords(self):
source = Array(
source=[[1, 2, 3], [4, 5, 6]],
coordinates=podpac.Coordinates([
podpac.crange("2018-01-01", "2018-01-02", "1,D"),
podpac.clinspace(45, 66, 3)
],
dims=["time", "lat"]),
)
node = YearSubstituteCoordinates(source=source, year="2018")
o = node.eval(COORDS)
assert o.time.dt.year.data[0] == 2018
assert o["time"].data != xr.DataArray(COORDS["time"].coordinates).data
def test_year_substitution_missing_coords_orig_coords(self):
source = Array(
source=[[1, 2, 3], [4, 5, 6]],
coordinates=podpac.Coordinates([
podpac.crange("2018-01-01", "2018-01-02", "1,D"),
podpac.clinspace(45, 66, 3)
],
dims=["time", "lat"]),
)
node = YearSubstituteCoordinates(source=source,
year="2018",
substitute_eval_coords=True)
o = node.eval(COORDS)
assert o.time.dt.year.data[0] == 2017
np.testing.assert_array_equal(o["time"], COORDS["time"].coordinates)
def test_selection_crs(self):
base = podpac.core.data.array_source.ArrayRaw(
source=[0, 1, 2],
coordinates=podpac.Coordinates(
[[1, 5, 9]],
dims=["time"],
crs="+proj=longlat +datum=WGS84 +no_defs +vunits=m"),
)
node = podpac.interpolators.Interpolate(source=base,
interpolation="linear")
tocrds = podpac.Coordinates([podpac.crange(1, 9, 1, "time")],
crs="EPSG:4326")
o = node.eval(tocrds)
assert o.crs == tocrds.crs
assert_array_equal(o.data, np.linspace(0, 2, 9))
def test_doy_window2_mean_rescale_float(self):
coords = podpac.coordinates.concat([
podpac.Coordinates(
[podpac.crange("1999-12-29", "2000-01-03", "1,D", "time")]),
podpac.Coordinates(
[podpac.crange("2001-12-30", "2002-01-02", "1,D", "time")]),
])
node = Arange()
nodedoywindow = FM(source=node,
window=2,
cache_output=False,
force_eval=True)
o = nodedoywindow.eval(coords)
nodedoywindow_s = FM(source=node,
window=2,
cache_output=False,
force_eval=True,
scale_float=[0, coords.size],
rescale=True)
o_s = nodedoywindow_s.eval(coords)
np.testing.assert_array_almost_equal(o, o_s)
def get_native_coordinates(self):
"""Summary
Returns
-------
TYPE
Description
"""
try:
return self.load_cached_obj('native.coordinates')
except:
pass
ds = self.dataset
base_date = ds['time'].attributes['units']
base_date = self.date_url_re.search(base_date).group()
times = (ds['time'][:]).astype('timedelta64[h]') + np.array(base_date, 'datetime64')
lons = podpac.crange(ds['lon'][0], ds['lon'][-1], ds['lon'][1] - ds['lon'][0])
lats = podpac.crange(ds['lat'][0], ds['lat'][-1], ds['lat'][1] - ds['lat'][0])
coords = podpac.Coordinates([times, lats, lons], dims=['time', 'lat', 'lon'])
self.cache_obj(coords, 'native.coordinates')
return coords
def setup_module():
global coords, source, data, multisource, bdata
coords = podpac.Coordinates(
[
podpac.clinspace(0, 1, 10),
podpac.clinspace(0, 1, 10),
podpac.crange("2018-01-01", "2018-01-10", "1,D")
],
dims=["lat", "lon", "time"],
)
a = np.random.random(coords.shape)
a[3, 0, 0] = np.nan
a[0, 3, 0] = np.nan
a[0, 0, 3] = np.nan
source = Array(source=a, coordinates=coords)
data = source.eval(coords)
ab = np.stack([a, 2 * a], -1)
multisource = Array(source=ab, coordinates=coords, outputs=["a", "b"])
bdata = 2 * data
def test_eval(self):
lat = clinspace(45, 66, 30, name="lat")
lon = clinspace(-80, 70, 40, name="lon")
time = crange("2017-09-01", "2017-10-31", "1,D", name="time")
kernel1d = [1, 2, 1]
kernel2d = [[1, 2, 1]]
kernel3d = [[[1, 2, 1]]]
node1d = Convolution(source=Arange(),
kernel=kernel1d,
kernel_dims=["time"])
node2d = Convolution(source=Arange(),
kernel=kernel2d,
kernel_dims=["lat", "lon"])
node3d = Convolution(source=Arange(),
kernel=kernel3d,
kernel_dims=["lon", "lat", "time"])
o = node1d.eval(Coordinates([time]))
o = node2d.eval(Coordinates([lat, lon]))
o = node3d.eval(Coordinates([lat, lon, time]))
def test_composited(self):
# specify source datetime, select forecast at evaluation from time coordinates
gfs_soim = gfs.GFS(parameter=self.parameter,
level=self.level,
date=self.date,
hour="1200",
anon=True)
# whole world forecast at 15:30
forecast_time = datetime.datetime.strptime(self.date + " 15:30",
"%Y%m%d %H:%M")
coords = gfs_soim.sources[0].coordinates
c = podpac.Coordinates([coords["lat"], coords["lon"], forecast_time],
dims=["lat", "lon", "time"])
o = gfs_soim.eval(c)
# time series: get the forecast at lat=42, lon=275 every hour for 6 hours
start = forecast_time
stop = forecast_time + datetime.timedelta(hours=6)
c = podpac.Coordinates(
[42, 282, podpac.crange(start, stop, "1,h")],
dims=["lat", "lon", "time"])
o = gfs_soim.eval(c)
from podpac.datalib import cosmos_stations
from podpac.datalib.modis_pds import MODIS
from podpac.datalib.satutils import Landsat8, Sentinel2
terrain2 = datalib.terraintiles.TerrainTiles(zoom=2)
terrain10 = datalib.terraintiles.TerrainTiles(zoom=10)
modis = MODIS(product="MCD43A4.006", data_key="B01")
landsat_b1 = Landsat8(asset="B1", min_bounds_span={"time": "4,D"})
cosmos = cosmos_stations.COSMOSStations()
from podpac.datalib.satutils import Landsat8, Sentinel2
sentinel2_b2 = Sentinel2(asset="B02", min_bounds_span={"time": "4,D"})
lat = podpac.crange(60, 10, -2.0) # (start, stop, step)
lon = podpac.crange(-130, -60, 2.0) # (start, stop, step)
# Specify date and time
time = "2019-04-23"
# Create the PODPAC Coordinates
# lat_lon_time_US = podpac.Coordinates([lat, lon, time], dims=['lat', 'lon', 'time'])
lat_lon_time_DHMC = podpac.Coordinates([
podpac.clinspace(43.7125, 43.6, 256),
podpac.clinspace(-72.3125, -72.3, 256), time
],
dims=["lat", "lon", "time"])
o = sentinel2_b2.eval(lat_lon_time_DHMC)
hour="1200",
cache_ctrl=cache_ctrl,
anon=True,
)
# whole world forecast at this time tomorrow
c = Coordinates(
[gfs_soim.coordinates["lat"], gfs_soim.coordinates["lon"], tomorrow],
dims=["lat", "lon", "time"])
o = gfs_soim.eval(c)
print(o)
# time series: get the forecast at lat=42, lon=275 every hour for the next 6 hours
start = now
stop = now + datetime.timedelta(hours=6)
c = Coordinates([42, 282, podpac.crange(start, stop, "1,h")],
dims=["lat", "lon", "time"])
o = gfs_soim.eval(c)
print(o)
# latest (get latest source, select forecast at evaluation)
print("GFSLatest node (parameter, level)")
gfs_soim = GFSLatest(parameter=parameter,
level=level,
cache_ctrl=cache_ctrl,
anon=True)
c = Coordinates(
[gfs_soim.coordinates["lat"], gfs_soim.coordinates["lon"], tomorrow],
dims=["lat", "lon", "time"])
o = gfs_soim.eval(c)
print(o)
cache_ctrl=["ram", "disk"],
interpolation={
"method": "nearest",
"params": {
"use_selector": False,
"remove_nan": True,
"time_scale": "1,M"
}
},
)
sd = cs.stations_data
ci = cs.source_coordinates.select(bounds)
ce = podpac.coordinates.merge_dims([
podpac.Coordinates(
[podpac.crange("2018-05-01", "2018-06-01", "1,D", "time")]), ci
])
cg = podpac.Coordinates([
podpac.clinspace(ci["lat"].bounds[1], ci["lat"].bounds[0], 12, "lat"),
podpac.clinspace(ci["lon"].bounds[1], ci["lon"].bounds[0], 16, "lon"),
ce["time"],
])
o = cs.eval(ce)
o_r = csr.eval(ce)
og = cs.eval(cg)
# Test helper functions
labels = cs.stations_label
lat_lon = cs.latlon_from_label("Manitou")
labels = cs.label_from_latlon(lat_lon)
lat_lon2 = cs.latlon_from_label("No Match Here")
def test_signal(self):
lat = podpac.clinspace(45, 66, 30)
lon = podpac.clinspace(-80, 70, 40)
time = podpac.crange('2017-09-01', '2017-10-31', '1,D')
coords = podpac.Coordinates([time, lat, lon],
dims=['time', 'lat', 'lon'])
coords_spatial = podpac.Coordinates([lat, lon], dims=['lat', 'lon'])
coords_time = podpac.Coordinates([time], dims=['time'])
kernel3 = np.array([[[1, 2, 1]]])
kernel2 = np.array([[1, 2, 1]])
kernel1 = np.array([1, 2, 1])
o = Arange().eval(coords)
node = Convolution(source=Arange(), kernel=kernel3)
o3d_full = node.eval(coords)
node = Convolution(source=Arange(), kernel=kernel2)
o2d_spatial1 = node.eval(coords_spatial)
node = SpatialConvolution(source=Arange(), kernel=kernel2)
o3d_spatial = node.eval(coords)
o2d_spatial2 = node.eval(coords_spatial)
node = TimeConvolution(source=Arange(), kernel=kernel1)
o3d_time = node.eval(coords)
o3d_time = node.eval(coords_time)
node = SpatialConvolution(source=Arange(),
kernel_type='gaussian, 3, 1')
o3d_spatial = node.eval(coords)
o2d_spatial2 = node.eval(coords_spatial)
node = SpatialConvolution(source=Arange(), kernel_type='mean, 3')
o3d_spatial = node.eval(coords)
o2d_spatial2 = node.eval(coords_spatial)
node = TimeConvolution(source=Arange(), kernel_type='gaussian, 3, 1')
o3d_time = node.eval(coords)
node = TimeConvolution(source=Arange(), kernel_type='mean, 3')
o3d_time = node.eval(coords)
node = Convolution(source=Arange(), kernel=kernel2)
with pytest.raises(Exception): # TODO which exception
node.eval(coords)
node = Convolution(source=Arange(), kernel=kernel1)
with pytest.raises(Exception): # TODO which exception?
node.eval(coords_spatial)
with pytest.raises(Exception): # TODO which exception?
node = SpatialConvolution(source=Arange(), kernel=kernel3)
with pytest.raises(Exception): # TODO which exception?
node = SpatialConvolution(source=Arange(), kernel=kernel1)
with pytest.raises(Exception): # TODO which exception?
node = TimeConvolution(source=Arange(), kernel=kernel3)
with pytest.raises(Exception): # TODO which exception?
node = TimeConvolution(source=Arange(), kernel=kernel2)
node = TimeConvolution(source=Arange(), kernel=kernel1)
with pytest.raises(Exception): # TODO which exception?
node.eval(coords_spatial)
import numpy as np
import podpac
import podpac.datalib
# Create some nodes to help get realistic coordinates
cosmos = podpac.datalib.cosmos_stations.COSMOSStations()
soilscape = podpac.datalib.soilscape.SoilSCAPE20min(site="Canton_OK")
# Now do the coordinates
time_points = podpac.crange("2016-01-01", "2016-02-01", "1,D", "time")
# Soilscape coordinates
soilscape_points = soilscape.make_coordinates(time="2016-01-01")
soilscape_region = podpac.Coordinates(
[
podpac.clinspace(soilscape_points["lat"].bounds[1],
soilscape_points["lat"].bounds[0], 64),
podpac.clinspace(soilscape_points["lon"].bounds[0],
soilscape_points["lon"].bounds[1], 64),
"2016-01-01",
4.0,
],
dims=["lat", "lon", "time", "alt"],
)
soilscape_timeseries = podpac.coordinates.merge_dims([
soilscape_points[:2].drop("time"),
podpac.Coordinates([time_points], crs=soilscape_points.crs)
])
# COSMOS coordinates
cosmos_points = cosmos.source_coordinates.select({
print("Sample DateFolder Sources:", sm_datefolder.sources[:3],
"... (%d)" % len(sm_datefolder.sources))
# sample SMAPSource info
sm_source = sm_datefolder.sources[0]
print("Sample DAP Source:", sm_source)
print("Sample DAP Source Definition:", sm_source.json_pretty)
print("Sample DAP Native Coordinates:", sm_source.coordinates)
print("Another Sample DAP Native Coordinates:",
sm_datefolder.sources[1].coordinates)
# eval whole world
c_world = Coordinates(
[
podpac.crange(90, -90, -2.0),
podpac.crange(-180, 180, 2.0), "2018-05-19T12:00:00"
],
dims=["lat", "lon", "time"],
)
o = sm.eval(c_world)
o.plot(cmap="gist_earth_r")
pyplot.axis("scaled")
# eval points over time
lat = [45.0, 45.0, 0.0, 45.0]
lon = [-100.0, 20.0, 20.0, 100.0]
c_pts = Coordinates(
[[lat, lon],
podpac.crange("2018-05-15T00", "2018-05-19T00", "3,h")],
dims=["lat_lon", "time"])
