def setup_method(self, method):
self.coord_src = podpac.Coordinates([
podpac.clinspace(45, 0, 16),
podpac.clinspace(-70, -65, 16),
podpac.clinspace(0, 1, 2)
],
dims=['lat', 'lon', 'time'])
LON, LAT, TIME = np.meshgrid(self.coord_src['lon'].coordinates,
self.coord_src['lat'].coordinates,
self.coord_src['time'].coordinates)
self.latSource = LAT
self.lonSource = LON
self.timeSource = TIME
self.nasLat = Array(source=LAT.astype(float),
native_coordinates=self.coord_src,
interpolation='bilinear')
self.nasLon = Array(source=LON.astype(float),
native_coordinates=self.coord_src,
interpolation='bilinear')
self.nasTime = Array(source=TIME.astype(float),
native_coordinates=self.coord_src,
interpolation='bilinear')
self.sources = np.array([self.nasLat, self.nasLon, self.nasTime])
def test_source_trait(self):
""" must be an ndarray """
node = Array(source=self.data, native_coordinates=self.coordinates)
# list is coercable to array
node = Array(source=[0, 1, 1], native_coordinates=self.coordinates)
def test_get_data(self):
""" defined get_data function"""
source = self.data
node = Array(source=source, native_coordinates=self.coordinates)
output = node.eval(self.coordinates)
assert isinstance(output, UnitsDataArray)
assert output.values[0, 0] == source[0, 0]
assert output.values[4, 5] == source[4, 5]
def setup_module():
global coords, source, data
coords = podpac.Coordinates(
[podpac.clinspace(0, 1, 10), podpac.clinspace(0, 1, 10), podpac.crange('2018-01-01', '2018-01-10', '1,D')],
dims=['lat', 'lon', 'time'])
a = np.random.random(coords.shape)
a[3, 0, 0] = np.nan
a[0, 3, 0] = np.nan
a[0, 0, 3] = np.nan
source = Array(source=a, native_coordinates=coords)
data = source.eval(coords)
def test_put_and_remove_array_datasource_numpy_array():
lat = [0, 1, 2]
lon = [10, 20, 30, 40]
dates = ['2018-01-01', '2018-01-02']
native_coordinates = Coordinates([lat, lon, dates], ['lat', 'lon', 'time'])
source = np.zeros(native_coordinates.shape)
array_data_source = Array(source=source,
native_coordinates=native_coordinates)
put_data = np.zeros(native_coordinates.shape)
cache.put(node=array_data_source,
data=put_data,
key='key',
coordinates=native_coordinates,
mode='all',
update=False)
cached_data = cache.get(node=array_data_source,
key='key',
coordinates=native_coordinates,
mode='all')
assert (cached_data == put_data).all()
cache.rem(node=array_data_source,
key='key',
coordinates=native_coordinates,
mode='all')
assert not cache.has(node=array_data_source,
key='key',
coordinates=native_coordinates,
mode='all')
with pytest.raises(CacheException):
cache.get(node=array_data_source,
key='key',
coordinates=native_coordinates,
mode='all')
cache.rem(node='*', key='*', coordinates='*',
mode='all') # clear the cache stores
def make_array_data_source(coords_func=None, data_func=None):
if data_func is None:
data_func = np.zeros
if coords_func is None:
coords_func = make_lat_lon_time_grid_coords
coords = coords_func()
return Array(source=data_func(coords.shape), native_coordinates=coords)
def test_array(self):
"""Test array data source"""
from podpac.core.data.types import Array
arr = np.random.rand(16, 11)
lat = np.random.rand(16)
lon = np.random.rand(16)
coord = Coordinate(lat_lon=(lat, lon),
time=(0, 10, 11),
order=['lat_lon', 'time'])
node = Array(source=arr, native_coordinates=coord)
coordg = Coordinate(lat=(0, 1, 8), lon=(0, 1, 8), order=('lat', 'lon'))
coordt = Coordinate(time=(3, 5, 2))
node.eval(coordt)
node.eval(coordg)
class TestBasicInterpolation(object):
""" Test interpolation methods"""
def setup_method(self, method):
self.coord_src = Coordinates([
clinspace(45, 0, 16),
clinspace(-70., -65., 16),
clinspace(0, 1, 2)
],
dims=['lat', 'lon', 'time'])
LON, LAT, TIME = np.meshgrid(self.coord_src['lon'].coordinates,
self.coord_src['lat'].coordinates,
self.coord_src['time'].coordinates)
self.latSource = LAT
self.lonSource = LON
self.timeSource = TIME
self.nasLat = Array(source=LAT.astype(float),
native_coordinates=self.coord_src,
interpolation='bilinear')
self.nasLon = Array(source=LON.astype(float),
native_coordinates=self.coord_src,
interpolation='bilinear')
self.nasTime = Array(source=TIME.astype(float),
native_coordinates=self.coord_src,
interpolation='bilinear')
def test_raster_to_raster(self):
coord_dst = Coordinates(
[clinspace(5., 40., 50),
clinspace(-68., -66., 100)],
dims=['lat', 'lon'])
oLat = self.nasLat.eval(coord_dst)
oLon = self.nasLon.eval(coord_dst)
LON, LAT = np.meshgrid(coord_dst['lon'].coordinates,
coord_dst['lat'].coordinates)
np.testing.assert_array_almost_equal(oLat.data[..., 0], LAT)
np.testing.assert_array_almost_equal(oLon.data[..., 0], LON)
def test_native_coordinates(self):
"""test that native coordinates get defined"""
node = Array(source=self.data)
with pytest.raises(NotImplementedError):
node.get_native_coordinates()
node = Array(source=self.data, native_coordinates=self.coordinates)
assert node.native_coordinates
node = Array(source=self.data, native_coordinates=self.coordinates)
native_coordinates = node.native_coordinates
get_native_coordinates = node.get_native_coordinates()
assert native_coordinates
assert get_native_coordinates
assert native_coordinates == get_native_coordinates
def test_heterogeous_sources_composited(self):
anative = podpac.Coordinates(
[podpac.clinspace((0, 1), (1, 2), size=3)], dims=['lat_lon'])
bnative = podpac.Coordinates(
[podpac.clinspace(-2, 3, 3),
podpac.clinspace(-1, 4, 3)],
dims=['lat', 'lon'])
a = Array(source=np.random.rand(3), native_coordinates=anative)
b = Array(source=np.random.rand(3, 3) + 2, native_coordinates=bnative)
c = OrderedCompositor(sources=np.array([a, b]),
interpolation='nearest')
coords = podpac.Coordinates(
[podpac.clinspace(-3, 4, 32),
podpac.clinspace(-2, 5, 32)],
dims=['lat', 'lon'])
o = c.eval(coords)
# Check that both data sources are being used in the interpolation
mask = o.data >= 2
assert mask.sum() > 0
mask = o.data <= 1
assert mask.sum() > 0
def test_put_something_new_into_existing_file():
lat = np.random.rand(3)
lon = np.random.rand(4)
dummy_coords = Coordinates([lat, lon], ['lat', 'lon'])
dummy_node = Array(source=np.random.random_sample(dummy_coords.shape),
native_coordinates=dummy_coords)
dummy_node_din = np.random.rand(6, 7, 8)
dummy_node_key = "key"
disk_stores = [c for c in cache._cache_stores if type(c) is DiskCacheStore]
for coord_f in coord_funcs:
for node_f in node_funcs:
for data_f in data_funcs:
c1, c2 = coord_f(), coord_f()
n1, n2 = node_f(), node_f()
din = data_f()
k = "key"
assert not cache.has(
node=n1, key=k, coordinates=c1, mode='all')
for store in disk_stores:
store.make_cache_dir(node=n1)
path = store.cache_path(node=n1, key=k, coordinates=c1)
listing = CacheListing(node=dummy_node,
key=dummy_node_key,
coordinates=dummy_coords,
data=dummy_node_din)
CachePickleContainer(listings=[listing]).save(path)
assert not cache.has(
node=n1, key=k, coordinates=c1, mode='all')
cache.put(node=n1,
data=din,
key=k,
coordinates=c1,
mode='all',
update=False)
assert cache.has(node=n1, key=k, coordinates=c1, mode='all')
dout = cache.get(node=n1, key=k, coordinates=c1, mode='all')
assert (din == dout).all()
dout = cache.get(node=n2, key=k, coordinates=c2, mode='all')
assert (din == dout).all()
cache.rem(node=n1, key=k, coordinates=c1, mode='all')
assert not cache.has(
node=n1, key=k, coordinates=c1, mode='all')
for store in disk_stores:
path = store.cache_path(node=n1, key=k, coordinates=c1)
assert os.path.exists(path)
c = CachePickleContainer.load(path)
listing = CacheListing(node=dummy_node,
key=dummy_node_key,
coordinates=dummy_coords,
data=dummy_node_din)
assert c.has(listing)
cache.rem(node='*', key='*', coordinates='*', mode='all')
def test_put_and_get_array_datasource_output():
lat = [0, 1, 2]
lon = [10, 20, 30, 40]
dates = ['2018-01-01', '2018-01-02']
native_coordinates = Coordinates([lat, lon, dates], ['lat', 'lon', 'time'])
source = np.zeros(native_coordinates.shape)
array_data_source = Array(source=source,
native_coordinates=native_coordinates)
output = array_data_source.eval(native_coordinates)
cache.put(node=array_data_source,
data=output,
key='output',
coordinates=native_coordinates,
mode='all',
update=False)
cached_output = cache.get(node=array_data_source,
key='output',
coordinates=native_coordinates,
mode='all')
assert (cached_output == output).all()
cache.rem(node='*', key='*', coordinates='*',
mode='all') # clear the cache stores
def test_compositor_implemented_functions(self):
acoords = podpac.Coordinates(
[podpac.clinspace(0, 1, 11),
podpac.clinspace(0, 1, 11)],
dims=['lat', 'lon'])
bcoords = podpac.Coordinates(
[podpac.clinspace(2, 3, 10),
podpac.clinspace(2, 3, 10)],
dims=['lat', 'lon'])
scoords = podpac.Coordinates([[(0.5, 2.5), (0.5, 2.5)]],
dims=['lat_lon'])
a = Array(source=np.random.random(acoords.shape),
native_coordinates=acoords)
b = Array(source=-np.random.random(bcoords.shape),
native_coordinates=bcoords)
composited = OrderedCompositor(sources=np.array([a, b]),
cache_native_coordinates=True,
source_coordinates=scoords,
interpolation='nearest')
c = podpac.Coordinates([0.5, 0.5], dims=['lat', 'lon'])
o = composited.eval(c)
np.testing.assert_array_equal(o.data, a.source[5, 5])
def test_put_and_get_with_different_instances_of_same_key_objects_array_datasource_output(
):
lat = [0, 1, 2]
lon = [10, 20, 30, 40]
dates = ['2018-01-01', '2018-01-02']
# create data source node and coordinates for put operation
native_coordinates_put = Coordinates([lat, lon, dates],
['lat', 'lon', 'time'])
source_put = np.zeros(native_coordinates_put.shape)
array_data_source_put = Array(source=source_put,
native_coordinates=native_coordinates_put)
output = array_data_source_put.eval(native_coordinates_put)
cache.put(node=array_data_source_put,
data=output,
key='output',
coordinates=native_coordinates_put,
mode='all',
update=False)
# create equivalent (but new objects) data source node and coordinates for get operation
native_coordinates_get = Coordinates([lat, lon, dates],
['lat', 'lon', 'time'])
source_get = np.zeros(native_coordinates_get.shape)
array_data_source_get = Array(source=source_get,
native_coordinates=native_coordinates_get)
cached_output = cache.get(node=array_data_source_get,
key='output',
coordinates=native_coordinates_get,
mode='all')
assert (cached_output == output).all()
cache.rem(node='*', key='*', coordinates='*',
mode='all') # clear the cache stores
def test_two_different_nodes_put_and_one_node_removes_all():
lat = np.random.rand(3)
lon = np.random.rand(4)
coords = Coordinates([lat, lon], ['lat', 'lon'])
persistent_node = Array(source=np.random.random_sample(coords.shape),
native_coordinates=coords)
persistent_node_din = np.random.rand(6, 7, 8)
persistent_node_key = "key"
cache.put(node=persistent_node,
data=persistent_node_din,
key=persistent_node_key,
coordinates=None)
for coord_f in coord_funcs:
for node_f in node_funcs:
for data_f in data_funcs:
c1, c2 = coord_f(), coord_f()
n1, n2 = node_f(), node_f()
d1 = data_f()
d2 = data_f() + np.pi
k1 = "key"
k2 = k1 + "2"
cache.put(node=n1,
data=d1,
key=k1,
coordinates=c1,
mode='all',
update=False)
cache.put(node=n1,
data=d1,
key=k2,
coordinates=c1,
mode='all',
update=False)
assert cache.has(node=n2, key=k1, coordinates=c2, mode='all')
assert cache.has(node=n2, key=k2, coordinates=c2, mode='all')
cache.rem(node=n2, key='*', coordinates='*', mode='all')
assert not cache.has(
node=n1, key=k1, coordinates=c1, mode='all')
assert not cache.has(
node=n1, key=k2, coordinates=c1, mode='all')
with pytest.raises(CacheException):
cache.get(node=n1, key=k2, coordinates=c1, mode='all')
with pytest.raises(CacheException):
cache.get(node=n1, key=k1, coordinates=c1, mode='all')
assert cache.has(node=persistent_node,
key=persistent_node_key,
coordinates=None)
cache.rem(node='*', key='*', coordinates='*', mode='all')
def test_definition(self):
node = Array(source=self.data)
pipeline = podpac.pipeline.Pipeline(definition=node.definition)
np.testing.assert_array_equal(pipeline.node.source, self.data)
def test_base_definition(self):
node = Array(source=self.data)
d = node.base_definition
source = np.array(d['source'])
np.testing.assert_array_equal(source, self.data)
def test_get_data(self):
"""test get data from reprojected source"""
datanode = Array(source=self.data, native_coordinates=self.native_coordinates)
node = ReprojectedSource(source=datanode, reprojected_coordinates=datanode.native_coordinates)
output = node.eval(node.native_coordinates)
assert isinstance(output, UnitsDataArray)
