def setup_method(self, method): self.coord_src = podpac.Coordinates([ podpac.clinspace(45, 0, 16), podpac.clinspace(-70, -65, 16), podpac.clinspace(0, 1, 2) ], dims=['lat', 'lon', 'time']) LON, LAT, TIME = np.meshgrid(self.coord_src['lon'].coordinates, self.coord_src['lat'].coordinates, self.coord_src['time'].coordinates) self.latSource = LAT self.lonSource = LON self.timeSource = TIME self.nasLat = Array(source=LAT.astype(float), native_coordinates=self.coord_src, interpolation='bilinear') self.nasLon = Array(source=LON.astype(float), native_coordinates=self.coord_src, interpolation='bilinear') self.nasTime = Array(source=TIME.astype(float), native_coordinates=self.coord_src, interpolation='bilinear') self.sources = np.array([self.nasLat, self.nasLon, self.nasTime])
def test_source_trait(self): """ must be an ndarray """ node = Array(source=self.data, native_coordinates=self.coordinates) # list is coercable to array node = Array(source=[0, 1, 1], native_coordinates=self.coordinates)
def test_get_data(self): """ defined get_data function""" source = self.data node = Array(source=source, native_coordinates=self.coordinates) output = node.eval(self.coordinates) assert isinstance(output, UnitsDataArray) assert output.values[0, 0] == source[0, 0] assert output.values[4, 5] == source[4, 5]
def setup_module(): global coords, source, data coords = podpac.Coordinates( [podpac.clinspace(0, 1, 10), podpac.clinspace(0, 1, 10), podpac.crange('2018-01-01', '2018-01-10', '1,D')], dims=['lat', 'lon', 'time']) a = np.random.random(coords.shape) a[3, 0, 0] = np.nan a[0, 3, 0] = np.nan a[0, 0, 3] = np.nan source = Array(source=a, native_coordinates=coords) data = source.eval(coords)
def test_put_and_remove_array_datasource_numpy_array(): lat = [0, 1, 2] lon = [10, 20, 30, 40] dates = ['2018-01-01', '2018-01-02'] native_coordinates = Coordinates([lat, lon, dates], ['lat', 'lon', 'time']) source = np.zeros(native_coordinates.shape) array_data_source = Array(source=source, native_coordinates=native_coordinates) put_data = np.zeros(native_coordinates.shape) cache.put(node=array_data_source, data=put_data, key='key', coordinates=native_coordinates, mode='all', update=False) cached_data = cache.get(node=array_data_source, key='key', coordinates=native_coordinates, mode='all') assert (cached_data == put_data).all() cache.rem(node=array_data_source, key='key', coordinates=native_coordinates, mode='all') assert not cache.has(node=array_data_source, key='key', coordinates=native_coordinates, mode='all') with pytest.raises(CacheException): cache.get(node=array_data_source, key='key', coordinates=native_coordinates, mode='all') cache.rem(node='*', key='*', coordinates='*', mode='all') # clear the cache stores
def make_array_data_source(coords_func=None, data_func=None): if data_func is None: data_func = np.zeros if coords_func is None: coords_func = make_lat_lon_time_grid_coords coords = coords_func() return Array(source=data_func(coords.shape), native_coordinates=coords)
def test_array(self): """Test array data source""" from podpac.core.data.types import Array arr = np.random.rand(16, 11) lat = np.random.rand(16) lon = np.random.rand(16) coord = Coordinate(lat_lon=(lat, lon), time=(0, 10, 11), order=['lat_lon', 'time']) node = Array(source=arr, native_coordinates=coord) coordg = Coordinate(lat=(0, 1, 8), lon=(0, 1, 8), order=('lat', 'lon')) coordt = Coordinate(time=(3, 5, 2)) node.eval(coordt) node.eval(coordg)
class TestBasicInterpolation(object): """ Test interpolation methods""" def setup_method(self, method): self.coord_src = Coordinates([ clinspace(45, 0, 16), clinspace(-70., -65., 16), clinspace(0, 1, 2) ], dims=['lat', 'lon', 'time']) LON, LAT, TIME = np.meshgrid(self.coord_src['lon'].coordinates, self.coord_src['lat'].coordinates, self.coord_src['time'].coordinates) self.latSource = LAT self.lonSource = LON self.timeSource = TIME self.nasLat = Array(source=LAT.astype(float), native_coordinates=self.coord_src, interpolation='bilinear') self.nasLon = Array(source=LON.astype(float), native_coordinates=self.coord_src, interpolation='bilinear') self.nasTime = Array(source=TIME.astype(float), native_coordinates=self.coord_src, interpolation='bilinear') def test_raster_to_raster(self): coord_dst = Coordinates( [clinspace(5., 40., 50), clinspace(-68., -66., 100)], dims=['lat', 'lon']) oLat = self.nasLat.eval(coord_dst) oLon = self.nasLon.eval(coord_dst) LON, LAT = np.meshgrid(coord_dst['lon'].coordinates, coord_dst['lat'].coordinates) np.testing.assert_array_almost_equal(oLat.data[..., 0], LAT) np.testing.assert_array_almost_equal(oLon.data[..., 0], LON)
def test_native_coordinates(self): """test that native coordinates get defined""" node = Array(source=self.data) with pytest.raises(NotImplementedError): node.get_native_coordinates() node = Array(source=self.data, native_coordinates=self.coordinates) assert node.native_coordinates node = Array(source=self.data, native_coordinates=self.coordinates) native_coordinates = node.native_coordinates get_native_coordinates = node.get_native_coordinates() assert native_coordinates assert get_native_coordinates assert native_coordinates == get_native_coordinates
def test_heterogeous_sources_composited(self): anative = podpac.Coordinates( [podpac.clinspace((0, 1), (1, 2), size=3)], dims=['lat_lon']) bnative = podpac.Coordinates( [podpac.clinspace(-2, 3, 3), podpac.clinspace(-1, 4, 3)], dims=['lat', 'lon']) a = Array(source=np.random.rand(3), native_coordinates=anative) b = Array(source=np.random.rand(3, 3) + 2, native_coordinates=bnative) c = OrderedCompositor(sources=np.array([a, b]), interpolation='nearest') coords = podpac.Coordinates( [podpac.clinspace(-3, 4, 32), podpac.clinspace(-2, 5, 32)], dims=['lat', 'lon']) o = c.eval(coords) # Check that both data sources are being used in the interpolation mask = o.data >= 2 assert mask.sum() > 0 mask = o.data <= 1 assert mask.sum() > 0
def test_put_something_new_into_existing_file(): lat = np.random.rand(3) lon = np.random.rand(4) dummy_coords = Coordinates([lat, lon], ['lat', 'lon']) dummy_node = Array(source=np.random.random_sample(dummy_coords.shape), native_coordinates=dummy_coords) dummy_node_din = np.random.rand(6, 7, 8) dummy_node_key = "key" disk_stores = [c for c in cache._cache_stores if type(c) is DiskCacheStore] for coord_f in coord_funcs: for node_f in node_funcs: for data_f in data_funcs: c1, c2 = coord_f(), coord_f() n1, n2 = node_f(), node_f() din = data_f() k = "key" assert not cache.has( node=n1, key=k, coordinates=c1, mode='all') for store in disk_stores: store.make_cache_dir(node=n1) path = store.cache_path(node=n1, key=k, coordinates=c1) listing = CacheListing(node=dummy_node, key=dummy_node_key, coordinates=dummy_coords, data=dummy_node_din) CachePickleContainer(listings=[listing]).save(path) assert not cache.has( node=n1, key=k, coordinates=c1, mode='all') cache.put(node=n1, data=din, key=k, coordinates=c1, mode='all', update=False) assert cache.has(node=n1, key=k, coordinates=c1, mode='all') dout = cache.get(node=n1, key=k, coordinates=c1, mode='all') assert (din == dout).all() dout = cache.get(node=n2, key=k, coordinates=c2, mode='all') assert (din == dout).all() cache.rem(node=n1, key=k, coordinates=c1, mode='all') assert not cache.has( node=n1, key=k, coordinates=c1, mode='all') for store in disk_stores: path = store.cache_path(node=n1, key=k, coordinates=c1) assert os.path.exists(path) c = CachePickleContainer.load(path) listing = CacheListing(node=dummy_node, key=dummy_node_key, coordinates=dummy_coords, data=dummy_node_din) assert c.has(listing) cache.rem(node='*', key='*', coordinates='*', mode='all')
def test_put_and_get_array_datasource_output(): lat = [0, 1, 2] lon = [10, 20, 30, 40] dates = ['2018-01-01', '2018-01-02'] native_coordinates = Coordinates([lat, lon, dates], ['lat', 'lon', 'time']) source = np.zeros(native_coordinates.shape) array_data_source = Array(source=source, native_coordinates=native_coordinates) output = array_data_source.eval(native_coordinates) cache.put(node=array_data_source, data=output, key='output', coordinates=native_coordinates, mode='all', update=False) cached_output = cache.get(node=array_data_source, key='output', coordinates=native_coordinates, mode='all') assert (cached_output == output).all() cache.rem(node='*', key='*', coordinates='*', mode='all') # clear the cache stores
def test_compositor_implemented_functions(self): acoords = podpac.Coordinates( [podpac.clinspace(0, 1, 11), podpac.clinspace(0, 1, 11)], dims=['lat', 'lon']) bcoords = podpac.Coordinates( [podpac.clinspace(2, 3, 10), podpac.clinspace(2, 3, 10)], dims=['lat', 'lon']) scoords = podpac.Coordinates([[(0.5, 2.5), (0.5, 2.5)]], dims=['lat_lon']) a = Array(source=np.random.random(acoords.shape), native_coordinates=acoords) b = Array(source=-np.random.random(bcoords.shape), native_coordinates=bcoords) composited = OrderedCompositor(sources=np.array([a, b]), cache_native_coordinates=True, source_coordinates=scoords, interpolation='nearest') c = podpac.Coordinates([0.5, 0.5], dims=['lat', 'lon']) o = composited.eval(c) np.testing.assert_array_equal(o.data, a.source[5, 5])
def test_put_and_get_with_different_instances_of_same_key_objects_array_datasource_output( ): lat = [0, 1, 2] lon = [10, 20, 30, 40] dates = ['2018-01-01', '2018-01-02'] # create data source node and coordinates for put operation native_coordinates_put = Coordinates([lat, lon, dates], ['lat', 'lon', 'time']) source_put = np.zeros(native_coordinates_put.shape) array_data_source_put = Array(source=source_put, native_coordinates=native_coordinates_put) output = array_data_source_put.eval(native_coordinates_put) cache.put(node=array_data_source_put, data=output, key='output', coordinates=native_coordinates_put, mode='all', update=False) # create equivalent (but new objects) data source node and coordinates for get operation native_coordinates_get = Coordinates([lat, lon, dates], ['lat', 'lon', 'time']) source_get = np.zeros(native_coordinates_get.shape) array_data_source_get = Array(source=source_get, native_coordinates=native_coordinates_get) cached_output = cache.get(node=array_data_source_get, key='output', coordinates=native_coordinates_get, mode='all') assert (cached_output == output).all() cache.rem(node='*', key='*', coordinates='*', mode='all') # clear the cache stores
def test_two_different_nodes_put_and_one_node_removes_all(): lat = np.random.rand(3) lon = np.random.rand(4) coords = Coordinates([lat, lon], ['lat', 'lon']) persistent_node = Array(source=np.random.random_sample(coords.shape), native_coordinates=coords) persistent_node_din = np.random.rand(6, 7, 8) persistent_node_key = "key" cache.put(node=persistent_node, data=persistent_node_din, key=persistent_node_key, coordinates=None) for coord_f in coord_funcs: for node_f in node_funcs: for data_f in data_funcs: c1, c2 = coord_f(), coord_f() n1, n2 = node_f(), node_f() d1 = data_f() d2 = data_f() + np.pi k1 = "key" k2 = k1 + "2" cache.put(node=n1, data=d1, key=k1, coordinates=c1, mode='all', update=False) cache.put(node=n1, data=d1, key=k2, coordinates=c1, mode='all', update=False) assert cache.has(node=n2, key=k1, coordinates=c2, mode='all') assert cache.has(node=n2, key=k2, coordinates=c2, mode='all') cache.rem(node=n2, key='*', coordinates='*', mode='all') assert not cache.has( node=n1, key=k1, coordinates=c1, mode='all') assert not cache.has( node=n1, key=k2, coordinates=c1, mode='all') with pytest.raises(CacheException): cache.get(node=n1, key=k2, coordinates=c1, mode='all') with pytest.raises(CacheException): cache.get(node=n1, key=k1, coordinates=c1, mode='all') assert cache.has(node=persistent_node, key=persistent_node_key, coordinates=None) cache.rem(node='*', key='*', coordinates='*', mode='all')
def test_definition(self): node = Array(source=self.data) pipeline = podpac.pipeline.Pipeline(definition=node.definition) np.testing.assert_array_equal(pipeline.node.source, self.data)
def test_base_definition(self): node = Array(source=self.data) d = node.base_definition source = np.array(d['source']) np.testing.assert_array_equal(source, self.data)
def test_get_data(self): """test get data from reprojected source""" datanode = Array(source=self.data, native_coordinates=self.native_coordinates) node = ReprojectedSource(source=datanode, reprojected_coordinates=datanode.native_coordinates) output = node.eval(node.native_coordinates) assert isinstance(output, UnitsDataArray)