def test_parallel_multi_thread_compute_fill_output(self):
node = CoordData(coord_name="time")
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node_p = Parallel(source=node, number_of_workers=2, chunks={"time": 2})
o = node.eval(coords)
o_p = node_p.eval(coords)
np.testing.assert_array_equal(o, o_p)
def test_parallel_process_zarr_async_starti(self):
# Can't use tempfile.TemporaryDirectory because multiple processess need access to dir
tmpdir = os.path.join(tempfile.gettempdir(),
"test_parallel_process_zarr_async_starti.zarr")
node = Process(source=CoordData(coord_name="time")) # , block=False)
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node_p = ParallelAsyncOutputZarr(source=node,
number_of_workers=5,
chunks={"time": 2},
fill_output=False,
zarr_file=tmpdir,
start_i=1)
o_zarr = node_p.eval(coords)
# print(o_zarr.info)
time.sleep(0.01)
np.testing.assert_array_equal([np.nan, np.nan, 3, 4, 5],
o_zarr["data"][:])
node_p = ParallelAsyncOutputZarr(source=node,
number_of_workers=5,
chunks={"time": 2},
fill_output=False,
zarr_file=tmpdir,
start_i=0)
o_zarr = node_p.eval(coords)
np.testing.assert_array_equal([1, 2, 3, 4, 5], o_zarr["data"][:])
shutil.rmtree(tmpdir)
def test_parallel_process_async(self):
node = Process(source=CoordData(coord_name="time")) # , block=False)
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node_p = ParallelAsync(source=node,
number_of_workers=2,
chunks={"time": 2},
fill_output=False)
node_p.eval(coords)
time.sleep(0.1)
def test_parallel_process(self):
node = Process(source=CoordData(coord_name="time"))
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node_p = Parallel(source=node, number_of_workers=2, chunks={"time": 2})
o = node.eval(coords)
o_p = o.copy()
o_p[:] = np.nan
node_p.eval(coords, output=o_p)
time.sleep(0.1)
np.testing.assert_array_equal(o, o_p)
def test_CoordData(self):
coords = podpac.Coordinates([[0, 1, 2], [0, 1, 2, 3, 4]],
dims=["lat", "lon"])
node = CoordData(coord_name="lat")
np.testing.assert_array_equal(node.eval(coords),
coords["lat"].coordinates)
node = CoordData(coord_name="lon")
np.testing.assert_array_equal(node.eval(coords),
coords["lon"].coordinates)
def test_invalid_dimension(self):
coords = podpac.Coordinates([[0, 1, 2], [0, 1, 2, 3, 4]],
dims=["lat", "lon"])
node = CoordData(coord_name="time")
with pytest.raises(ValueError):
node.eval(coords)