def test_mp_results_the_same(self):
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node = Arange()
o_sp = node.eval(coords)
node_mp = Process(source=node)
o_mp = node_mp.eval(coords)
np.testing.assert_array_equal(o_sp.data, o_mp.data)
def test_parallel_process(self):
node = Process(source=CoordData(coord_name="time"))
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node_p = Parallel(source=node, number_of_workers=2, chunks={"time": 2})
o = node.eval(coords)
o_p = o.copy()
o_p[:] = np.nan
node_p.eval(coords, output=o_p)
time.sleep(0.1)
np.testing.assert_array_equal(o, o_p)
def test_mp_results_the_same_set_output(self):
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node = Arange()
o_sp = node.eval(coords)
output = o_sp.copy()
output[:] = np.nan
node_mp = Process(source=node)
o_mp = node_mp.eval(coords, output=output)
np.testing.assert_array_equal(o_sp, output)
def test_parallel_process_zarr_async_starti(self):
# Can't use tempfile.TemporaryDirectory because multiple processess need access to dir
tmpdir = os.path.join(tempfile.gettempdir(),
"test_parallel_process_zarr_async_starti.zarr")
node = Process(source=CoordData(coord_name="time")) # , block=False)
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node_p = ParallelAsyncOutputZarr(source=node,
number_of_workers=5,
chunks={"time": 2},
fill_output=False,
zarr_file=tmpdir,
start_i=1)
o_zarr = node_p.eval(coords)
# print(o_zarr.info)
time.sleep(0.01)
np.testing.assert_array_equal([np.nan, np.nan, 3, 4, 5],
o_zarr["data"][:])
node_p = ParallelAsyncOutputZarr(source=node,
number_of_workers=5,
chunks={"time": 2},
fill_output=False,
zarr_file=tmpdir,
start_i=0)
o_zarr = node_p.eval(coords)
np.testing.assert_array_equal([1, 2, 3, 4, 5], o_zarr["data"][:])
shutil.rmtree(tmpdir)
def test_parallel_process_async(self):
node = Process(source=CoordData(coord_name="time")) # , block=False)
coords = Coordinates([[1, 2, 3, 4, 5]], ["time"])
node_p = ParallelAsync(source=node,
number_of_workers=2,
chunks={"time": 2},
fill_output=False)
node_p.eval(coords)
time.sleep(0.1)
def test_mp_results_outputs(self):
node = Arange(outputs=["a", "b"])
node_mp = Process(source=node)
assert node.outputs == node_mp.outputs