def test_iterates_incomplete_dataset():
mds = utils.make_mock()
mds.dataset[:2, :, :, :] = 1
mds.dataset[2, 0:5, :, ] = 1
key_paths = ["incomplete"]
f = {"incomplete": mds}
kf = KeyFollower(f, key_paths, timeout=0.1)
kf.check_datasets()
current_key = 0
for key in kf:
current_key += 1
assert current_key == 25
def test_multiple_keys_from_node():
mds = utils.make_mock()
mds.dataset[:, :, :, :] = 1
mdsi = utils.make_mock()
mdsi.dataset[:2, :, :, :] = 1
mdsi.dataset[2, 0:5, :, ] = 1
key_paths = ["keys"]
f = {"keys": ["a", "b"], "keys/a": mds, "keys/b": mdsi}
kf = KeyFollower(f, key_paths, timeout=0.1)
kf.check_datasets()
current_key = 0
for key in kf:
current_key += 1
assert current_key == 25
def test_iterates_complete_dataset():
key_paths = ["complete"]
mds = utils.make_mock()
mds.dataset = mds.dataset + 1
f = {"complete": mds}
kf = KeyFollower(f, key_paths, timeout=0.1)
kf.check_datasets()
assert kf.scan_rank == 2
assert kf.maxshape == [5, 10]
current_key = 0
for key in kf:
current_key += 1
assert current_key == 50
def test_iterates_incomplete_dataset():
key_paths = ["keys"]
f = {"keys": {"incomplete": Dataset.incomplete_dataset()}}
kf = KeyFollower.Follower(f, key_paths, timeout=0.1)
current_key = 0
for key in kf:
current_key += 1
assert current_key == 25
def test_update_changes_shape():
key_paths = ["keys"]
f = {"keys": {"small_incomplete": Dataset.small_incomplete_dataset()}}
kf = KeyFollower.Follower(f, key_paths, timeout=0.1)
current_key = 0
for i in range(5):
next(kf)
current_key += 1
kf.hdf5_file = {"keys": {"small_incomplete": Dataset.complete_dataset()}}
for key in kf:
current_key += 1
assert current_key == 50
def test_index_independent_of_key_value():
key_paths = ['keys']
f = {
"keys": {
"small_incomplete": Dataset.complete_dataset_random_integers()
}
}
current_key = 0
kf = KeyFollower.Follower(f, key_paths, timeout=0.1)
for key in kf:
assert current_key == key
current_key += 1
def test_iterates_snake_scan():
mds = utils.make_mock()
mds.dataset[:2, :, :, :] = 1
mds.dataset[2, 1:, :, :] = 1
key_paths = ["incomplete"]
f = {"incomplete": mds}
kf = KeyFollower(f, key_paths, timeout=0.1)
current_key = 0
for key in kf:
current_key += 1
assert current_key == 20
def test_complete_keys(tmp_path):
f = str(tmp_path / "f.h5")
with h5py.File(f, "w") as fh:
fh.create_dataset("complete", data=np.ones((10)), maxshape=(10, ))
d1 = np.ones((10))
fh.create_dataset("complete2", data=d1, maxshape=(10, ))
ps = ["/"]
with h5py.File(f, "r") as fh:
kf = KeyFollower(fh, ps, timeout=0.1)
kf.check_datasets()
assert kf.scan_rank == 1
current_key = 0
for key in kf:
current_key += 1
assert current_key == 10
def test_reads_updates():
key_paths = ["keys"]
f = {"keys": {"incomplete": Dataset.incomplete_dataset()}}
kf = KeyFollower.Follower(f, key_paths, timeout=0.1)
current_key = 0
for i in range(5):
next(kf)
current_key += 1
kf.hdf5_file = {"keys": {"updating": Dataset.complete_dataset()}}
for key in kf:
current_key += 1
assert current_key == 50
def test_reads_updates():
mds = utils.make_mock()
mds.dataset.reshape((-1))[:26] = 1
key_paths = ["incomplete"]
f = {"incomplete": mds}
kf = KeyFollower(f, key_paths, timeout=0.1)
current_key = 0
for key in kf:
current_key += 1
if current_key == 25:
mds.dataset[...] = 1
assert current_key == 50
def test_update_changes_shape():
mds = utils.make_mock(shape=[2, 10, 1, 1])
mds.dataset[...] = 1
key_paths = ["incomplete"]
f = {"incomplete": mds}
kf = KeyFollower(f, key_paths, timeout=0.1)
current_key = 0
for key in kf:
current_key += 1
if current_key == 20:
mds.dataset.resize((5, 10, 1, 1), refcheck=False)
mds.dataset[...] = 1
assert current_key == 50
def test_first_frame_jagged():
key_paths = ["k1", "k2", "k3"]
k1 = utils.make_mock([2])
k2 = utils.make_mock([2])
k3 = utils.make_mock([1], maxshape=[2])
f = {"k1": k1, "k2": k2, "k3": k3}
kf = KeyFollower(f, key_paths, timeout=0.1)
kf.check_datasets()
assert kf.maxshape == [2]
assert kf.scan_rank == 1
current_key = -1
for key in kf:
current_key += 1
assert current_key == -1
kf.reset()
k1.dataset[0] = 1
k2.dataset[0] = 1
current_key = -1
for key in kf:
current_key += 1
assert current_key == -1
kf.reset()
k3.dataset[0] = 1
current_key = -1
for key in kf:
current_key += 1
assert current_key == 0