def test_ESPnetDataset_pipe_wav(pipe_wav):
dataset = IterableESPnetDataset(
path_name_type_list=[(pipe_wav, "data1", "pipe_wav")], preprocess=preprocess,
)
for key, data in dataset:
if key == "a":
assert data["data1"].shape == (160000,)
if key == "b":
assert data["data1"].shape == (80000,)
def test_ESPnetDataset_feats_scp(feats_scp,):
dataset = IterableESPnetDataset(
path_name_type_list=[(feats_scp, "data2", "kaldi_ark")], preprocess=preprocess,
)
for key, data in dataset:
if key == "a":
assert data["data2"].shape == (100, 80,)
if key == "b":
assert data["data2"].shape == (150, 80,)
def test_ESPnetDataset_csv_int(csv_int):
dataset = IterableESPnetDataset(
path_name_type_list=[(csv_int, "data8", "csv_int")], preprocess=preprocess,
)
for key, data in dataset:
if key == "a":
assert tuple(data["data8"]) == (0, 1, 2)
if key == "b":
assert tuple(data["data8"]) == (2, 3, 4)
def test_ESPnetDataset_csv_float(csv_float):
dataset = IterableESPnetDataset(
path_name_type_list=[(csv_float, "data8", "csv_float")], preprocess=preprocess,
)
for key, data in dataset:
if key == "a":
assert all((data["data8"]) == np.array([1.4, 3.4], dtype=np.float32))
if key == "b":
assert all((data["data8"]) == np.array([0.9, 9.3], dtype=np.float32))
def test_ESPnetDataset_text(text):
dataset = IterableESPnetDataset(
path_name_type_list=[(text, "data7", "text")], preprocess=preprocess,
)
for key, data in dataset:
if key == "a":
assert tuple(data["data7"]) == (0,)
if key == "b":
assert tuple(data["data7"]) == (1,)
def test_ESPnetDataset_h5file_1(h5file_1):
dataset = IterableESPnetDataset(
path_name_type_list=[(h5file_1, "data4", "hdf5")], preprocess=preprocess,
)
for key, data in dataset:
if key == "a":
assert data["data4"].shape == (100, 80,)
if key == "b":
assert data["data4"].shape == (150, 80,)
def test_ESPnetDataset_npy_scp(npy_scp):
dataset = IterableESPnetDataset(
path_name_type_list=[(npy_scp, "data3", "npy")], preprocess=preprocess,
)
for key, data in dataset:
if key == "a":
assert data["data3"].shape == (100, 80,)
if key == "b":
assert data["data3"].shape == (150, 80,)
def test_ESPnetDataset_rand_int(shape_file):
dataset = IterableESPnetDataset(
path_name_type_list=[(shape_file, "data6", "rand_int_0_10")],
preprocess=preprocess,
)
for key, data in dataset:
if key == "a":
assert data["data6"].shape == (100, 80,)
if key == "b":
assert data["data6"].shape == (150, 80,)
def test_ESPnetDataset_sound_scp(sound_scp):
dataset = IterableESPnetDataset(
path_name_type_list=[(sound_scp, "data1", "sound")], preprocess=preprocess,
)
print(dataset)
print(dataset.names())
assert dataset.has_name("data1")
for key, data in dataset:
if key == "a":
assert data["data1"].shape == (160000,)
if key == "b":
assert data["data1"].shape == (80000,)
