def test_read_random_feat_permissive():
r = SDRF.create(feat_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0])
key1.append('unk')
# binary
r = RDRF.create('p,' + feat_scp_b, path_prefix=input_prefix)
data2 = r.read(key1)
for d1, d2 in zip(data1, data2[:-1]):
assert_allclose(d1, d2)
assert data2[-1].shape == (0, )
# text
r = RDRF.create('p,' + feat_scp_t, path_prefix=input_prefix)
data2 = r.read(key1)
for d1, d2 in zip(data1, data2[:-1]):
assert_allclose(d1, d2, rtol=1e-5)
assert data2[-1].shape == (0, )
def test_read_random_scp_feat():
r = SDRF.create(feat_scp_ho)
key1, data1 = r.read(0)
r = RDRF.create(feat_scp_ho)
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
def test_read_random_file_vec():
r = SDRF.create(vec_h5_ho)
key1, data1 = r.read(0)
r = RDRF.create(vec_h5_ho)
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
def test_read_shapes_compress_random_file_feat():
r = SDRF.create(feat_scp_ho)
key1, data1 = r.read_shapes(0)
for i, cm in enumerate(compression_methods):
r = RDRF.create(feat_scp_hco[i])
data2 = r.read_shapes(key1)
for d1, d2 in zip(data1, data2):
assert d1 == d2, 'Wrong shape for method %s' % cm
def test_with_read_random_scp_feat():
# without with
r = SDRF.create(feat_scp_ho)
key1, data1 = r.read(0)
# with with
with RDRF.create(feat_scp_ho) as r:
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
def test_read_random_scp_feat_permissive():
r = SDRF.create(feat_scp_ho)
key1, data1 = r.read(0)
key1.append('unk')
data1.append(np.array([]))
r = RDRF.create('p,' + feat_scp_ho)
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
def test_read_random_vec():
r = SDRF.create(vec_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0])
# binary
r = RDRF.create(vec_scp_b, path_prefix=input_prefix)
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
# text
r = RDRF.create(vec_scp_t, path_prefix=input_prefix)
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2, rtol=1e-5)
def test_read_num_rows_random_scp_feat():
r = SDRF.create(feat_scp_ho)
key1 = []
data1 = []
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0].shape[0])
r = RDRF.create(feat_scp_ho)
data2 = r.read_num_rows(key1)
for d1, d2 in zip(data1, data2):
assert d1 == d2
def test_read_shapes_random_file_feat():
r = SDRF.create(feat_h5_ho[0])
key1 = []
data1 = []
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0].shape)
r = RDRF.create(feat_h5_ho[0])
data2 = r.read_shapes(key1)
for d1, d2 in zip(data1, data2):
assert d1 == d2
def test_read_compress_random_scp_feat():
for i, cm in enumerate(compression_methods):
r = SDRF.create(feat_scp_hco[i])
key1, data1 = r.read(0)
r = RDRF.create(feat_scp_hco[i])
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1,
d2,
rtol=1e-5,
atol=1e-4,
err_msg=('Read compression %s failed' % cm))
def test_read_dims_random_feat():
r = SDRF.create(feat_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0].shape[1])
r = RDRF.create(feat_scp_b, path_prefix=input_prefix)
data2 = r.read_dims(key1)
for d1, d2 in zip(data1, data2):
assert d1 == d2
def test_with_read_random_feat():
r = SDRF.create(feat_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0])
# binary with
with RDRF.create(feat_scp_b, path_prefix=input_prefix) as r:
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
def test_read_shapes_compress_random_feat():
r = SDRF.create(feat_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0].shape)
for i, cm in enumerate(compression_methods):
r = RDRF.create(feat_scp_c[i], path_prefix=input_prefix)
data2 = r.read_shapes(key1)
for d1, d2 in zip(data1, data2):
assert d1 == d2, 'Wrong shape for method %s' % cm
def test_read_compress_random_feat():
for i, cm in enumerate(compression_methods):
# scp uncompressed binary
r = SDRF.create(feat_scp_uc[i], path_prefix=input_prefix)
key1, data1 = r.read(0)
# scp compressed
r = RDRF.create(feat_scp_c[i], path_prefix=input_prefix)
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1,
d2,
rtol=1e-5,
atol=1e-4,
err_msg=('Read compression %s failed' % cm))
def test_read_range_shapes_compress_random_scp_feat():
r = SDRF.create(feat_scp_ho)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0][i:i + 50].shape)
i += 1
for k, cm in enumerate(compression_methods):
r = RDRF.create(feat_range_hco[k])
data2 = r.read_shapes(key1)
for d1, d2 in zip(data1, data2):
assert d1 == d2, 'Wrong shape for method %s' % cm
def test_read_shapes_random_vec():
r = SDRF.create(vec_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0].shape)
r = RDRF.create(vec_scp_b, path_prefix=input_prefix)
data2 = r.read_shapes(key1)
for d1, d2 in zip(data1, data2):
assert d1 == d2
assert len(d1) == 1
assert len(d2) == 1
def test_read_range_shapes_random_scp_feat():
r = SDRF.create(feat_scp_ho)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0][i:i + 50].shape)
i += 1
r = RDRF.create(feat_range_ho)
data2 = r.read_shapes(key1)
for d1, d2 in zip(data1, data2):
assert d1 == d2
def test_read_range2_random_file_feat():
r = SDRF.create(feat_h5_ho[0])
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0][i:i + 10])
i += 1
r = RDRF.create(feat_h5_ho[0])
row_offset = [i for i in xrange(len(key1))]
data2 = r.read(key1, row_offset=row_offset, num_rows=10)
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
def test_read_range_shapes_random_feat():
r = SDRF.create(feat_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0][i:i + 50].shape)
i += 1
# binary
r = RDRF.create(feat_range_b, path_prefix=input_prefix)
data2 = r.read_shapes(key1)
for d1, d2 in zip(data1, data2):
assert d1 == d2
def test_read_range_random_scp_feat():
r = SDRF.create(feat_scp_ho)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0][i:i + 50])
i += 1
# binary
r = RDRF.create(feat_range_ho)
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
def test_read_rangex2_random_feat():
r = SDRF.create(feat_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0][2 * i:2 * i + 10])
i += 1
# binary
r = RDRF.create(feat_range_b, path_prefix=input_prefix)
row_offset = [i for i in xrange(len(key1))]
data2 = r.read(key1, row_offset=row_offset, num_rows=10)
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
def test_read_squeeze_random_scp_feat():
r = SDRF.create(feat_scp_ho)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0][i:i + 10])
i += 1
r = RDRF.create(feat_scp_ho)
row_offset = [i for i in xrange(len(key1))]
data2 = r.read(key1, squeeze=True, row_offset=row_offset, num_rows=10)
assert isinstance(data2, np.ndarray)
assert data2.ndim == 3
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
def test_read_squeeze_random_vec():
r = SDRF.create(vec_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0])
i += 1
# binary
r = RDRF.create(vec_scp_b, path_prefix=input_prefix)
row_offset = [i for i in xrange(len(key1))]
data2 = r.read(key1, squeeze=True)
assert isinstance(data2, np.ndarray)
assert data2.ndim == 2
for d1, d2 in zip(data1, data2):
assert_allclose(d1, d2)
def test_read_squeeze_random_scp_feat_permissive():
r = SDRF.create(feat_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0][i:i + 10])
i += 1
key1.append('unk')
r = RDRF.create('p,' + feat_scp_ho)
row_offset = [i for i in xrange(len(key1))]
data2 = r.read(key1, squeeze=True, row_offset=row_offset, num_rows=10)
assert isinstance(data2, np.ndarray)
assert data2.ndim == 3
for d1, d2 in zip(data1, data2[:-1]):
assert_allclose(d1, d2)
assert_allclose(data2[-1], np.zeros(data2[0].shape))
def test_read_squeeze_random_vec_permissive():
r = SDRF.create(vec_scp_b, path_prefix=input_prefix)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0])
i += 1
# binary
key1.append('unk')
r = RDRF.create('p,' + vec_scp_b, path_prefix=input_prefix)
data2 = r.read(key1, squeeze=True)
assert isinstance(data2, np.ndarray)
assert data2.ndim == 2
for d1, d2 in zip(data1, data2[:-1]):
assert_allclose(d1, d2)
assert_allclose(data2[-1], np.zeros(data2[0].shape))
def test_read_range2_compress_random_feat():
for k, cm in enumerate(compression_methods):
r = SDRF.create(feat_scp_uc[k], path_prefix=input_prefix)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0][i:i + 10])
i += 1
r = RDRF.create(feat_scp_c[k], path_prefix=input_prefix)
row_offset = [i for i in xrange(len(key1))]
data2 = r.read(key1, row_offset=row_offset, num_rows=10)
for d1, d2 in zip(data1, data2):
assert_allclose(d1,
d2,
rtol=1e-5,
atol=1e-4,
err_msg=('Read compression %s failed' % cm))
def test_read_range_compress_random_feat():
for k, cm in enumerate(compression_methods):
# scp uncompressed binary
r = SDRF.create(feat_scp_uc[k], path_prefix=input_prefix)
key1 = []
data1 = []
i = 0
while not r.eof():
key_i, data_i = r.read(1)
key1.append(key_i[0])
data1.append(data_i[0][i:i + 50])
i += 1
# scp compressed binary
r = RDRF.create(feat_range_c[k], path_prefix=input_prefix)
data2 = r.read(key1)
for d1, d2 in zip(data1, data2):
assert_allclose(d1,
d2,
rtol=1e-5,
atol=1e-4,
err_msg=('Read compression %s failed' % cm))
