def test_read_random_file_feat(): r = SDRF.create(feat_h5_ho[0]) key1, data1 = r.read(0) r = RDRF.create(feat_h5_ho[0]) data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2)
def test_read_random_scp_vec(): r = SDRF.create(vec_scp_ho) key1, data1 = r.read(0) r = RDRF.create(vec_scp_ho) data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2)
def test_read_shapes_compress_random_file_feat(): r = SDRF.create(feat_scp_ho) key1, data1 = r.read_shapes(0) for i, cm in enumerate(compression_methods): r = RDRF.create(feat_scp_hco[i]) data2 = r.read_shapes(key1) for d1, d2 in zip(data1, data2): assert d1 == d2, 'Wrong shape for method %s' % cm
def test_with_read_random_scp_feat(): # without with r = SDRF.create(feat_scp_ho) key1, data1 = r.read(0) # with with with RDRF.create(feat_scp_ho) as r: data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2)
def test_read_random_scp_feat_permissive(): r = SDRF.create(feat_scp_ho) key1, data1 = r.read(0) key1.append('unk') data1.append(np.array([])) r = RDRF.create('p,' + feat_scp_ho) data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2)
def test_read_random_vec(): r = SDRF.create(vec_scp_b, path_prefix=input_prefix) key1 = [] data1 = [] while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0]) # binary r = RDRF.create(vec_scp_b, path_prefix=input_prefix) data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2) # text r = RDRF.create(vec_scp_t, path_prefix=input_prefix) data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2, rtol=1e-5)
def test_read_num_rows_random_scp_feat(): r = SDRF.create(feat_scp_ho) key1 = [] data1 = [] while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0].shape[0]) r = RDRF.create(feat_scp_ho) data2 = r.read_num_rows(key1) for d1, d2 in zip(data1, data2): assert d1 == d2
def test_read_shapes_random_file_feat(): r = SDRF.create(feat_h5_ho[0]) key1 = [] data1 = [] while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0].shape) r = RDRF.create(feat_h5_ho[0]) data2 = r.read_shapes(key1) for d1, d2 in zip(data1, data2): assert d1 == d2
def test_read_compress_random_scp_feat(): for i, cm in enumerate(compression_methods): r = SDRF.create(feat_scp_hco[i]) key1, data1 = r.read(0) r = RDRF.create(feat_scp_hco[i]) data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2, rtol=1e-5, atol=1e-4, err_msg=('Read compression %s failed' % cm))
def test_read_dims_random_feat(): r = SDRF.create(feat_scp_b, path_prefix=input_prefix) key1 = [] data1 = [] while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0].shape[1]) r = RDRF.create(feat_scp_b, path_prefix=input_prefix) data2 = r.read_dims(key1) for d1, d2 in zip(data1, data2): assert d1 == d2
def test_with_read_random_feat(): r = SDRF.create(feat_scp_b, path_prefix=input_prefix) key1 = [] data1 = [] while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0]) # binary with with RDRF.create(feat_scp_b, path_prefix=input_prefix) as r: data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2)
def test_read_shapes_compress_random_feat(): r = SDRF.create(feat_scp_b, path_prefix=input_prefix) key1 = [] data1 = [] while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0].shape) for i, cm in enumerate(compression_methods): r = RDRF.create(feat_scp_c[i], path_prefix=input_prefix) data2 = r.read_shapes(key1) for d1, d2 in zip(data1, data2): assert d1 == d2, 'Wrong shape for method %s' % cm
def test_read_compress_random_feat(): for i, cm in enumerate(compression_methods): # scp uncompressed binary r = SDRF.create(feat_scp_uc[i], path_prefix=input_prefix) key1, data1 = r.read(0) # scp compressed r = RDRF.create(feat_scp_c[i], path_prefix=input_prefix) data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2, rtol=1e-5, atol=1e-4, err_msg=('Read compression %s failed' % cm))
def test_read_range_shapes_compress_random_scp_feat(): r = SDRF.create(feat_scp_ho) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0][i:i + 50].shape) i += 1 for k, cm in enumerate(compression_methods): r = RDRF.create(feat_range_hco[k]) data2 = r.read_shapes(key1) for d1, d2 in zip(data1, data2): assert d1 == d2, 'Wrong shape for method %s' % cm
def test_read_shapes_random_vec(): r = SDRF.create(vec_scp_b, path_prefix=input_prefix) key1 = [] data1 = [] while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0].shape) r = RDRF.create(vec_scp_b, path_prefix=input_prefix) data2 = r.read_shapes(key1) for d1, d2 in zip(data1, data2): assert d1 == d2 assert len(d1) == 1 assert len(d2) == 1
def test_read_range_shapes_random_scp_feat(): r = SDRF.create(feat_scp_ho) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0][i:i + 50].shape) i += 1 r = RDRF.create(feat_range_ho) data2 = r.read_shapes(key1) for d1, d2 in zip(data1, data2): assert d1 == d2
def test_read_range_random_scp_feat(): r = SDRF.create(feat_scp_ho) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0][i:i + 50]) i += 1 # binary r = RDRF.create(feat_range_ho) data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2)
def test_read_range2_random_file_feat(): r = SDRF.create(feat_h5_ho[0]) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0][i:i + 10]) i += 1 r = RDRF.create(feat_h5_ho[0]) row_offset = [i for i in xrange(len(key1))] data2 = r.read(key1, row_offset=row_offset, num_rows=10) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2)
def test_read_range_shapes_random_feat(): r = SDRF.create(feat_scp_b, path_prefix=input_prefix) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0][i:i + 50].shape) i += 1 # binary r = RDRF.create(feat_range_b, path_prefix=input_prefix) data2 = r.read_shapes(key1) for d1, d2 in zip(data1, data2): assert d1 == d2
def test_read_rangex2_random_feat(): r = SDRF.create(feat_scp_b, path_prefix=input_prefix) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0][2 * i:2 * i + 10]) i += 1 # binary r = RDRF.create(feat_range_b, path_prefix=input_prefix) row_offset = [i for i in xrange(len(key1))] data2 = r.read(key1, row_offset=row_offset, num_rows=10) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2)
def test_read_squeeze_random_scp_feat(): r = SDRF.create(feat_scp_ho) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0][i:i + 10]) i += 1 r = RDRF.create(feat_scp_ho) row_offset = [i for i in xrange(len(key1))] data2 = r.read(key1, squeeze=True, row_offset=row_offset, num_rows=10) assert isinstance(data2, np.ndarray) assert data2.ndim == 3 for d1, d2 in zip(data1, data2): assert_allclose(d1, d2)
def test_read_squeeze_random_vec(): r = SDRF.create(vec_scp_b, path_prefix=input_prefix) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0]) i += 1 # binary r = RDRF.create(vec_scp_b, path_prefix=input_prefix) row_offset = [i for i in xrange(len(key1))] data2 = r.read(key1, squeeze=True) assert isinstance(data2, np.ndarray) assert data2.ndim == 2 for d1, d2 in zip(data1, data2): assert_allclose(d1, d2)
def test_read_squeeze_random_scp_feat_permissive(): r = SDRF.create(feat_scp_b, path_prefix=input_prefix) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0][i:i + 10]) i += 1 key1.append('unk') r = RDRF.create('p,' + feat_scp_ho) row_offset = [i for i in xrange(len(key1))] data2 = r.read(key1, squeeze=True, row_offset=row_offset, num_rows=10) assert isinstance(data2, np.ndarray) assert data2.ndim == 3 for d1, d2 in zip(data1, data2[:-1]): assert_allclose(d1, d2) assert_allclose(data2[-1], np.zeros(data2[0].shape))
def test_read_squeeze_random_vec_permissive(): r = SDRF.create(vec_scp_b, path_prefix=input_prefix) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0]) i += 1 # binary key1.append('unk') r = RDRF.create('p,' + vec_scp_b, path_prefix=input_prefix) data2 = r.read(key1, squeeze=True) assert isinstance(data2, np.ndarray) assert data2.ndim == 2 for d1, d2 in zip(data1, data2[:-1]): assert_allclose(d1, d2) assert_allclose(data2[-1], np.zeros(data2[0].shape))
def test_read_range2_compress_random_feat(): for k, cm in enumerate(compression_methods): r = SDRF.create(feat_scp_uc[k], path_prefix=input_prefix) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0][i:i + 10]) i += 1 r = RDRF.create(feat_scp_c[k], path_prefix=input_prefix) row_offset = [i for i in xrange(len(key1))] data2 = r.read(key1, row_offset=row_offset, num_rows=10) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2, rtol=1e-5, atol=1e-4, err_msg=('Read compression %s failed' % cm))
def test_read_range_compress_random_feat(): for k, cm in enumerate(compression_methods): # scp uncompressed binary r = SDRF.create(feat_scp_uc[k], path_prefix=input_prefix) key1 = [] data1 = [] i = 0 while not r.eof(): key_i, data_i = r.read(1) key1.append(key_i[0]) data1.append(data_i[0][i:i + 50]) i += 1 # scp compressed binary r = RDRF.create(feat_range_c[k], path_prefix=input_prefix) data2 = r.read(key1) for d1, d2 in zip(data1, data2): assert_allclose(d1, d2, rtol=1e-5, atol=1e-4, err_msg=('Read compression %s failed' % cm))