def test_segments(tmpdir, func, dtype):
# Create wav.scp
path = tmpdir.mkdir('test')
wavscp = path.join('wav.scp').strpath
rate = 500
with open(wavscp, 'w') as f:
wav = path.join('0.wav').strpath
array0 = np.random.randint(0, 10, 2000, dtype=dtype)
write_wav(wav, rate, array0)
f.write('wav0 {}\n'.format(wav))
wav = path.join('1.wav').strpath
array1 = np.random.randint(0, 10, 2000, dtype=dtype)
write_wav(wav, rate, array1)
f.write('wav1 {}\n'.format(wav))
# Create segments
segments = path.join('segments').strpath
with open(segments, 'w') as f:
f.write('utt1 wav0 0.1 0.2\n')
f.write('utt2 wav0 0.4 0.6\n')
f.write('utt3 wav1 0.4 0.5\n')
f.write('utt4 wav1 0.6 0.8\n')
d = dict(func(wavscp, segments=segments))
np.testing.assert_array_equal(d['utt1'][1],
array0[int(0.1 * rate):int(0.2 * rate)])
np.testing.assert_array_equal(d['utt2'][1],
array0[int(0.4 * rate):int(0.6 * rate)])
np.testing.assert_array_equal(d['utt3'][1],
array1[int(0.4 * rate):int(0.5 * rate)])
np.testing.assert_array_equal(d['utt4'][1],
array1[int(0.6 * rate):int(0.8 * rate)])
def test_segments(tmpdir, func, dtype):
# Create wav.scp
path = tmpdir.mkdir("test")
wavscp = path.join("wav.scp").strpath
rate = 500
with open(wavscp, "w") as f:
wav = path.join("0.wav").strpath
array0 = np.random.randint(0, 10, 2000, dtype=dtype)
write_wav(wav, rate, array0)
f.write("wav0 {}\n".format(wav))
wav = path.join("1.wav").strpath
array1 = np.random.randint(0, 10, 2000, dtype=dtype)
write_wav(wav, rate, array1)
f.write("wav1 {}\n".format(wav))
# Create segments
segments = path.join("segments").strpath
with open(segments, "w") as f:
f.write("utt1 wav0 0.1 0.2\n")
f.write("utt2 wav0 0.4 0.6\n")
f.write("utt3 wav1 0.4 0.5\n")
f.write("utt4 wav1 0.6 0.8\n")
d = dict(func(wavscp, segments=segments))
np.testing.assert_array_equal(d["utt1"][1],
array0[int(0.1 * rate):int(0.2 * rate)])
np.testing.assert_array_equal(d["utt2"][1],
array0[int(0.4 * rate):int(0.6 * rate)])
np.testing.assert_array_equal(d["utt3"][1],
array1[int(0.4 * rate):int(0.5 * rate)])
np.testing.assert_array_equal(d["utt4"][1],
array1[int(0.6 * rate):int(0.8 * rate)])
def test_read_wav(tmpdir, func, dtype):
path = tmpdir.mkdir('test')
wav = path.join('a.wav').strpath
# Write as pcm16
array = np.random.randint(0, 10, 10, dtype=dtype)
write_wav(wav, 8000, array)
with open(wav, 'rb') as f:
rate, array2 = func(f)
np.testing.assert_array_equal(array, array2)
def test_load_wav_scp(tmpdir, func, dtype):
path = tmpdir.mkdir('test')
wav = path.join('a.wav').strpath
scp = path.join('wav.scp').strpath
# Write as pcm16
array = np.random.randint(0, 10, 10, dtype=dtype)
write_wav(wav, 8000, array)
with open(scp, 'w') as f:
f.write('aaa {wav}\n'.format(wav=wav))
rate, array2 = list(dict(func(scp)).values())[0]
np.testing.assert_array_equal(array, array2)
def test_scpwav_stream(tmpdir, func, dtype):
path = tmpdir.mkdir('test')
wav = path.join('aaa.wav').strpath
wav2 = path.join('bbb.wav').strpath
scp = path.join('wav.scp').strpath
# Write as pcm16
array = np.random.randint(0, 10, 40, dtype=dtype).reshape(5, 8)
write_wav(wav, 8000, array)
array2 = np.random.randint(0, 10, 10, dtype=dtype)
write_wav(wav2, 8000, array2)
with open(scp, 'w') as f:
f.write('aaa sox {wav} -t wav - |\n'.format(wav=wav))
f.write('bbb cat {wav} |\n'.format(wav=wav2))
rate, test = dict(func(scp))['aaa']
rate, test2 = dict(func(scp))['bbb']
np.testing.assert_array_equal(array, test)
np.testing.assert_array_equal(array2, test2)
def test_scpwav_stream(tmpdir, func, dtype):
path = tmpdir.mkdir("test")
wav = path.join("aaa.wav").strpath
wav2 = path.join("bbb.wav").strpath
scp = path.join("wav.scp").strpath
# Write as pcm16
array = np.random.randint(0, 10, 40, dtype=dtype).reshape(5, 8)
write_wav(wav, 8000, array)
array2 = np.random.randint(0, 10, 10, dtype=dtype)
write_wav(wav2, 8000, array2)
with open(scp, "w") as f:
f.write("aaa sox {wav} -t wav - |\n".format(wav=wav))
f.write("bbb cat {wav} |\n".format(wav=wav2))
rate, test = dict(func(scp))["aaa"]
rate, test2 = dict(func(scp))["bbb"]
np.testing.assert_array_equal(array, test)
np.testing.assert_array_equal(array2, test2)
def test_scpwav_stream(tmpdir):
path = tmpdir.mkdir("test")
wav = path.join("aaa.wav").strpath
wav2 = path.join("bbb.wav").strpath
scp = path.join("wav.scp").strpath
# Write as pcm16
array = numpy.random.randint(0, 10, 10, dtype=numpy.int16)
write_wav(wav, 8000, array)
array2 = numpy.random.randint(0, 10, 10, dtype=numpy.int16)
write_wav(wav2, 8000, array2)
valid = {"aaa": array, "bbb": array2}
with open(scp, "w") as f:
f.write("aaa cat {wav} |\n".format(wav=wav))
f.write("bbb cat {wav} |\n".format(wav=wav2))
with ReadHelper("scp:{}".format(scp)) as r:
for k, (rate, array) in r:
assert rate == 8000
numpy.testing.assert_array_equal(array, valid[k])
def test_scpwav_stream(tmpdir):
path = tmpdir.mkdir('test')
wav = path.join('aaa.wav').strpath
wav2 = path.join('bbb.wav').strpath
scp = path.join('wav.scp').strpath
# Write as pcm16
array = numpy.random.randint(0, 10, 10, dtype=numpy.int16)
write_wav(wav, 8000, array)
array2 = numpy.random.randint(0, 10, 10, dtype=numpy.int16)
write_wav(wav2, 8000, array2)
valid = {'aaa': array, 'bbb': array2}
with open(scp, 'w') as f:
f.write('aaa cat {wav} |\n'.format(wav=wav))
f.write('bbb cat {wav} |\n'.format(wav=wav2))
with ReadHelper('scp:{}'.format(scp)) as r:
for k, (rate, array) in r:
assert rate == 8000
numpy.testing.assert_array_equal(array, valid[k])
def test_segments(tmpdir):
# Create wav.scp
path = tmpdir.mkdir("test")
wavscp = path.join("wav.scp").strpath
rate = 500
with open(wavscp, "w") as f:
wav = path.join("0.wav").strpath
array0 = numpy.random.randint(0, 10, 2000, dtype=numpy.int16)
write_wav(wav, rate, array0)
f.write("wav0 {}\n".format(wav))
wav = path.join("1.wav").strpath
array1 = numpy.random.randint(0, 10, 2000, dtype=numpy.int16)
write_wav(wav, rate, array1)
f.write("wav1 {}\n".format(wav))
# Create segments
segments = path.join("segments").strpath
with open(segments, "w") as f:
f.write("utt1 wav0 0.1 0.2\n")
f.write("utt2 wav0 0.4 0.6\n")
f.write("utt3 wav1 0.4 0.5\n")
f.write("utt4 wav1 0.6 0.8\n")
with ReadHelper("scp:{}".format(wavscp), segments=segments) as r:
d = {k: a for k, a in r}
numpy.testing.assert_array_equal(
d["utt1"][1], array0[int(0.1 * rate):int(0.2 * rate)])
numpy.testing.assert_array_equal(
d["utt2"][1], array0[int(0.4 * rate):int(0.6 * rate)])
numpy.testing.assert_array_equal(
d["utt3"][1], array1[int(0.4 * rate):int(0.5 * rate)])
numpy.testing.assert_array_equal(
d["utt4"][1], array1[int(0.6 * rate):int(0.8 * rate)])
def test_segments(tmpdir):
# Create wav.scp
path = tmpdir.mkdir('test')
wavscp = path.join('wav.scp').strpath
rate = 500
with open(wavscp, 'w') as f:
wav = path.join('0.wav').strpath
array0 = numpy.random.randint(0, 10, 2000, dtype=numpy.int16)
write_wav(wav, rate, array0)
f.write('wav0 {}\n'.format(wav))
wav = path.join('1.wav').strpath
array1 = numpy.random.randint(0, 10, 2000, dtype=numpy.int16)
write_wav(wav, rate, array1)
f.write('wav1 {}\n'.format(wav))
# Create segments
segments = path.join('segments').strpath
with open(segments, 'w') as f:
f.write('utt1 wav0 0.1 0.2\n')
f.write('utt2 wav0 0.4 0.6\n')
f.write('utt3 wav1 0.4 0.5\n')
f.write('utt4 wav1 0.6 0.8\n')
with ReadHelper('scp:{}'.format(wavscp), segments=segments) as r:
d = {k: a for k, a in r}
numpy.testing.assert_array_equal(
d['utt1'][1], array0[int(0.1 * rate):int(0.2 * rate)])
numpy.testing.assert_array_equal(
d['utt2'][1], array0[int(0.4 * rate):int(0.6 * rate)])
numpy.testing.assert_array_equal(
d['utt3'][1], array1[int(0.4 * rate):int(0.5 * rate)])
numpy.testing.assert_array_equal(
d['utt4'][1], array1[int(0.6 * rate):int(0.8 * rate)])
def save_ark(ark,
array_dict,
scp=None,
append=False,
text=False,
as_bytes=False,
endian='<',
compression_method=None):
"""Write ark
Args:
ark (str or fd):
array_dict (dict):
scp (str or fd):
append (bool): If True is specified, open the file
with appendable mode
text (bool): If True, saving in text ark format.
as_bytes (bool): Save the value of the input array_dict as just a
bytes string.
endian (str):
compression_method (int):
"""
if isinstance(ark, string_types):
seekable = True
# Maybe, never match with this
elif not hasattr(ark, 'tell'):
seekable = False
else:
try:
ark.tell()
seekable = True
except Exception:
seekable = False
if scp is not None and not isinstance(ark, string_types):
if not seekable:
raise TypeError('scp file can be created only '
'if the output ark file is a file or '
'a seekable file descriptor.')
# Write ark
mode = 'ab' if append else 'wb'
pos_list = []
with open_or_fd(ark, mode) as fd:
if seekable:
offset = fd.tell()
else:
offset = 0
size = 0
for key in array_dict:
encode_key = (key + ' ').encode(encoding=default_encoding)
fd.write(encode_key)
size += len(encode_key)
pos_list.append(size)
if as_bytes:
byte = bytes(array_dict[key])
size += len(byte)
fd.write(byte)
else:
data = array_dict[key]
if isinstance(data, (list, tuple)):
rate, array = data
size += write_wav(fd, rate, array)
elif text:
size += write_array_ascii(fd, data, endian)
else:
size += write_array(fd, data, endian, compression_method)
# Write scp
mode = 'a' if append else 'w'
if scp is not None:
name = ark if isinstance(ark, string_types) else ark.name
with open_or_fd(scp, mode) as fd:
for key, position in zip(array_dict, pos_list):
fd.write(key + u' ' + name + ':' + str(position + offset) +
'\n')
def save_ark(
ark,
array_dict,
scp=None,
append=False,
text=False,
endian="<",
compression_method=None,
write_function=None,
):
"""Write ark
Args:
ark (str or fd):
array_dict (dict):
scp (str or fd):
append (bool): If True is specified, open the file
with appendable mode
text (bool): If True, saving in text ark format.
endian (str):
compression_method (int):
write_function: (str):
"""
if isinstance(ark, string_types):
seekable = True
# Maybe, never match with this
elif not hasattr(ark, "tell"):
seekable = False
else:
try:
ark.tell()
seekable = True
except Exception:
seekable = False
if scp is not None and not isinstance(ark, string_types):
if not seekable:
raise TypeError("scp file can be created only "
"if the output ark file is a file or "
"a seekable file descriptor.")
# Write ark
mode = "ab" if append else "wb"
pos_list = []
with open_or_fd(ark, mode) as fd:
if seekable:
offset = fd.tell()
else:
offset = 0
size = 0
for key in array_dict:
encode_key = (key + " ").encode(encoding=default_encoding)
fd.write(encode_key)
size += len(encode_key)
pos_list.append(size)
data = array_dict[key]
if write_function is not None:
# Ignore case
write_function = write_function.lower()
if write_function.startswith("soundfile"):
import soundfile
if "flac" in write_function:
audio_format = "flac"
elif "wav" in write_function:
audio_format = "wav"
else:
audio_format = "wav"
def _write_function(fd, data):
if not isinstance(data, (list, tuple)):
raise TypeError(
"Expected list or tuple type, but got {}".
format(type(data)))
elif len(data) != 2:
raise ValueError(
"Expected length=2, bot got {}".format(
len(data)))
_fd = BytesIO()
if isinstance(data[0], np.ndarray) and isinstance(
data[1], int):
soundfile.write(_fd,
data[0],
data[1],
format=audio_format)
elif isinstance(data[1], np.ndarray) and isinstance(
data[0], int):
soundfile.write(_fd,
data[1],
data[0],
format=audio_format)
else:
raise ValueError(
"Expected Tuple[int, np.ndarray] or "
"Tuple[np.ndarray, int]: "
"but got Tuple[{}, {}]".format(
type(data[0]), type(data[1])))
fd.write(b"AUDIO")
buf = _fd.getbuffer()
# Write the information for the length
bytes_length = _write_length_header(fd, len(buf))
fd.write(buf)
return len(buf) + len(b"AUDIO") + bytes_length
elif write_function == "pickle":
def _write_function(fd, data):
# Note that we don't need size information for pickle!
fd.write(b"PKL")
_fd = BytesIO()
pickle.dump(data, _fd)
buf = _fd.getbuffer()
fd.write(buf)
return len(buf) + len("PKL")
elif write_function == "numpy":
def _write_function(fd, data):
# Write numpy file in BytesIO
_fd = BytesIO()
np.save(_fd, data)
fd.write(b"NPY")
buf = _fd.getbuffer()
# Write the information for the length
bytes_length = _write_length_header(fd, len(buf))
# Write numpy to real file object
fd.write(buf)
return len(buf) + len(b"NPY") + bytes_length
else:
raise RuntimeError(
"Not supported: write_function={}".format(
write_function))
size += _write_function(fd, data)
elif isinstance(data, (list, tuple)):
rate, array = data
size += write_wav(fd, rate, array)
elif text:
size += write_array_ascii(fd, data, endian)
else:
size += write_array(fd, data, endian, compression_method)
# Write scp
mode = "a" if append else "w"
if scp is not None:
name = ark if isinstance(ark, string_types) else ark.name
with open_or_fd(scp, mode) as fd:
for key, position in zip(array_dict, pos_list):
fd.write(key + " " + name + ":" + str(position + offset) +
"\n")
