def __init__(self, wspecifier, compression_method=None, write_function=None):
self.initialized = False
self.closed = False
self.compression_method = compression_method
self.write_function = write_function
spec_dict = parse_specifier(wspecifier)
if spec_dict["scp"] is not None and spec_dict["ark"] is None:
raise ValueError(
"Writing only in a scp file is not supported. "
"Please specify a ark file with a scp file."
)
for k in spec_dict:
if spec_dict[k] and k not in ("scp", "ark", "t", "f"):
warnings.warn(
"{} option is given, but currently it never affects".format(k)
)
self.text = spec_dict["t"]
self.flush = spec_dict["f"]
ark_file = spec_dict["ark"]
self.fark = open_like_kaldi(ark_file, "wb")
if spec_dict["scp"] is not None:
self.fscp = open_like_kaldi(spec_dict["scp"], "w")
else:
self.fscp = None
self.initialized = True
def test_open_like_kaldi(tmpdir):
with open_like_kaldi('echo hello |', 'r') as f:
assert f.read() == 'hello\n'
txt = tmpdir.mkdir('test').join('out.txt').strpath
with open_like_kaldi('| cat > {}'.format(txt), 'w') as f:
f.write('hello')
with open(txt, 'r') as f:
assert f.read() == 'hello'
def test_open_stdsteadm():
with open_like_kaldi('-', 'w') as f:
assert f is sys.stdout
with open_like_kaldi('-', 'wb'):
pass
with open_like_kaldi('-', 'r') as f:
assert f is sys.stdin
with open_like_kaldi('-', 'rb'):
pass
def test_open_like_kaldi(tmpdir):
with open_like_kaldi("echo あああ |", "r") as f:
if PY3:
assert f.read() == "あああ\n"
else:
assert f.read().decode("utf-8") == "あああ\n"
txt = tmpdir.mkdir("test").join("out.txt").strpath
with open_like_kaldi("| cat > {}".format(txt), "w") as f:
if PY3:
f.write("あああ")
else:
f.write("あああ".encode("utf-8"))
with io.open(txt, "r", encoding="utf-8") as f:
assert f.read() == "あああ"
def test_open_like_kaldi(tmpdir):
with open_like_kaldi('echo あああ |', 'r') as f:
if PY3:
assert f.read() == 'あああ\n'
else:
assert f.read().decode('utf-8') == 'あああ\n'
txt = tmpdir.mkdir('test').join('out.txt').strpath
with open_like_kaldi('| cat > {}'.format(txt), 'w') as f:
if PY3:
f.write('あああ')
else:
f.write('あああ'.encode('utf-8'))
with io.open(txt, 'r', encoding='utf-8') as f:
assert f.read() == 'あああ'
def load_scp_sequential(fname, endian="<", separator=None, segments=None):
"""Lazy loader for kaldi scp file.
Args:
fname (str or file(text mode)):
endian (str):
separator (str):
segments (str): The path of segments
"""
assert endian in ("<", ">"), endian
if segments is None:
with open_like_kaldi(fname, "r") as fd:
prev_ark = None
prev_arkfd = None
try:
for line in fd:
seps = line.split(separator, 1)
if len(seps) != 2:
raise ValueError(
"Invalid line is found:\n> {}".format(line))
token, arkname = seps
arkname = arkname.rstrip()
ark, offset, slices = _parse_arkpath(arkname)
if prev_ark == ark:
arkfd = prev_arkfd
mat = _load_mat(arkfd, offset, slices, endian=endian)
else:
if prev_arkfd is not None:
prev_arkfd.close()
arkfd = open_like_kaldi(ark, "rb")
mat = _load_mat(arkfd, offset, slices, endian=endian)
prev_ark = ark
prev_arkfd = arkfd
yield token, mat
except Exception:
if prev_arkfd is not None:
prev_arkfd.close()
raise
else:
for data in SegmentsExtractor(fname,
separator=separator,
segments=segments).generator():
yield data
def load_scp(fname, endian='<', separator=None, as_bytes=False, segments=None):
"""Lazy loader for kaldi scp file.
Args:
fname (str or file(text mode)):
endian (str):
separator (str):
as_bytes (bool): Read as raw bytes string
segments (str): The path of segments
"""
assert endian in ('<', '>'), endian
if segments is None:
load_func = partial(load_mat, endian=endian, as_bytes=as_bytes)
loader = LazyLoader(load_func)
with open_like_kaldi(fname, 'r') as fd:
for line in fd:
seps = line.split(separator, 1)
if len(seps) != 2:
raise ValueError(
'Invalid line is found:\n> {}'.format(line))
token, arkname = seps
loader[token] = arkname.rstrip()
return loader
else:
return SegmentsExtractor(fname, separator=separator, segments=segments)
def __init__(self, wspecifier, segments=None):
self.initialized = False
self.scp = None
self.closed = False
self.generator = None
self.segments = segments
spec_dict = parse_specifier(wspecifier)
if spec_dict["scp"] is not None and spec_dict["ark"] is not None:
raise RuntimeError("Specify one of scp or ark in rspecifier")
for k in spec_dict:
if spec_dict[k] and k not in ("scp", "ark", "p"):
warnings.warn(
"{} option is given, but currently it never affects".format(k)
)
self.permissive = spec_dict["p"]
if spec_dict["scp"] is not None:
self.scp = spec_dict["scp"]
else:
self.scp = False
if self.scp:
self.generator = load_scp_sequential(spec_dict["scp"], segments=segments)
self.file = None
else:
if segments is not None:
raise ValueError('Not supporting "segments" argument with ark file')
self.file = open_like_kaldi(spec_dict["ark"], "rb")
self.initialized = True
def load_scp(fname, endian="<", separator=None, segments=None, max_cache_fd=0):
"""Lazy loader for kaldi scp file.
Args:
fname (str or file(text mode)):
endian (str):
separator (str):
segments (str): The path of segments
"""
assert endian in ("<", ">"), endian
if max_cache_fd != 0:
if segments is not None:
raise ValueError("max_cache_fd is not supported for segments mode")
d = LimitedSizeDict(max_cache_fd)
else:
d = None
if segments is None:
load_func = partial(load_mat, endian=endian, fd_dict=d)
loader = LazyLoader(load_func)
with open_like_kaldi(fname, "r") as fd:
for line in fd:
seps = line.split(separator, 1)
if len(seps) != 2:
raise ValueError(
"Invalid line is found:\n> {}".format(line))
token, arkname = seps
loader[token] = arkname.rstrip()
return loader
else:
return SegmentsExtractor(fname, separator=separator, segments=segments)
def test_wavark_stream(tmpdir, dtype, write_function):
path = tmpdir.mkdir("test")
ark = path.join("a.ark").strpath
# Write as pcm16
array = np.random.randint(-1000, 1000, 100).astype(np.double) / abs(
np.iinfo(np.int16).min)
array2 = np.random.randint(-1000, 1000, 100).astype(np.double) / abs(
np.iinfo(np.int16).min)
if write_function == "numpy":
d = {"utt": array, "utt2": array2}
else:
d = {"utt": (8000, array), "utt2": (8000, array2)}
save_ark(ark, d, write_function=write_function)
with open_like_kaldi("cat {}|".format(ark), "rb") as f:
d = dict(load_ark(f))
if write_function == "numpy":
test = d["utt"]
else:
rate, test = d["utt"]
assert rate == 8000
np.testing.assert_allclose(array, test)
if write_function == "numpy":
test = d["utt2"]
else:
rate, test = d["utt2"]
assert rate == 8000
np.testing.assert_allclose(array2, test)
def load_mat(ark_name, endian="<", fd_dict=None):
assert endian in ("<", ">"), endian
if fd_dict is not None and not isinstance(fd_dict, Mapping):
raise RuntimeError("fd_dict must be dict or None, bot got {}".format(
type(fd_dict)))
ark, offset, slices = _parse_arkpath(ark_name)
if fd_dict is not None and not (ark.strip()[-1] == "|"
or ark.strip()[0] == "|"):
if ark not in fd_dict:
fd_dict[ark] = open_like_kaldi(ark, "rb")
fd = fd_dict[ark]
return _load_mat(fd, offset, slices, endian=endian)
else:
with open_like_kaldi(ark, "rb") as fd:
return _load_mat(fd, offset, slices, endian=endian)
def from_file(
cls, audio_file, target_sr=None, int_values=False, offset=0, duration=0, trim=False, orig_sr=None,
):
"""
Load a file supported by librosa and return as an AudioSegment.
:param audio_file: path of file to load
:param target_sr: the desired sample rate
:param int_values: if true, load samples as 32-bit integers
:param offset: offset in seconds when loading audio
:param duration: duration in seconds when loading audio
:return: numpy array of samples
"""
samples = None
if not isinstance(audio_file, str) or os.path.splitext(audio_file)[-1] in sf_supported_formats:
try:
with sf.SoundFile(audio_file, 'r') as f:
dtype = 'int32' if int_values else 'float32'
sample_rate = f.samplerate
if offset > 0:
f.seek(int(offset * sample_rate))
if duration > 0:
samples = f.read(int(duration * sample_rate), dtype=dtype)
else:
samples = f.read(dtype=dtype)
samples = samples.transpose()
except RuntimeError as e:
logging.error(
f"Loading {audio_file} via SoundFile raised RuntimeError: `{e}`. "
f"NeMo will fallback to loading via pydub."
)
elif isinstance(audio_file, str) and audio_file.strip()[-1] == "|":
f = open_like_kaldi(audio_file, "rb")
sample_rate, samples = read_kaldi(f)
if offset > 0:
samples = samples[int(offset * sample_rate) :]
if duration > 0:
samples = samples[: int(duration * sample_rate)]
if not int_values:
abs_max_value = np.abs(samples).max()
samples = np.array(samples, dtype=np.float) / abs_max_value
if samples is None:
try:
samples = Audio.from_file(audio_file)
sample_rate = samples.frame_rate
if offset > 0:
# pydub does things in milliseconds
seconds = offset * 1000
samples = samples[int(seconds) :]
if duration > 0:
seconds = duration * 1000
samples = samples[: int(seconds)]
samples = np.array(samples.get_array_of_samples())
except CouldntDecodeError as e:
logging.error(f"Loading {audio_file} via pydub raised CouldntDecodeError: `{e}`.")
return cls(samples, sample_rate, target_sr=target_sr, trim=trim, orig_sr=orig_sr)
def __init__(self, wspecifier):
spec_dict = parse_specifier(wspecifier)
if set(spec_dict) == {'scp'}:
raise ValueError('Writing only in a scp file is not supported. '
'Please specify a ark file with a scp file.')
if 'ark,t' in spec_dict:
ark_file = spec_dict['ark,t']
self.text = True
else:
ark_file = spec_dict['ark']
self.text = False
self.fark = open_like_kaldi(ark_file, 'wb')
if 'scp' in spec_dict:
self.fscp = open_like_kaldi(spec_dict['scp'], 'w')
else:
self.fscp = None
self.closed = False
def load_mat(ark_name, endian='<', as_bytes=False):
assert endian in ('<', '>'), endian
ark, offset, slices = _parse_arkpath(ark_name)
with open_like_kaldi(ark, 'rb') as fd:
return _load_mat(fd,
offset,
slices,
endian=endian,
as_bytes=as_bytes,
use_scipy_wav=offset is None)
def __init__(self, wspecifier):
spec_dict = parse_specifier(wspecifier)
if len(spec_dict) != 1:
raise RuntimeError('Specify one of scp or ark in rspecifier')
if 'scp' in spec_dict:
self.scp = True
mode = 'r'
else:
self.scp = False
mode = 'rb'
if self.scp:
with open_like_kaldi(next(iter(spec_dict.values())), mode) as f:
self.dict = load_scp(f)
self.file = None
else:
self.dict = None
self.file = open_like_kaldi(next(iter(spec_dict.values())), mode)
self.closed = False
def test_wavark_stream(tmpdir, dtype):
path = tmpdir.mkdir('test')
ark = path.join('a.ark').strpath
# Write as pcm16
array = np.random.randint(0, 10, 10, dtype=dtype)
array2 = np.random.randint(0, 10, 10, dtype=dtype)
d = {'utt': (8000, array), 'utt2': (8000, array2)}
save_ark(ark, d)
with open_like_kaldi('cat {}|'.format(ark), 'rb') as f:
d = dict(load_ark(f))
rate, test = d['utt']
assert rate == 8000
np.testing.assert_array_equal(array, test)
rate, test = d['utt2']
assert rate == 8000
np.testing.assert_array_equal(array2, test)
def test_wavark_stream(tmpdir, dtype):
path = tmpdir.mkdir("test")
ark = path.join("a.ark").strpath
# Write as pcm16
array = np.random.randint(0, 10, 10, dtype=dtype)
array2 = np.random.randint(0, 10, 10, dtype=dtype)
d = {"utt": (8000, array), "utt2": (8000, array2)}
save_ark(ark, d)
with open_like_kaldi("cat {}|".format(ark), "rb") as f:
d = dict(load_ark(f))
rate, test = d["utt"]
assert rate == 8000
np.testing.assert_array_equal(array, test)
rate, test = d["utt2"]
assert rate == 8000
np.testing.assert_array_equal(array2, test)
def load_mat(ark_name, endian="<"):
assert endian in ("<", ">"), endian
ark, offset, slices = _parse_arkpath(ark_name)
with open_like_kaldi(ark, "rb") as fd:
return _load_mat(fd, offset, slices, endian=endian)
def load_scp_sequential(fname,
endian='<',
separator=None,
as_bytes=False,
segments=None):
"""Lazy loader for kaldi scp file.
Args:
fname (str or file(text mode)):
endian (str):
separator (str):
as_bytes (bool): Read as raw bytes string
segments (str): The path of segments
"""
assert endian in ('<', '>'), endian
if segments is None:
with open_or_fd(fname, 'r') as fd:
prev_ark = None
prev_arkfd = None
try:
for line in fd:
seps = line.split(separator, 1)
if len(seps) != 2:
raise ValueError(
'Invalid line is found:\n> {}'.format(line))
token, arkname = seps
arkname = arkname.rstrip()
ark, offset, slices = _parse_arkpath(arkname)
if prev_ark == ark:
arkfd = prev_arkfd
mat = _load_mat(arkfd,
offset,
slices,
endian=endian,
as_bytes=as_bytes)
else:
if prev_arkfd is not None:
prev_arkfd.close()
arkfd = open_like_kaldi(ark, 'rb')
mat = _load_mat(arkfd,
offset,
slices,
endian=endian,
as_bytes=as_bytes)
prev_ark = ark
prev_arkfd = arkfd
yield token, mat
except Exception:
if prev_arkfd is not None:
prev_arkfd.close()
raise
else:
for data in SegmentsExtractor(fname,
separator=separator,
segments=segments).generator():
yield data
