def __init__(self, wspecifier, compression_method=None, write_function=None): self.initialized = False self.closed = False self.compression_method = compression_method self.write_function = write_function spec_dict = parse_specifier(wspecifier) if spec_dict["scp"] is not None and spec_dict["ark"] is None: raise ValueError( "Writing only in a scp file is not supported. " "Please specify a ark file with a scp file." ) for k in spec_dict: if spec_dict[k] and k not in ("scp", "ark", "t", "f"): warnings.warn( "{} option is given, but currently it never affects".format(k) ) self.text = spec_dict["t"] self.flush = spec_dict["f"] ark_file = spec_dict["ark"] self.fark = open_like_kaldi(ark_file, "wb") if spec_dict["scp"] is not None: self.fscp = open_like_kaldi(spec_dict["scp"], "w") else: self.fscp = None self.initialized = True
def test_open_like_kaldi(tmpdir): with open_like_kaldi('echo hello |', 'r') as f: assert f.read() == 'hello\n' txt = tmpdir.mkdir('test').join('out.txt').strpath with open_like_kaldi('| cat > {}'.format(txt), 'w') as f: f.write('hello') with open(txt, 'r') as f: assert f.read() == 'hello'
def test_open_stdsteadm(): with open_like_kaldi('-', 'w') as f: assert f is sys.stdout with open_like_kaldi('-', 'wb'): pass with open_like_kaldi('-', 'r') as f: assert f is sys.stdin with open_like_kaldi('-', 'rb'): pass
def test_open_like_kaldi(tmpdir): with open_like_kaldi("echo あああ |", "r") as f: if PY3: assert f.read() == "あああ\n" else: assert f.read().decode("utf-8") == "あああ\n" txt = tmpdir.mkdir("test").join("out.txt").strpath with open_like_kaldi("| cat > {}".format(txt), "w") as f: if PY3: f.write("あああ") else: f.write("あああ".encode("utf-8")) with io.open(txt, "r", encoding="utf-8") as f: assert f.read() == "あああ"
def test_open_like_kaldi(tmpdir): with open_like_kaldi('echo あああ |', 'r') as f: if PY3: assert f.read() == 'あああ\n' else: assert f.read().decode('utf-8') == 'あああ\n' txt = tmpdir.mkdir('test').join('out.txt').strpath with open_like_kaldi('| cat > {}'.format(txt), 'w') as f: if PY3: f.write('あああ') else: f.write('あああ'.encode('utf-8')) with io.open(txt, 'r', encoding='utf-8') as f: assert f.read() == 'あああ'
def load_scp_sequential(fname, endian="<", separator=None, segments=None): """Lazy loader for kaldi scp file. Args: fname (str or file(text mode)): endian (str): separator (str): segments (str): The path of segments """ assert endian in ("<", ">"), endian if segments is None: with open_like_kaldi(fname, "r") as fd: prev_ark = None prev_arkfd = None try: for line in fd: seps = line.split(separator, 1) if len(seps) != 2: raise ValueError( "Invalid line is found:\n> {}".format(line)) token, arkname = seps arkname = arkname.rstrip() ark, offset, slices = _parse_arkpath(arkname) if prev_ark == ark: arkfd = prev_arkfd mat = _load_mat(arkfd, offset, slices, endian=endian) else: if prev_arkfd is not None: prev_arkfd.close() arkfd = open_like_kaldi(ark, "rb") mat = _load_mat(arkfd, offset, slices, endian=endian) prev_ark = ark prev_arkfd = arkfd yield token, mat except Exception: if prev_arkfd is not None: prev_arkfd.close() raise else: for data in SegmentsExtractor(fname, separator=separator, segments=segments).generator(): yield data
def load_scp(fname, endian='<', separator=None, as_bytes=False, segments=None): """Lazy loader for kaldi scp file. Args: fname (str or file(text mode)): endian (str): separator (str): as_bytes (bool): Read as raw bytes string segments (str): The path of segments """ assert endian in ('<', '>'), endian if segments is None: load_func = partial(load_mat, endian=endian, as_bytes=as_bytes) loader = LazyLoader(load_func) with open_like_kaldi(fname, 'r') as fd: for line in fd: seps = line.split(separator, 1) if len(seps) != 2: raise ValueError( 'Invalid line is found:\n> {}'.format(line)) token, arkname = seps loader[token] = arkname.rstrip() return loader else: return SegmentsExtractor(fname, separator=separator, segments=segments)
def __init__(self, wspecifier, segments=None): self.initialized = False self.scp = None self.closed = False self.generator = None self.segments = segments spec_dict = parse_specifier(wspecifier) if spec_dict["scp"] is not None and spec_dict["ark"] is not None: raise RuntimeError("Specify one of scp or ark in rspecifier") for k in spec_dict: if spec_dict[k] and k not in ("scp", "ark", "p"): warnings.warn( "{} option is given, but currently it never affects".format(k) ) self.permissive = spec_dict["p"] if spec_dict["scp"] is not None: self.scp = spec_dict["scp"] else: self.scp = False if self.scp: self.generator = load_scp_sequential(spec_dict["scp"], segments=segments) self.file = None else: if segments is not None: raise ValueError('Not supporting "segments" argument with ark file') self.file = open_like_kaldi(spec_dict["ark"], "rb") self.initialized = True
def load_scp(fname, endian="<", separator=None, segments=None, max_cache_fd=0): """Lazy loader for kaldi scp file. Args: fname (str or file(text mode)): endian (str): separator (str): segments (str): The path of segments """ assert endian in ("<", ">"), endian if max_cache_fd != 0: if segments is not None: raise ValueError("max_cache_fd is not supported for segments mode") d = LimitedSizeDict(max_cache_fd) else: d = None if segments is None: load_func = partial(load_mat, endian=endian, fd_dict=d) loader = LazyLoader(load_func) with open_like_kaldi(fname, "r") as fd: for line in fd: seps = line.split(separator, 1) if len(seps) != 2: raise ValueError( "Invalid line is found:\n> {}".format(line)) token, arkname = seps loader[token] = arkname.rstrip() return loader else: return SegmentsExtractor(fname, separator=separator, segments=segments)
def test_wavark_stream(tmpdir, dtype, write_function): path = tmpdir.mkdir("test") ark = path.join("a.ark").strpath # Write as pcm16 array = np.random.randint(-1000, 1000, 100).astype(np.double) / abs( np.iinfo(np.int16).min) array2 = np.random.randint(-1000, 1000, 100).astype(np.double) / abs( np.iinfo(np.int16).min) if write_function == "numpy": d = {"utt": array, "utt2": array2} else: d = {"utt": (8000, array), "utt2": (8000, array2)} save_ark(ark, d, write_function=write_function) with open_like_kaldi("cat {}|".format(ark), "rb") as f: d = dict(load_ark(f)) if write_function == "numpy": test = d["utt"] else: rate, test = d["utt"] assert rate == 8000 np.testing.assert_allclose(array, test) if write_function == "numpy": test = d["utt2"] else: rate, test = d["utt2"] assert rate == 8000 np.testing.assert_allclose(array2, test)
def load_mat(ark_name, endian="<", fd_dict=None): assert endian in ("<", ">"), endian if fd_dict is not None and not isinstance(fd_dict, Mapping): raise RuntimeError("fd_dict must be dict or None, bot got {}".format( type(fd_dict))) ark, offset, slices = _parse_arkpath(ark_name) if fd_dict is not None and not (ark.strip()[-1] == "|" or ark.strip()[0] == "|"): if ark not in fd_dict: fd_dict[ark] = open_like_kaldi(ark, "rb") fd = fd_dict[ark] return _load_mat(fd, offset, slices, endian=endian) else: with open_like_kaldi(ark, "rb") as fd: return _load_mat(fd, offset, slices, endian=endian)
def from_file( cls, audio_file, target_sr=None, int_values=False, offset=0, duration=0, trim=False, orig_sr=None, ): """ Load a file supported by librosa and return as an AudioSegment. :param audio_file: path of file to load :param target_sr: the desired sample rate :param int_values: if true, load samples as 32-bit integers :param offset: offset in seconds when loading audio :param duration: duration in seconds when loading audio :return: numpy array of samples """ samples = None if not isinstance(audio_file, str) or os.path.splitext(audio_file)[-1] in sf_supported_formats: try: with sf.SoundFile(audio_file, 'r') as f: dtype = 'int32' if int_values else 'float32' sample_rate = f.samplerate if offset > 0: f.seek(int(offset * sample_rate)) if duration > 0: samples = f.read(int(duration * sample_rate), dtype=dtype) else: samples = f.read(dtype=dtype) samples = samples.transpose() except RuntimeError as e: logging.error( f"Loading {audio_file} via SoundFile raised RuntimeError: `{e}`. " f"NeMo will fallback to loading via pydub." ) elif isinstance(audio_file, str) and audio_file.strip()[-1] == "|": f = open_like_kaldi(audio_file, "rb") sample_rate, samples = read_kaldi(f) if offset > 0: samples = samples[int(offset * sample_rate) :] if duration > 0: samples = samples[: int(duration * sample_rate)] if not int_values: abs_max_value = np.abs(samples).max() samples = np.array(samples, dtype=np.float) / abs_max_value if samples is None: try: samples = Audio.from_file(audio_file) sample_rate = samples.frame_rate if offset > 0: # pydub does things in milliseconds seconds = offset * 1000 samples = samples[int(seconds) :] if duration > 0: seconds = duration * 1000 samples = samples[: int(seconds)] samples = np.array(samples.get_array_of_samples()) except CouldntDecodeError as e: logging.error(f"Loading {audio_file} via pydub raised CouldntDecodeError: `{e}`.") return cls(samples, sample_rate, target_sr=target_sr, trim=trim, orig_sr=orig_sr)
def __init__(self, wspecifier): spec_dict = parse_specifier(wspecifier) if set(spec_dict) == {'scp'}: raise ValueError('Writing only in a scp file is not supported. ' 'Please specify a ark file with a scp file.') if 'ark,t' in spec_dict: ark_file = spec_dict['ark,t'] self.text = True else: ark_file = spec_dict['ark'] self.text = False self.fark = open_like_kaldi(ark_file, 'wb') if 'scp' in spec_dict: self.fscp = open_like_kaldi(spec_dict['scp'], 'w') else: self.fscp = None self.closed = False
def load_mat(ark_name, endian='<', as_bytes=False): assert endian in ('<', '>'), endian ark, offset, slices = _parse_arkpath(ark_name) with open_like_kaldi(ark, 'rb') as fd: return _load_mat(fd, offset, slices, endian=endian, as_bytes=as_bytes, use_scipy_wav=offset is None)
def __init__(self, wspecifier): spec_dict = parse_specifier(wspecifier) if len(spec_dict) != 1: raise RuntimeError('Specify one of scp or ark in rspecifier') if 'scp' in spec_dict: self.scp = True mode = 'r' else: self.scp = False mode = 'rb' if self.scp: with open_like_kaldi(next(iter(spec_dict.values())), mode) as f: self.dict = load_scp(f) self.file = None else: self.dict = None self.file = open_like_kaldi(next(iter(spec_dict.values())), mode) self.closed = False
def test_wavark_stream(tmpdir, dtype): path = tmpdir.mkdir('test') ark = path.join('a.ark').strpath # Write as pcm16 array = np.random.randint(0, 10, 10, dtype=dtype) array2 = np.random.randint(0, 10, 10, dtype=dtype) d = {'utt': (8000, array), 'utt2': (8000, array2)} save_ark(ark, d) with open_like_kaldi('cat {}|'.format(ark), 'rb') as f: d = dict(load_ark(f)) rate, test = d['utt'] assert rate == 8000 np.testing.assert_array_equal(array, test) rate, test = d['utt2'] assert rate == 8000 np.testing.assert_array_equal(array2, test)
def test_wavark_stream(tmpdir, dtype): path = tmpdir.mkdir("test") ark = path.join("a.ark").strpath # Write as pcm16 array = np.random.randint(0, 10, 10, dtype=dtype) array2 = np.random.randint(0, 10, 10, dtype=dtype) d = {"utt": (8000, array), "utt2": (8000, array2)} save_ark(ark, d) with open_like_kaldi("cat {}|".format(ark), "rb") as f: d = dict(load_ark(f)) rate, test = d["utt"] assert rate == 8000 np.testing.assert_array_equal(array, test) rate, test = d["utt2"] assert rate == 8000 np.testing.assert_array_equal(array2, test)
def load_mat(ark_name, endian="<"): assert endian in ("<", ">"), endian ark, offset, slices = _parse_arkpath(ark_name) with open_like_kaldi(ark, "rb") as fd: return _load_mat(fd, offset, slices, endian=endian)
def load_scp_sequential(fname, endian='<', separator=None, as_bytes=False, segments=None): """Lazy loader for kaldi scp file. Args: fname (str or file(text mode)): endian (str): separator (str): as_bytes (bool): Read as raw bytes string segments (str): The path of segments """ assert endian in ('<', '>'), endian if segments is None: with open_or_fd(fname, 'r') as fd: prev_ark = None prev_arkfd = None try: for line in fd: seps = line.split(separator, 1) if len(seps) != 2: raise ValueError( 'Invalid line is found:\n> {}'.format(line)) token, arkname = seps arkname = arkname.rstrip() ark, offset, slices = _parse_arkpath(arkname) if prev_ark == ark: arkfd = prev_arkfd mat = _load_mat(arkfd, offset, slices, endian=endian, as_bytes=as_bytes) else: if prev_arkfd is not None: prev_arkfd.close() arkfd = open_like_kaldi(ark, 'rb') mat = _load_mat(arkfd, offset, slices, endian=endian, as_bytes=as_bytes) prev_ark = ark prev_arkfd = arkfd yield token, mat except Exception: if prev_arkfd is not None: prev_arkfd.close() raise else: for data in SegmentsExtractor(fname, separator=separator, segments=segments).generator(): yield data