def test_segments(tmpdir, func, dtype): # Create wav.scp path = tmpdir.mkdir('test') wavscp = path.join('wav.scp').strpath rate = 500 with open(wavscp, 'w') as f: wav = path.join('0.wav').strpath array0 = np.random.randint(0, 10, 2000, dtype=dtype) write_wav(wav, rate, array0) f.write('wav0 {}\n'.format(wav)) wav = path.join('1.wav').strpath array1 = np.random.randint(0, 10, 2000, dtype=dtype) write_wav(wav, rate, array1) f.write('wav1 {}\n'.format(wav)) # Create segments segments = path.join('segments').strpath with open(segments, 'w') as f: f.write('utt1 wav0 0.1 0.2\n') f.write('utt2 wav0 0.4 0.6\n') f.write('utt3 wav1 0.4 0.5\n') f.write('utt4 wav1 0.6 0.8\n') d = dict(func(wavscp, segments=segments)) np.testing.assert_array_equal(d['utt1'][1], array0[int(0.1 * rate):int(0.2 * rate)]) np.testing.assert_array_equal(d['utt2'][1], array0[int(0.4 * rate):int(0.6 * rate)]) np.testing.assert_array_equal(d['utt3'][1], array1[int(0.4 * rate):int(0.5 * rate)]) np.testing.assert_array_equal(d['utt4'][1], array1[int(0.6 * rate):int(0.8 * rate)])
def test_segments(tmpdir, func, dtype): # Create wav.scp path = tmpdir.mkdir("test") wavscp = path.join("wav.scp").strpath rate = 500 with open(wavscp, "w") as f: wav = path.join("0.wav").strpath array0 = np.random.randint(0, 10, 2000, dtype=dtype) write_wav(wav, rate, array0) f.write("wav0 {}\n".format(wav)) wav = path.join("1.wav").strpath array1 = np.random.randint(0, 10, 2000, dtype=dtype) write_wav(wav, rate, array1) f.write("wav1 {}\n".format(wav)) # Create segments segments = path.join("segments").strpath with open(segments, "w") as f: f.write("utt1 wav0 0.1 0.2\n") f.write("utt2 wav0 0.4 0.6\n") f.write("utt3 wav1 0.4 0.5\n") f.write("utt4 wav1 0.6 0.8\n") d = dict(func(wavscp, segments=segments)) np.testing.assert_array_equal(d["utt1"][1], array0[int(0.1 * rate):int(0.2 * rate)]) np.testing.assert_array_equal(d["utt2"][1], array0[int(0.4 * rate):int(0.6 * rate)]) np.testing.assert_array_equal(d["utt3"][1], array1[int(0.4 * rate):int(0.5 * rate)]) np.testing.assert_array_equal(d["utt4"][1], array1[int(0.6 * rate):int(0.8 * rate)])
def test_read_wav(tmpdir, func, dtype): path = tmpdir.mkdir('test') wav = path.join('a.wav').strpath # Write as pcm16 array = np.random.randint(0, 10, 10, dtype=dtype) write_wav(wav, 8000, array) with open(wav, 'rb') as f: rate, array2 = func(f) np.testing.assert_array_equal(array, array2)
def test_load_wav_scp(tmpdir, func, dtype): path = tmpdir.mkdir('test') wav = path.join('a.wav').strpath scp = path.join('wav.scp').strpath # Write as pcm16 array = np.random.randint(0, 10, 10, dtype=dtype) write_wav(wav, 8000, array) with open(scp, 'w') as f: f.write('aaa {wav}\n'.format(wav=wav)) rate, array2 = list(dict(func(scp)).values())[0] np.testing.assert_array_equal(array, array2)
def test_scpwav_stream(tmpdir, func, dtype): path = tmpdir.mkdir('test') wav = path.join('aaa.wav').strpath wav2 = path.join('bbb.wav').strpath scp = path.join('wav.scp').strpath # Write as pcm16 array = np.random.randint(0, 10, 40, dtype=dtype).reshape(5, 8) write_wav(wav, 8000, array) array2 = np.random.randint(0, 10, 10, dtype=dtype) write_wav(wav2, 8000, array2) with open(scp, 'w') as f: f.write('aaa sox {wav} -t wav - |\n'.format(wav=wav)) f.write('bbb cat {wav} |\n'.format(wav=wav2)) rate, test = dict(func(scp))['aaa'] rate, test2 = dict(func(scp))['bbb'] np.testing.assert_array_equal(array, test) np.testing.assert_array_equal(array2, test2)
def test_scpwav_stream(tmpdir, func, dtype): path = tmpdir.mkdir("test") wav = path.join("aaa.wav").strpath wav2 = path.join("bbb.wav").strpath scp = path.join("wav.scp").strpath # Write as pcm16 array = np.random.randint(0, 10, 40, dtype=dtype).reshape(5, 8) write_wav(wav, 8000, array) array2 = np.random.randint(0, 10, 10, dtype=dtype) write_wav(wav2, 8000, array2) with open(scp, "w") as f: f.write("aaa sox {wav} -t wav - |\n".format(wav=wav)) f.write("bbb cat {wav} |\n".format(wav=wav2)) rate, test = dict(func(scp))["aaa"] rate, test2 = dict(func(scp))["bbb"] np.testing.assert_array_equal(array, test) np.testing.assert_array_equal(array2, test2)
def test_scpwav_stream(tmpdir): path = tmpdir.mkdir("test") wav = path.join("aaa.wav").strpath wav2 = path.join("bbb.wav").strpath scp = path.join("wav.scp").strpath # Write as pcm16 array = numpy.random.randint(0, 10, 10, dtype=numpy.int16) write_wav(wav, 8000, array) array2 = numpy.random.randint(0, 10, 10, dtype=numpy.int16) write_wav(wav2, 8000, array2) valid = {"aaa": array, "bbb": array2} with open(scp, "w") as f: f.write("aaa cat {wav} |\n".format(wav=wav)) f.write("bbb cat {wav} |\n".format(wav=wav2)) with ReadHelper("scp:{}".format(scp)) as r: for k, (rate, array) in r: assert rate == 8000 numpy.testing.assert_array_equal(array, valid[k])
def test_scpwav_stream(tmpdir): path = tmpdir.mkdir('test') wav = path.join('aaa.wav').strpath wav2 = path.join('bbb.wav').strpath scp = path.join('wav.scp').strpath # Write as pcm16 array = numpy.random.randint(0, 10, 10, dtype=numpy.int16) write_wav(wav, 8000, array) array2 = numpy.random.randint(0, 10, 10, dtype=numpy.int16) write_wav(wav2, 8000, array2) valid = {'aaa': array, 'bbb': array2} with open(scp, 'w') as f: f.write('aaa cat {wav} |\n'.format(wav=wav)) f.write('bbb cat {wav} |\n'.format(wav=wav2)) with ReadHelper('scp:{}'.format(scp)) as r: for k, (rate, array) in r: assert rate == 8000 numpy.testing.assert_array_equal(array, valid[k])
def test_segments(tmpdir): # Create wav.scp path = tmpdir.mkdir("test") wavscp = path.join("wav.scp").strpath rate = 500 with open(wavscp, "w") as f: wav = path.join("0.wav").strpath array0 = numpy.random.randint(0, 10, 2000, dtype=numpy.int16) write_wav(wav, rate, array0) f.write("wav0 {}\n".format(wav)) wav = path.join("1.wav").strpath array1 = numpy.random.randint(0, 10, 2000, dtype=numpy.int16) write_wav(wav, rate, array1) f.write("wav1 {}\n".format(wav)) # Create segments segments = path.join("segments").strpath with open(segments, "w") as f: f.write("utt1 wav0 0.1 0.2\n") f.write("utt2 wav0 0.4 0.6\n") f.write("utt3 wav1 0.4 0.5\n") f.write("utt4 wav1 0.6 0.8\n") with ReadHelper("scp:{}".format(wavscp), segments=segments) as r: d = {k: a for k, a in r} numpy.testing.assert_array_equal( d["utt1"][1], array0[int(0.1 * rate):int(0.2 * rate)]) numpy.testing.assert_array_equal( d["utt2"][1], array0[int(0.4 * rate):int(0.6 * rate)]) numpy.testing.assert_array_equal( d["utt3"][1], array1[int(0.4 * rate):int(0.5 * rate)]) numpy.testing.assert_array_equal( d["utt4"][1], array1[int(0.6 * rate):int(0.8 * rate)])
def test_segments(tmpdir): # Create wav.scp path = tmpdir.mkdir('test') wavscp = path.join('wav.scp').strpath rate = 500 with open(wavscp, 'w') as f: wav = path.join('0.wav').strpath array0 = numpy.random.randint(0, 10, 2000, dtype=numpy.int16) write_wav(wav, rate, array0) f.write('wav0 {}\n'.format(wav)) wav = path.join('1.wav').strpath array1 = numpy.random.randint(0, 10, 2000, dtype=numpy.int16) write_wav(wav, rate, array1) f.write('wav1 {}\n'.format(wav)) # Create segments segments = path.join('segments').strpath with open(segments, 'w') as f: f.write('utt1 wav0 0.1 0.2\n') f.write('utt2 wav0 0.4 0.6\n') f.write('utt3 wav1 0.4 0.5\n') f.write('utt4 wav1 0.6 0.8\n') with ReadHelper('scp:{}'.format(wavscp), segments=segments) as r: d = {k: a for k, a in r} numpy.testing.assert_array_equal( d['utt1'][1], array0[int(0.1 * rate):int(0.2 * rate)]) numpy.testing.assert_array_equal( d['utt2'][1], array0[int(0.4 * rate):int(0.6 * rate)]) numpy.testing.assert_array_equal( d['utt3'][1], array1[int(0.4 * rate):int(0.5 * rate)]) numpy.testing.assert_array_equal( d['utt4'][1], array1[int(0.6 * rate):int(0.8 * rate)])
def save_ark(ark, array_dict, scp=None, append=False, text=False, as_bytes=False, endian='<', compression_method=None): """Write ark Args: ark (str or fd): array_dict (dict): scp (str or fd): append (bool): If True is specified, open the file with appendable mode text (bool): If True, saving in text ark format. as_bytes (bool): Save the value of the input array_dict as just a bytes string. endian (str): compression_method (int): """ if isinstance(ark, string_types): seekable = True # Maybe, never match with this elif not hasattr(ark, 'tell'): seekable = False else: try: ark.tell() seekable = True except Exception: seekable = False if scp is not None and not isinstance(ark, string_types): if not seekable: raise TypeError('scp file can be created only ' 'if the output ark file is a file or ' 'a seekable file descriptor.') # Write ark mode = 'ab' if append else 'wb' pos_list = [] with open_or_fd(ark, mode) as fd: if seekable: offset = fd.tell() else: offset = 0 size = 0 for key in array_dict: encode_key = (key + ' ').encode(encoding=default_encoding) fd.write(encode_key) size += len(encode_key) pos_list.append(size) if as_bytes: byte = bytes(array_dict[key]) size += len(byte) fd.write(byte) else: data = array_dict[key] if isinstance(data, (list, tuple)): rate, array = data size += write_wav(fd, rate, array) elif text: size += write_array_ascii(fd, data, endian) else: size += write_array(fd, data, endian, compression_method) # Write scp mode = 'a' if append else 'w' if scp is not None: name = ark if isinstance(ark, string_types) else ark.name with open_or_fd(scp, mode) as fd: for key, position in zip(array_dict, pos_list): fd.write(key + u' ' + name + ':' + str(position + offset) + '\n')
def save_ark( ark, array_dict, scp=None, append=False, text=False, endian="<", compression_method=None, write_function=None, ): """Write ark Args: ark (str or fd): array_dict (dict): scp (str or fd): append (bool): If True is specified, open the file with appendable mode text (bool): If True, saving in text ark format. endian (str): compression_method (int): write_function: (str): """ if isinstance(ark, string_types): seekable = True # Maybe, never match with this elif not hasattr(ark, "tell"): seekable = False else: try: ark.tell() seekable = True except Exception: seekable = False if scp is not None and not isinstance(ark, string_types): if not seekable: raise TypeError("scp file can be created only " "if the output ark file is a file or " "a seekable file descriptor.") # Write ark mode = "ab" if append else "wb" pos_list = [] with open_or_fd(ark, mode) as fd: if seekable: offset = fd.tell() else: offset = 0 size = 0 for key in array_dict: encode_key = (key + " ").encode(encoding=default_encoding) fd.write(encode_key) size += len(encode_key) pos_list.append(size) data = array_dict[key] if write_function is not None: # Ignore case write_function = write_function.lower() if write_function.startswith("soundfile"): import soundfile if "flac" in write_function: audio_format = "flac" elif "wav" in write_function: audio_format = "wav" else: audio_format = "wav" def _write_function(fd, data): if not isinstance(data, (list, tuple)): raise TypeError( "Expected list or tuple type, but got {}". format(type(data))) elif len(data) != 2: raise ValueError( "Expected length=2, bot got {}".format( len(data))) _fd = BytesIO() if isinstance(data[0], np.ndarray) and isinstance( data[1], int): soundfile.write(_fd, data[0], data[1], format=audio_format) elif isinstance(data[1], np.ndarray) and isinstance( data[0], int): soundfile.write(_fd, data[1], data[0], format=audio_format) else: raise ValueError( "Expected Tuple[int, np.ndarray] or " "Tuple[np.ndarray, int]: " "but got Tuple[{}, {}]".format( type(data[0]), type(data[1]))) fd.write(b"AUDIO") buf = _fd.getbuffer() # Write the information for the length bytes_length = _write_length_header(fd, len(buf)) fd.write(buf) return len(buf) + len(b"AUDIO") + bytes_length elif write_function == "pickle": def _write_function(fd, data): # Note that we don't need size information for pickle! fd.write(b"PKL") _fd = BytesIO() pickle.dump(data, _fd) buf = _fd.getbuffer() fd.write(buf) return len(buf) + len("PKL") elif write_function == "numpy": def _write_function(fd, data): # Write numpy file in BytesIO _fd = BytesIO() np.save(_fd, data) fd.write(b"NPY") buf = _fd.getbuffer() # Write the information for the length bytes_length = _write_length_header(fd, len(buf)) # Write numpy to real file object fd.write(buf) return len(buf) + len(b"NPY") + bytes_length else: raise RuntimeError( "Not supported: write_function={}".format( write_function)) size += _write_function(fd, data) elif isinstance(data, (list, tuple)): rate, array = data size += write_wav(fd, rate, array) elif text: size += write_array_ascii(fd, data, endian) else: size += write_array(fd, data, endian, compression_method) # Write scp mode = "a" if append else "w" if scp is not None: name = ark if isinstance(ark, string_types) else ark.name with open_or_fd(scp, mode) as fd: for key, position in zip(array_dict, pos_list): fd.write(key + " " + name + ":" + str(position + offset) + "\n")