def test_save_load_pretrained_additional_features(self):
processor = Speech2TextProcessor(
tokenizer=self.get_tokenizer(),
feature_extractor=self.get_feature_extractor())
processor.save_pretrained(self.tmpdirname)
tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)",
eos_token="(EOS)")
feature_extractor_add_kwargs = self.get_feature_extractor(
do_normalize=False, padding_value=1.0)
processor = Speech2TextProcessor.from_pretrained(self.tmpdirname,
bos_token="(BOS)",
eos_token="(EOS)",
do_normalize=False,
padding_value=1.0)
self.assertEqual(processor.tokenizer.get_vocab(),
tokenizer_add_kwargs.get_vocab())
self.assertIsInstance(processor.tokenizer, Speech2TextTokenizer)
self.assertEqual(processor.feature_extractor.to_json_string(),
feature_extractor_add_kwargs.to_json_string())
self.assertIsInstance(processor.feature_extractor,
Speech2TextFeatureExtractor)
def test_save_load_pretrained_default(self):
tokenizer = self.get_tokenizer()
feature_extractor = self.get_feature_extractor()
processor = Speech2TextProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
processor.save_pretrained(self.tmpdirname)
processor = Speech2TextProcessor.from_pretrained(self.tmpdirname)
self.assertEqual(processor.tokenizer.get_vocab(), tokenizer.get_vocab())
self.assertIsInstance(processor.tokenizer, Speech2TextTokenizer)
self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor.to_json_string())
self.assertIsInstance(processor.feature_extractor, Speech2TextFeatureExtractor)
def test_tokenizer_decode(self):
feature_extractor = self.get_feature_extractor()
tokenizer = self.get_tokenizer()
processor = Speech2TextProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
decoded_processor = processor.batch_decode(predicted_ids)
decoded_tok = tokenizer.batch_decode(predicted_ids)
self.assertListEqual(decoded_tok, decoded_processor)
def test_feature_extractor(self):
feature_extractor = self.get_feature_extractor()
tokenizer = self.get_tokenizer()
processor = Speech2TextProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
raw_speech = floats_list((3, 1000))
input_feat_extract = feature_extractor(raw_speech, return_tensors="np")
input_processor = processor(raw_speech, return_tensors="np")
for key in input_feat_extract.keys():
self.assertAlmostEqual(input_feat_extract[key].sum(), input_processor[key].sum(), delta=1e-2)
def s2t_predictions(audio_file):
device = 'cuda' if torch.cuda.is_available() else 'cpu'
audio_array = s2t_audio_to_array(audio_file)
model = Speech2TextForConditionalGeneration.from_pretrained(
"facebook/s2t-small-librispeech-asr").to(device).eval()
processor = Speech2TextProcessor.from_pretrained(
"facebook/s2t-small-librispeech-asr", do_upper_case=True)
features = processor(audio_array, sampling_rate=16000, return_tensors="pt")
input_features = features.input_features.to(device)
attention_mask = features.attention_mask.to(device)
gen_tokens = model.generate(input_ids=input_features)
text = processor.batch_decode(gen_tokens, skip_special_tokens=True)
return text
def test_tokenizer(self):
feature_extractor = self.get_feature_extractor()
tokenizer = self.get_tokenizer()
processor = Speech2TextProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor)
input_str = "This is a test string"
encoded_processor = processor(text=input_str)
encoded_tok = tokenizer(input_str)
for key in encoded_tok.keys():
self.assertListEqual(encoded_tok[key], encoded_processor[key])
def translate(data,
sampling_rate,
pretrained_model_name="facebook/s2t-small-librispeech-asr"):
model = Speech2TextForConditionalGeneration.from_pretrained(
pretrained_model_name)
feature_extractor = Speech2TextFeatureExtractor.from_pretrained(
pretrained_model_name)
tokenizer = Speech2TextTokenizer.from_pretrained(pretrained_model_name)
processor = Speech2TextProcessor(feature_extractor=feature_extractor,
tokenizer=tokenizer)
inputs = processor(data, sampling_rate=sampling_rate, return_tensors="pt")
generated_ids = model.generate(input_ids=inputs["input_features"],
attention_mask=inputs["attention_mask"])
transcription = processor.batch_decode(generated_ids)
return transcription
def default_processor(self):
return Speech2TextProcessor.from_pretrained(
"facebook/s2t-small-librispeech-asr")
import numpy as np
from scipy.io.wavfile import read
from transformers import Speech2TextProcessor, Speech2TextForConditionalGeneration, Speech2TextFeatureExtractor, Speech2TextTokenizer
if __name__ == "__main__":
audio_path = "../data/review#1.wav"
samplerate, data = read(audio_path)
data = (data - np.mean(data)) / np.std(data)
print(samplerate, len(data))
model = Speech2TextForConditionalGeneration.from_pretrained(
"facebook/s2t-small-librispeech-asr")
feature_extractor = Speech2TextFeatureExtractor.from_pretrained(
"facebook/s2t-small-librispeech-asr")
tokenizer = Speech2TextTokenizer.from_pretrained(
"facebook/s2t-small-librispeech-asr")
processor = Speech2TextProcessor(feature_extractor=feature_extractor,
tokenizer=tokenizer)
inputs = processor(data, sampling_rate=samplerate, return_tensors="pt")
generated_ids = model.generate(input_ids=inputs["input_features"],
attention_mask=inputs["attention_mask"])
transcription = processor.batch_decode(generated_ids)
print(transcription)
