def pronunciation_assessment_from_microphone(): """"performs one-shot pronunciation assessment asynchronously with input from microphone.""" # Creates an instance of a speech config with specified subscription key and service region. # Replace with your own subscription key and service region (e.g., "westus"). # Note: The pronunciation assessment feature is currently only available on westus, eastasia and centralindia regions. # And this feature is currently only available on en-US language. config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region) reference_text = "" # create pronunciation assessment config, set grading system, granularity and if enable miscue based on your requirement. pronunciation_config = speechsdk.PronunciationAssessmentConfig(reference_text=reference_text, grading_system=speechsdk.PronunciationAssessmentGradingSystem.HundredMark, granularity=speechsdk.PronunciationAssessmentGranularity.Phoneme, enable_miscue=True) recognizer = speechsdk.SpeechRecognizer(speech_config=config) while True: # Receives reference text from console input. print('Enter reference text you want to assess, or enter empty text to exit.') print('> ') try: reference_text = input() except EOFError: break pronunciation_config.reference_text = reference_text pronunciation_config.apply_to(recognizer) # Starts recognizing. print('Read out "{}" for pronunciation assessment ...'.format(reference_text)) # Note: Since recognize_once() returns only a single utterance, it is suitable only for single # shot evaluation. # For long-running multi-utterance pronunciation evaluation, use start_continuous_recognition() instead. result = recognizer.recognize_once_async().get() # Check the result if result.reason == speechsdk.ResultReason.RecognizedSpeech: print('Recognized: {}'.format(result.text)) print(' Pronunciation Assessment Result:') pronunciation_result = speechsdk.PronunciationAssessmentResult(result) print(' Accuracy score: {}, Pronunciation score: {}, Completeness score : {}, FluencyScore: {}'.format( pronunciation_result.accuracy_score, pronunciation_result.pronunciation_score, pronunciation_result.completeness_score, pronunciation_result.fluency_score )) print(' Word-level details:') for idx, word in enumerate(pronunciation_result.words): print(' {}: word: {}, accuracy score: {}, error type: {};'.format( idx + 1, word.word, word.accuracy_score, word.error_type )) elif result.reason == speechsdk.ResultReason.NoMatch: print("No speech could be recognized") elif result.reason == speechsdk.ResultReason.Canceled: cancellation_details = result.cancellation_details print("Speech Recognition canceled: {}".format(cancellation_details.reason)) if cancellation_details.reason == speechsdk.CancellationReason.Error: print("Error details: {}".format(cancellation_details.error_details))
def pronunciation_assessment_continuous_from_file(): """ Performs continuous speech recognition asynchronously with input from an audio file See more information at https://aka.ms/csspeech/pa """ import difflib import json # Creates an instance of a speech config with specified subscription key and service region. # Replace with your own subscription key and service region (e.g., "westus"). # Note: The pronunciation assessment feature is currently only available on en-US language. speech_config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region) audio_config = speechsdk.audio.AudioConfig(filename=weatherfilename) reference_text = "What's the weather like?" # create pronunciation assessment config, set grading system, granularity and if enable miscue based on your requirement. enable_miscue = True pronunciation_config = speechsdk.PronunciationAssessmentConfig( reference_text=reference_text, grading_system=speechsdk.PronunciationAssessmentGradingSystem.HundredMark, granularity=speechsdk.PronunciationAssessmentGranularity.Phoneme, enable_miscue=enable_miscue) # Creates a speech recognizer using a file as audio input. language = 'en-US' speech_recognizer = speechsdk.SpeechRecognizer(speech_config=speech_config, language=language, audio_config=audio_config) # apply pronunciation assessment config to speech recognizer pronunciation_config.apply_to(speech_recognizer) done = False recognized_words = [] accuracy_scores = [] durations = [] valid_durations = [] start_offset, end_offset = None, None def stop_cb(evt: speechsdk.SessionEventArgs): """callback that signals to stop continuous recognition upon receiving an event `evt`""" print('CLOSING on {}'.format(evt)) nonlocal done done = True def recognized(evt: speechsdk.SpeechRecognitionEventArgs): print('pronunciation assessment for: {}'.format(evt.result.text)) pronunciation_result = speechsdk.PronunciationAssessmentResult(evt.result) print(' Accuracy score: {}, pronunciation score: {}, completeness score : {}, fluency score: {}'.format( pronunciation_result.accuracy_score, pronunciation_result.pronunciation_score, pronunciation_result.completeness_score, pronunciation_result.fluency_score )) nonlocal recognized_words, accuracy_scores, durations, valid_durations, start_offset, end_offset recognized_words += pronunciation_result.words accuracy_scores.append(pronunciation_result.accuracy_score) json_result = evt.result.properties.get(speechsdk.PropertyId.SpeechServiceResponse_JsonResult) jo = json.loads(json_result) nb = jo['NBest'][0] durations.append(sum([int(w['Duration']) for w in nb['Words']])) if start_offset is None: start_offset = nb['Words'][0]['Offset'] end_offset = nb['Words'][-1]['Offset'] + nb['Words'][-1]['Duration'] + 100000 for w, d in zip(pronunciation_result.words, nb['Words']): if w.error_type == 'None': valid_durations.append(d['Duration'] + 100000) # Connect callbacks to the events fired by the speech recognizer speech_recognizer.recognized.connect(recognized) speech_recognizer.session_started.connect(lambda evt: print('SESSION STARTED: {}'.format(evt))) speech_recognizer.session_stopped.connect(lambda evt: print('SESSION STOPPED {}'.format(evt))) speech_recognizer.canceled.connect(lambda evt: print('CANCELED {}'.format(evt))) # stop continuous recognition on either session stopped or canceled events speech_recognizer.session_stopped.connect(stop_cb) speech_recognizer.canceled.connect(stop_cb) # Start continuous pronunciation assessment speech_recognizer.start_continuous_recognition() while not done: time.sleep(.5) speech_recognizer.stop_continuous_recognition() # We can calculate whole accuracy and fluency scores by duration weighted averaging accuracy_score = sum(i[0] * i[1] for i in zip(accuracy_scores, durations)) / sum(durations) # Re-calculate fluency score if start_offset is not None: fluency_score = sum(valid_durations) / (end_offset - start_offset) * 100 # we need to convert the reference text to lower case, and split to words, then remove the punctuations. if language == 'zh-CN': # Use jieba package to split words for Chinese import jieba import zhon.hanzi jieba.suggest_freq([x.word for x in recognized_words], True) reference_words = [w for w in jieba.cut(reference_text) if w not in zhon.hanzi.punctuation] else: reference_words = [w.strip(string.punctuation) for w in reference_text.lower().split()] # For continuous pronunciation assessment mode, the service won't return the words with `Insertion` or `Omission` # even if miscue is enabled. # We need to compare with the reference text after received all recognized words to get these error words. if enable_miscue: diff = difflib.SequenceMatcher(None, reference_words, [x.word for x in recognized_words]) final_words = [] for tag, i1, i2, j1, j2 in diff.get_opcodes(): if tag in ['insert', 'replace']: for word in recognized_words[j1:j2]: if word.error_type == 'None': word._error_type = 'Insertion' final_words.append(word) if tag in ['delete', 'replace']: for word_text in reference_words[i1:i2]: word = speechsdk.PronunciationAssessmentWordResult({ 'Word': word_text, 'PronunciationAssessment': { 'ErrorType': 'Omission', } }) final_words.append(word) if tag == 'equal': final_words += recognized_words[j1:j2] else: final_words = recognized_words # Calculate whole completeness score completeness_score = len([w for w in final_words if w.error_type == 'None']) / len(reference_words) * 100 print(' Paragraph accuracy score: {}, completeness score: {}, fluency score: {}'.format( accuracy_score, completeness_score, fluency_score )) for idx, word in enumerate(final_words): print(' {}: word: {}\taccuracy score: {}\terror type: {};'.format( idx + 1, word.word, word.accuracy_score, word.error_type ))
def pronunciation_assessment_from_microphone(): """" Performs one-shot pronunciation assessment asynchronously with input from microphone. See more information at https://aka.ms/csspeech/pa """ # Creates an instance of a speech config with specified subscription key and service region. # Replace with your own subscription key and service region (e.g., "westus"). config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region) # The pronunciation assessment service has a longer default end silence timeout (5 seconds) than normal STT # as the pronunciation assessment is widely used in education scenario where kids have longer break in reading. # You can adjust the end silence timeout based on your real scenario. config.set_property(speechsdk.PropertyId.SpeechServiceConnection_EndSilenceTimeoutMs, "3000") reference_text = "" # create pronunciation assessment config, set grading system, granularity and if enable miscue based on your requirement. pronunciation_config = speechsdk.PronunciationAssessmentConfig( reference_text=reference_text, grading_system=speechsdk.PronunciationAssessmentGradingSystem.HundredMark, granularity=speechsdk.PronunciationAssessmentGranularity.Phoneme, enable_miscue=True) # Creates a speech recognizer, also specify the speech language recognizer = speechsdk.SpeechRecognizer(speech_config=config, language="en-US") while True: # Receives reference text from console input. print('Enter reference text you want to assess, or enter empty text to exit.') print('> ') try: reference_text = input() except EOFError: break pronunciation_config.reference_text = reference_text pronunciation_config.apply_to(recognizer) # Starts recognizing. print('Read out "{}" for pronunciation assessment ...'.format(reference_text)) # Note: Since recognize_once() returns only a single utterance, it is suitable only for single # shot evaluation. # For long-running multi-utterance pronunciation evaluation, use start_continuous_recognition() instead. result = recognizer.recognize_once_async().get() # Check the result if result.reason == speechsdk.ResultReason.RecognizedSpeech: print('Recognized: {}'.format(result.text)) print(' Pronunciation Assessment Result:') pronunciation_result = speechsdk.PronunciationAssessmentResult(result) print(' Accuracy score: {}, Pronunciation score: {}, Completeness score : {}, FluencyScore: {}'.format( pronunciation_result.accuracy_score, pronunciation_result.pronunciation_score, pronunciation_result.completeness_score, pronunciation_result.fluency_score )) print(' Word-level details:') for idx, word in enumerate(pronunciation_result.words): print(' {}: word: {}, accuracy score: {}, error type: {};'.format( idx + 1, word.word, word.accuracy_score, word.error_type )) elif result.reason == speechsdk.ResultReason.NoMatch: print("No speech could be recognized") elif result.reason == speechsdk.ResultReason.Canceled: cancellation_details = result.cancellation_details print("Speech Recognition canceled: {}".format(cancellation_details.reason)) if cancellation_details.reason == speechsdk.CancellationReason.Error: print("Error details: {}".format(cancellation_details.error_details))
def pronunciation_assessment_continuous_from_file(): """performs continuous speech recognition asynchronously with input from an audio file""" import difflib # Creates an instance of a speech config with specified subscription key and service region. # Replace with your own subscription key and service region (e.g., "westus"). # Note: The pronunciation assessment feature is currently only available on westus, eastasia and centralindia regions. # And this feature is currently only available on en-US language. speech_config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region) audio_config = speechsdk.audio.AudioConfig(filename=weatherfilename) reference_text = "What's the weather like?" # create pronunciation assessment config, set grading system, granularity and if enable miscue based on your requirement. enable_miscue = True pronunciation_config = speechsdk.PronunciationAssessmentConfig(reference_text=reference_text, grading_system=speechsdk.PronunciationAssessmentGradingSystem.HundredMark, granularity=speechsdk.PronunciationAssessmentGranularity.Phoneme, enable_miscue=enable_miscue) # Creates a speech recognizer using a file as audio input. # The default language is "en-us". speech_recognizer = speechsdk.SpeechRecognizer(speech_config=speech_config, audio_config=audio_config) # apply pronunciation assessment config to speech recognizer pronunciation_config.apply_to(speech_recognizer) done = False recognized_words = [] def stop_cb(evt): """callback that signals to stop continuous recognition upon receiving an event `evt`""" print('CLOSING on {}'.format(evt)) nonlocal done done = True def recognized(evt): print('pronunciation assessment for: {}'.format(evt.result.text)) pronunciation_result = speechsdk.PronunciationAssessmentResult(evt.result) print(' Accuracy score: {}, Pronunciation score: {}, Completeness score : {}, FluencyScore: {}'.format( pronunciation_result.accuracy_score, pronunciation_result.pronunciation_score, pronunciation_result.completeness_score, pronunciation_result.fluency_score )) nonlocal recognized_words recognized_words += pronunciation_result.words # Connect callbacks to the events fired by the speech recognizer speech_recognizer.recognized.connect(recognized) speech_recognizer.session_started.connect(lambda evt: print('SESSION STARTED: {}'.format(evt))) speech_recognizer.session_stopped.connect(lambda evt: print('SESSION STOPPED {}'.format(evt))) speech_recognizer.canceled.connect(lambda evt: print('CANCELED {}'.format(evt))) # stop continuous recognition on either session stopped or canceled events speech_recognizer.session_stopped.connect(stop_cb) speech_recognizer.canceled.connect(stop_cb) # Start continuous pronunciation assessment speech_recognizer.start_continuous_recognition() while not done: time.sleep(.5) speech_recognizer.stop_continuous_recognition() # For continuous pronunciation assessment mode, the service won't return the words with `Insertion` or `Omission` even if miscue is enabled. # We need to compare with the reference text after received all recognized words to get these error words. if enable_miscue: # we need to convert the reference text to lower case, and split to words, then remove the punctuations. reference_words = [w.strip(string.punctuation) for w in reference_text.lower().split()] diff = difflib.SequenceMatcher(None, reference_words, [x.word for x in recognized_words]) final_words = [] for tag, i1, i2, j1, j2 in diff.get_opcodes(): if tag == 'insert': for word in recognized_words[j1:j2]: if word.error_type == 'None': word._error_type = 'Insertion' final_words.append(word) elif tag == 'delete': for word_text in reference_words[i1:i2]: word = speechsdk.PronunciationAssessmentWordResult({ 'Word': word_text, 'PronunciationAssessment': { 'ErrorType': 'Omission', } }) final_words.append(word) else: final_words += recognized_words[j1:j2] else: final_words = recognized_words for idx, word in enumerate(final_words): print(' {}: word: {}\taccuracy score: {}\terror type: {};'.format( idx + 1, word.word, word.accuracy_score, word.error_type ))