def pronunciation_assessment_from_microphone():
""""performs one-shot pronunciation assessment asynchronously with input from microphone."""
# Creates an instance of a speech config with specified subscription key and service region.
# Replace with your own subscription key and service region (e.g., "westus").
# Note: The pronunciation assessment feature is currently only available on westus, eastasia and centralindia regions.
# And this feature is currently only available on en-US language.
config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region)
reference_text = ""
# create pronunciation assessment config, set grading system, granularity and if enable miscue based on your requirement.
pronunciation_config = speechsdk.PronunciationAssessmentConfig(reference_text=reference_text,
grading_system=speechsdk.PronunciationAssessmentGradingSystem.HundredMark,
granularity=speechsdk.PronunciationAssessmentGranularity.Phoneme,
enable_miscue=True)
recognizer = speechsdk.SpeechRecognizer(speech_config=config)
while True:
# Receives reference text from console input.
print('Enter reference text you want to assess, or enter empty text to exit.')
print('> ')
try:
reference_text = input()
except EOFError:
break
pronunciation_config.reference_text = reference_text
pronunciation_config.apply_to(recognizer)
# Starts recognizing.
print('Read out "{}" for pronunciation assessment ...'.format(reference_text))
# Note: Since recognize_once() returns only a single utterance, it is suitable only for single
# shot evaluation.
# For long-running multi-utterance pronunciation evaluation, use start_continuous_recognition() instead.
result = recognizer.recognize_once_async().get()
# Check the result
if result.reason == speechsdk.ResultReason.RecognizedSpeech:
print('Recognized: {}'.format(result.text))
print(' Pronunciation Assessment Result:')
pronunciation_result = speechsdk.PronunciationAssessmentResult(result)
print(' Accuracy score: {}, Pronunciation score: {}, Completeness score : {}, FluencyScore: {}'.format(
pronunciation_result.accuracy_score, pronunciation_result.pronunciation_score,
pronunciation_result.completeness_score, pronunciation_result.fluency_score
))
print(' Word-level details:')
for idx, word in enumerate(pronunciation_result.words):
print(' {}: word: {}, accuracy score: {}, error type: {};'.format(
idx + 1, word.word, word.accuracy_score, word.error_type
))
elif result.reason == speechsdk.ResultReason.NoMatch:
print("No speech could be recognized")
elif result.reason == speechsdk.ResultReason.Canceled:
cancellation_details = result.cancellation_details
print("Speech Recognition canceled: {}".format(cancellation_details.reason))
if cancellation_details.reason == speechsdk.CancellationReason.Error:
print("Error details: {}".format(cancellation_details.error_details))
def pronunciation_assessment_continuous_from_file():
"""
Performs continuous speech recognition asynchronously with input from an audio file
See more information at https://aka.ms/csspeech/pa
"""
import difflib
import json
# Creates an instance of a speech config with specified subscription key and service region.
# Replace with your own subscription key and service region (e.g., "westus").
# Note: The pronunciation assessment feature is currently only available on en-US language.
speech_config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region)
audio_config = speechsdk.audio.AudioConfig(filename=weatherfilename)
reference_text = "What's the weather like?"
# create pronunciation assessment config, set grading system, granularity and if enable miscue based on your requirement.
enable_miscue = True
pronunciation_config = speechsdk.PronunciationAssessmentConfig(
reference_text=reference_text,
grading_system=speechsdk.PronunciationAssessmentGradingSystem.HundredMark,
granularity=speechsdk.PronunciationAssessmentGranularity.Phoneme,
enable_miscue=enable_miscue)
# Creates a speech recognizer using a file as audio input.
language = 'en-US'
speech_recognizer = speechsdk.SpeechRecognizer(speech_config=speech_config, language=language, audio_config=audio_config)
# apply pronunciation assessment config to speech recognizer
pronunciation_config.apply_to(speech_recognizer)
done = False
recognized_words = []
accuracy_scores = []
durations = []
valid_durations = []
start_offset, end_offset = None, None
def stop_cb(evt: speechsdk.SessionEventArgs):
"""callback that signals to stop continuous recognition upon receiving an event `evt`"""
print('CLOSING on {}'.format(evt))
nonlocal done
done = True
def recognized(evt: speechsdk.SpeechRecognitionEventArgs):
print('pronunciation assessment for: {}'.format(evt.result.text))
pronunciation_result = speechsdk.PronunciationAssessmentResult(evt.result)
print(' Accuracy score: {}, pronunciation score: {}, completeness score : {}, fluency score: {}'.format(
pronunciation_result.accuracy_score, pronunciation_result.pronunciation_score,
pronunciation_result.completeness_score, pronunciation_result.fluency_score
))
nonlocal recognized_words, accuracy_scores, durations, valid_durations, start_offset, end_offset
recognized_words += pronunciation_result.words
accuracy_scores.append(pronunciation_result.accuracy_score)
json_result = evt.result.properties.get(speechsdk.PropertyId.SpeechServiceResponse_JsonResult)
jo = json.loads(json_result)
nb = jo['NBest'][0]
durations.append(sum([int(w['Duration']) for w in nb['Words']]))
if start_offset is None:
start_offset = nb['Words'][0]['Offset']
end_offset = nb['Words'][-1]['Offset'] + nb['Words'][-1]['Duration'] + 100000
for w, d in zip(pronunciation_result.words, nb['Words']):
if w.error_type == 'None':
valid_durations.append(d['Duration'] + 100000)
# Connect callbacks to the events fired by the speech recognizer
speech_recognizer.recognized.connect(recognized)
speech_recognizer.session_started.connect(lambda evt: print('SESSION STARTED: {}'.format(evt)))
speech_recognizer.session_stopped.connect(lambda evt: print('SESSION STOPPED {}'.format(evt)))
speech_recognizer.canceled.connect(lambda evt: print('CANCELED {}'.format(evt)))
# stop continuous recognition on either session stopped or canceled events
speech_recognizer.session_stopped.connect(stop_cb)
speech_recognizer.canceled.connect(stop_cb)
# Start continuous pronunciation assessment
speech_recognizer.start_continuous_recognition()
while not done:
time.sleep(.5)
speech_recognizer.stop_continuous_recognition()
# We can calculate whole accuracy and fluency scores by duration weighted averaging
accuracy_score = sum(i[0] * i[1] for i in zip(accuracy_scores, durations)) / sum(durations)
# Re-calculate fluency score
if start_offset is not None:
fluency_score = sum(valid_durations) / (end_offset - start_offset) * 100
# we need to convert the reference text to lower case, and split to words, then remove the punctuations.
if language == 'zh-CN':
# Use jieba package to split words for Chinese
import jieba
import zhon.hanzi
jieba.suggest_freq([x.word for x in recognized_words], True)
reference_words = [w for w in jieba.cut(reference_text) if w not in zhon.hanzi.punctuation]
else:
reference_words = [w.strip(string.punctuation) for w in reference_text.lower().split()]
# For continuous pronunciation assessment mode, the service won't return the words with `Insertion` or `Omission`
# even if miscue is enabled.
# We need to compare with the reference text after received all recognized words to get these error words.
if enable_miscue:
diff = difflib.SequenceMatcher(None, reference_words, [x.word for x in recognized_words])
final_words = []
for tag, i1, i2, j1, j2 in diff.get_opcodes():
if tag in ['insert', 'replace']:
for word in recognized_words[j1:j2]:
if word.error_type == 'None':
word._error_type = 'Insertion'
final_words.append(word)
if tag in ['delete', 'replace']:
for word_text in reference_words[i1:i2]:
word = speechsdk.PronunciationAssessmentWordResult({
'Word': word_text,
'PronunciationAssessment': {
'ErrorType': 'Omission',
}
})
final_words.append(word)
if tag == 'equal':
final_words += recognized_words[j1:j2]
else:
final_words = recognized_words
# Calculate whole completeness score
completeness_score = len([w for w in final_words if w.error_type == 'None']) / len(reference_words) * 100
print(' Paragraph accuracy score: {}, completeness score: {}, fluency score: {}'.format(
accuracy_score, completeness_score, fluency_score
))
for idx, word in enumerate(final_words):
print(' {}: word: {}\taccuracy score: {}\terror type: {};'.format(
idx + 1, word.word, word.accuracy_score, word.error_type
))
def pronunciation_assessment_from_microphone():
""""
Performs one-shot pronunciation assessment asynchronously with input from microphone.
See more information at https://aka.ms/csspeech/pa
"""
# Creates an instance of a speech config with specified subscription key and service region.
# Replace with your own subscription key and service region (e.g., "westus").
config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region)
# The pronunciation assessment service has a longer default end silence timeout (5 seconds) than normal STT
# as the pronunciation assessment is widely used in education scenario where kids have longer break in reading.
# You can adjust the end silence timeout based on your real scenario.
config.set_property(speechsdk.PropertyId.SpeechServiceConnection_EndSilenceTimeoutMs, "3000")
reference_text = ""
# create pronunciation assessment config, set grading system, granularity and if enable miscue based on your requirement.
pronunciation_config = speechsdk.PronunciationAssessmentConfig(
reference_text=reference_text,
grading_system=speechsdk.PronunciationAssessmentGradingSystem.HundredMark,
granularity=speechsdk.PronunciationAssessmentGranularity.Phoneme,
enable_miscue=True)
# Creates a speech recognizer, also specify the speech language
recognizer = speechsdk.SpeechRecognizer(speech_config=config, language="en-US")
while True:
# Receives reference text from console input.
print('Enter reference text you want to assess, or enter empty text to exit.')
print('> ')
try:
reference_text = input()
except EOFError:
break
pronunciation_config.reference_text = reference_text
pronunciation_config.apply_to(recognizer)
# Starts recognizing.
print('Read out "{}" for pronunciation assessment ...'.format(reference_text))
# Note: Since recognize_once() returns only a single utterance, it is suitable only for single
# shot evaluation.
# For long-running multi-utterance pronunciation evaluation, use start_continuous_recognition() instead.
result = recognizer.recognize_once_async().get()
# Check the result
if result.reason == speechsdk.ResultReason.RecognizedSpeech:
print('Recognized: {}'.format(result.text))
print(' Pronunciation Assessment Result:')
pronunciation_result = speechsdk.PronunciationAssessmentResult(result)
print(' Accuracy score: {}, Pronunciation score: {}, Completeness score : {}, FluencyScore: {}'.format(
pronunciation_result.accuracy_score, pronunciation_result.pronunciation_score,
pronunciation_result.completeness_score, pronunciation_result.fluency_score
))
print(' Word-level details:')
for idx, word in enumerate(pronunciation_result.words):
print(' {}: word: {}, accuracy score: {}, error type: {};'.format(
idx + 1, word.word, word.accuracy_score, word.error_type
))
elif result.reason == speechsdk.ResultReason.NoMatch:
print("No speech could be recognized")
elif result.reason == speechsdk.ResultReason.Canceled:
cancellation_details = result.cancellation_details
print("Speech Recognition canceled: {}".format(cancellation_details.reason))
if cancellation_details.reason == speechsdk.CancellationReason.Error:
print("Error details: {}".format(cancellation_details.error_details))
def pronunciation_assessment_continuous_from_file():
"""performs continuous speech recognition asynchronously with input from an audio file"""
import difflib
# Creates an instance of a speech config with specified subscription key and service region.
# Replace with your own subscription key and service region (e.g., "westus").
# Note: The pronunciation assessment feature is currently only available on westus, eastasia and centralindia regions.
# And this feature is currently only available on en-US language.
speech_config = speechsdk.SpeechConfig(subscription=speech_key, region=service_region)
audio_config = speechsdk.audio.AudioConfig(filename=weatherfilename)
reference_text = "What's the weather like?"
# create pronunciation assessment config, set grading system, granularity and if enable miscue based on your requirement.
enable_miscue = True
pronunciation_config = speechsdk.PronunciationAssessmentConfig(reference_text=reference_text,
grading_system=speechsdk.PronunciationAssessmentGradingSystem.HundredMark,
granularity=speechsdk.PronunciationAssessmentGranularity.Phoneme,
enable_miscue=enable_miscue)
# Creates a speech recognizer using a file as audio input.
# The default language is "en-us".
speech_recognizer = speechsdk.SpeechRecognizer(speech_config=speech_config, audio_config=audio_config)
# apply pronunciation assessment config to speech recognizer
pronunciation_config.apply_to(speech_recognizer)
done = False
recognized_words = []
def stop_cb(evt):
"""callback that signals to stop continuous recognition upon receiving an event `evt`"""
print('CLOSING on {}'.format(evt))
nonlocal done
done = True
def recognized(evt):
print('pronunciation assessment for: {}'.format(evt.result.text))
pronunciation_result = speechsdk.PronunciationAssessmentResult(evt.result)
print(' Accuracy score: {}, Pronunciation score: {}, Completeness score : {}, FluencyScore: {}'.format(
pronunciation_result.accuracy_score, pronunciation_result.pronunciation_score,
pronunciation_result.completeness_score, pronunciation_result.fluency_score
))
nonlocal recognized_words
recognized_words += pronunciation_result.words
# Connect callbacks to the events fired by the speech recognizer
speech_recognizer.recognized.connect(recognized)
speech_recognizer.session_started.connect(lambda evt: print('SESSION STARTED: {}'.format(evt)))
speech_recognizer.session_stopped.connect(lambda evt: print('SESSION STOPPED {}'.format(evt)))
speech_recognizer.canceled.connect(lambda evt: print('CANCELED {}'.format(evt)))
# stop continuous recognition on either session stopped or canceled events
speech_recognizer.session_stopped.connect(stop_cb)
speech_recognizer.canceled.connect(stop_cb)
# Start continuous pronunciation assessment
speech_recognizer.start_continuous_recognition()
while not done:
time.sleep(.5)
speech_recognizer.stop_continuous_recognition()
# For continuous pronunciation assessment mode, the service won't return the words with `Insertion` or `Omission` even if miscue is enabled.
# We need to compare with the reference text after received all recognized words to get these error words.
if enable_miscue:
# we need to convert the reference text to lower case, and split to words, then remove the punctuations.
reference_words = [w.strip(string.punctuation) for w in reference_text.lower().split()]
diff = difflib.SequenceMatcher(None, reference_words, [x.word for x in recognized_words])
final_words = []
for tag, i1, i2, j1, j2 in diff.get_opcodes():
if tag == 'insert':
for word in recognized_words[j1:j2]:
if word.error_type == 'None':
word._error_type = 'Insertion'
final_words.append(word)
elif tag == 'delete':
for word_text in reference_words[i1:i2]:
word = speechsdk.PronunciationAssessmentWordResult({
'Word': word_text,
'PronunciationAssessment': {
'ErrorType': 'Omission',
}
})
final_words.append(word)
else:
final_words += recognized_words[j1:j2]
else:
final_words = recognized_words
for idx, word in enumerate(final_words):
print(' {}: word: {}\taccuracy score: {}\terror type: {};'.format(
idx + 1, word.word, word.accuracy_score, word.error_type
))