def assist(self, text_query):
"""Send a text request to the Assistant and playback the response.
"""
def iter_assist_requests():
dialog_state_in = embedded_assistant_pb2.DialogStateIn(
language_code=self.language_code, conversation_state=b'')
if self.conversation_state:
dialog_state_in.conversation_state = self.conversation_state
config = embedded_assistant_pb2.AssistConfig(
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
encoding='LINEAR16',
sample_rate_hertz=16000,
volume_percentage=0,
),
dialog_state_in=dialog_state_in,
device_config=embedded_assistant_pb2.DeviceConfig(
device_id=self.device_id,
device_model_id=self.device_model_id,
),
text_query=text_query,
)
req = embedded_assistant_pb2.AssistRequest(config=config)
assistant_helpers.log_assist_request_without_audio(req)
yield req
display_text = None
for resp in self.assistant.Assist(iter_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
self.conversation_state = conversation_state
if resp.dialog_state_out.supplemental_display_text:
display_text = resp.dialog_state_out.supplemental_display_text
return display_text
def assistant_triggered(self):
while True:
self.conversation_stream.start_recording()
if (self.conversation_stream.read(10) != b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'):
print(self.conversation_stream.read(10))
logging.info('Heard Sound')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
# logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
# logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
break
if resp.speech_results:
logging.info('Transcript of user request: "%s".',
' '.join(r.transcript
for r in resp.speech_results))
# sys.stdout.flush()
if any(word.lower() in (' '.join(r.transcript for r in resp.speech_results)).lower() for word in hot_phrases):
print('message==|DTriggered')
sys.stdout.flush()
return
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
self.conversation_stream.start_playback()
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected')
self.conversation_stream.stop_recording()
if resp.speech_results:
logging.info('Transcript of user request: "%s".',
' '.join(r.transcript
for r in resp.speech_results))
logging.info('Playing assistant response.')
if len(resp.audio_out.audio_data) > 0:
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json
)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self, text_query):
"""Send a text request to the Assistant and playback the response.
"""
def iter_assist_requests():
config = embedded_assistant_pb2.AssistConfig(
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
encoding='LINEAR16',
sample_rate_hertz=16000,
volume_percentage=0,
),
dialog_state_in=embedded_assistant_pb2.DialogStateIn(
language_code=self.language_code,
conversation_state=self.conversation_state,
is_new_conversation=self.is_new_conversation,
),
device_config=embedded_assistant_pb2.DeviceConfig(
device_id=self.device_id,
device_model_id=self.device_model_id,
),
text_query=text_query,
)
# Continue current conversation with later requests.
self.is_new_conversation = False
if self.display:
config.screen_out_config.screen_mode = PLAYING
req = embedded_assistant_pb2.AssistRequest(config=config)
assistant_helpers.log_assist_request_without_audio(req)
yield req
text_response = None
html_response = None
device_actions_futures = []
for resp in self.assistant.Assist(iter_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.screen_out.data:
html_response = resp.screen_out.data
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
self.conversation_state = conversation_state
if resp.dialog_state_out.supplemental_display_text:
text_response = resp.dialog_state_out.supplemental_display_text
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if len(device_actions_futures):
concurrent.futures.wait(device_actions_futures)
return text_response, html_response
def assist(self, text_query):
"""Send a text request to the Assistant and playback the response.
"""
#Need to start a new conversation stream to allow playback of assistant
self.conversation_stream.start_recording()
self.conversation_stream.stop_recording()
#Set default language to English for the text to speech
def iter_assist_requests():
dialog_state_in = embedded_assistant_pb2.DialogStateIn(
language_code='en-US',
conversation_state=b''
)
if self.conversation_state:
dialog_state_in.conversation_state = self.conversation_state
config = embedded_assistant_pb2.AssistConfig(
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
encoding='LINEAR16',
sample_rate_hertz=16000,
volume_percentage=0,
),
dialog_state_in=dialog_state_in,
device_config=embedded_assistant_pb2.DeviceConfig(
device_id=self.device_id,
device_model_id=self.device_model_id,
),
text_query=text_query,
)
req = embedded_assistant_pb2.AssistRequest(config=config)
assistant_helpers.log_assist_request_without_audio(req)
yield req
self.conversation_stream.start_playback()
display_text = None
for resp in self.assistant.Assist(iter_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
self.conversation_state = conversation_state
if resp.dialog_state_out.supplemental_display_text:
display_text = resp.dialog_state_out.supplemental_display_text
if len(resp.audio_out.audio_data) > 0:
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
self.conversation_stream.close()
return display_text
def assist(self, text_query):
"""Send a text request to the Assistant and playback the response.
"""
def iter_assist_requests():
config = embedded_assistant_pb2.AssistConfig(
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
encoding='LINEAR16',
sample_rate_hertz=16000,
volume_percentage=0,
),
dialog_state_in=embedded_assistant_pb2.DialogStateIn(
language_code=self.language_code,
conversation_state=self.conversation_state,
is_new_conversation=self.is_new_conversation,
),
device_config=embedded_assistant_pb2.DeviceConfig(
device_id=self.device_id,
device_model_id=self.device_model_id,
),
text_query=text_query,
)
# Continue current conversation with later requests.
self.is_new_conversation = False
if self.display:
config.screen_out_config.screen_mode = PLAYING
req = embedded_assistant_pb2.AssistRequest(config=config)
assistant_helpers.log_assist_request_without_audio(req)
yield req
def get_assistant():
# Create an authorized gRPC channel.
http_request = google.auth.transport.requests.Request()
self.credentials.refresh(http_request)
grpc_channel = google.auth.transport.grpc.secure_authorized_channel(
self.credentials, http_request, ASSISTANT_API_ENDPOINT)
_LOGGER.debug('Connecting to %s', ASSISTANT_API_ENDPOINT)
return embedded_assistant_pb2_grpc.EmbeddedAssistantStub(
grpc_channel)
text_response = None
html_response = None
assistant = get_assistant()
for resp in assistant.Assist(iter_assist_requests(), self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.screen_out.data:
html_response = resp.screen_out.data
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
self.conversation_state = conversation_state
if resp.dialog_state_out.supplemental_display_text:
text_response = resp.dialog_state_out.supplemental_display_text
return text_response, html_response
def assist(self, text_query):
"""Send a text request to the Assistant and playback the response.
"""
def iter_assist_requests():
config = embedded_assistant_pb2.AssistConfig(
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
encoding='LINEAR16',
sample_rate_hertz=16000,
volume_percentage=0,
),
dialog_state_in=embedded_assistant_pb2.DialogStateIn(
language_code=self.language_code,
conversation_state=self.conversation_state,
is_new_conversation=self.is_new_conversation,
),
device_config=embedded_assistant_pb2.DeviceConfig(
device_id=self.device_id,
device_model_id=self.device_model_id,
),
text_query=text_query,
)
# Continue current conversation with later requests.
self.is_new_conversation = False
if self.display:
config.screen_out_config.screen_mode = PLAYING
req = embedded_assistant_pb2.AssistRequest(config=config)
# This can be used to output the assistant request
# assistant_helpers.log_assist_request_without_audio(req)
assistant_helpers.log_assist_request_without_audio(req)
yield req
text_response = None
html_response = None
for resp in self.assistant.Assist(iter_assist_requests(),
self.deadline):
# This can be used to output the assistant response
# assistant_helpers.log_assist_response_without_audio(resp)
assistant_helpers.log_assist_response_without_audio(resp)
if resp.screen_out.data:
html_response = resp.screen_out.data
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
self.conversation_state = conversation_state
if resp.dialog_state_out.supplemental_display_text:
text_response = resp.dialog_state_out.supplemental_display_text(
)
return text_response, html_response
def assist_text(self, text_query):
"""Send a text request to the Assistant and receive text back.
"""
def iter_assist_requests():
config = embedded_assistant_pb2.AssistConfig(
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
encoding='LINEAR16',
sample_rate_hertz=16000,
volume_percentage=0,
),
dialog_state_in=embedded_assistant_pb2.DialogStateIn(
language_code=self.language_code,
conversation_state=self.conversation_state,
is_new_conversation=self.is_new_conversation,
),
device_config=embedded_assistant_pb2.DeviceConfig(
device_id=self.device_id,
device_model_id=self.device_model_id,
),
text_query=text_query,
)
# Continue current conversation with later requests.
self.is_new_conversation = False
if self.display:
config.screen_out_config.screen_mode = PLAYING
req = embedded_assistant_pb2.AssistRequest(config=config)
assistant_helpers.log_assist_request_without_audio(req)
yield req
text_response = 'Google Assistant gave no response'
html_response = None
for resp in self.assistant.Assist(iter_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.screen_out.data:
html_response = resp.screen_out.data
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
self.conversation_state = conversation_state
if resp.dialog_state_out.supplemental_display_text:
text_response = resp.dialog_state_out.supplemental_display_text
if any(p in text_response.lower()
for p in ['public ip', 'ip address', '::; 1']):
text_response = 'I need permission to display that information'
return text_response, html_response
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
#사용자 추가 액션여부
isUserAction = False
led.turnOn(Colors.BLUE)
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
speech = (''.join(r.transcript for r in resp.speech_results)).replace(' ', '')
logging.info('Transcript of user request: "%s".', speech)
if speech in ['전등꺼', '불꺼', '꺼주세요', '빨간색', '초록색', '흰색']:
isUserAction = True
if speech in ['전등꺼', '불꺼', '꺼주세요']:
led.turnOff()
self.currentColor = None
if speech == '빨간색':
led.turnOn(Colors.RED)
self.currentColor = Colors.RED
if speech == '초록색':
led.turnOn(Colors.GREEN)
self.currentColor = Colors.GREEN
if speech == '흰색':
led.turnOn(Colors.WHITE)
self.currentColor = Colors.WHITE
if len(resp.audio_out.audio_data) > 0 and isUserAction == False:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json
)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
if self.currentColor == None:
led.turnOff()
else:
led.turnOn(self.currentColor)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self, commands=None, is_respon=True):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
moving = False
device_actions_futures = []
# ----
stt_list = []
stt_tmp = ''
# ----
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
# voice recognition end
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
# stt
if resp.speech_results:
stt_tmp = ' '.join(r.transcript for r in resp.speech_results)
#logging.info('Transcript of user request: %s.',
# ' '.join(r.transcript
# for r in resp.speech_results))
if commands != None:
for command in commands:
if command in stt_tmp:
moving = True
break
# response
if moving == False and is_respon == True:
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation, stt_tmp
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
face_flag = False
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
logging.info(
'Transcript of user request: "%s".',
' '.join(r.transcript for r in resp.speech_results))
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.supplemental_display_text:
#logging.info('Assistant display text: "%s"',
# resp.dialog_state_out.supplemental_display_text)
string = resp.dialog_state_out.supplemental_display_text
#logging.info("output data: \"%s\"", string)
target_string = "face recognition"
#logging.info(string)
if target_string in string.lower():
logging.info("Executing face recognition!!!!!!")
#os.system('ls -l /home/pi/MagicMirror/modules/third_party/FaceRecognition/')
#os.system('. /home/pi/MagicMirror/modules/third_party/FaceRecognition/my_import.sh')
#os.system('cd /home/pi/MagicMirror/modules/third_party/FaceRecognition/')
#os.system('ls -l')
#os.system('./my_recog_run.sh')
face_flag = True
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
if face_flag:
face_flag = False
logging.info("Executing face recognition!!!!!!")
os.system('./my_recog_run.sh')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self, text_query=None):
"""Send a voice request to the Assistant and playback the response.
:returns: True if conversation should continue.
:rtype: boolean
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
if (text_query):
config = embedded_assistant_pb2.AssistConfig(
audio_out_config=embedded_assistant_pb2.AudioOutConfig(
encoding='LINEAR16',
sample_rate_hertz=16000,
volume_percentage=0,
),
dialog_state_in=embedded_assistant_pb2.DialogStateIn(
language_code=self.language_code,
conversation_state=self.conversation_state,
is_new_conversation=self.is_new_conversation,
),
device_config=embedded_assistant_pb2.DeviceConfig(
device_id=self.device_id,
device_model_id=self.device_model_id,
),
text_query=text_query,
)
self.is_new_conversation = False
req = embedded_assistant_pb2.AssistRequest(config=config)
assistant_helpers.log_assist_request_without_audio(req)
yield req
else:
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
logging.info(
'Transcript of user request: "%s".',
' '.join(r.transcript for r in resp.speech_results))
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
previous_event_type = 0
for resp in self.assistant.Assist(iter_log_assist_requests(), self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
print('End of audio request detected.')
print('Stopping recording.')
print("test")
self.conversation_stream.stop_recording()
if resp.speech_results:
if previous_event_type == END_OF_UTTERANCE:
print(resp.speech_results[0].transcript)
transcript = str(resp.speech_results[0].transcript).lower()
[query, itemType] = audioIsSpotifySearch(transcript)
if query != "":
data = {"query": query, "type":itemType}
client.publish("smartmirror/search", str(data))
break
elif transcript.find("skip song") != -1:
client.publish("smartmirror/skip", "")
break
elif transcript.find("pause music") != -1:
client.publish("smartmirror/pause", "")
break
elif transcript.find("play music") != -1:
client.publish("smartmirror/resume", "")
break
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
print('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json
)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
previous_event_type = resp.event_type
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('무엇을 도와드릴까요? :)')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
myscript = ''.join(r.transcript for r in resp.speech_results)
logging.info('Transcript of user request: "%s".', myscript)
# auto_control
if '자동 제어' in myscript:
client.publish("home/auto", "1")
return continue_conversation
if '수동 제어' in myscript:
client.publish("home/auto", "0")
return continue_conversation
# bathroom_control
if '화장실 불 켜' in myscript:
client.publish("home/bathroom_state/led", "1")
return continue_conversation
if '화장실 불 꺼' in myscript:
client.publish("home/bathroom_state/led", "0")
return continue_conversation
# room_control
if '방 불 켜' in myscript:
client.publish("home/room_state/room_led", "1")
return continue_conversation
if '방 불 꺼' in myscript:
client.publish("home/room_state/room_led", "0")
return continue_conversation
# livingroom_control
if '거실 불 켜' in myscript:
client.publish("home/livingroom_state/livingroom_led", "1")
return continue_conversation
if '거실 불 꺼' in myscript:
client.publish("home/livingroom_state/livingroom_led", "0")
return continue_conversation
if '블라인드 열어' in myscript:
client.publish("home/livingroom_state/blind", "1")
return continue_conversation
if '블라인드 닫아' in myscript:
client.publish("home/livingroom_state/blind", "0")
return continue_conversation
if '에어컨 켜' in myscript:
client.publish("home/livingroom_state/aircon", "1")
return continue_conversation
if '에어컨 꺼' in myscript:
client.publish("home/livingroom_state/aircon", "0")
return continue_conversation
if '히터 켜' in myscript:
client.publish("home/livingroom_state/heater", "1")
return continue_conversation
if '히터 꺼' in myscript:
client.publish("home/livingroom_state/heater", "0")
return continue_conversation
if '제습기 켜' in myscript:
client.publish("home/livingroom_state/airdry", "1")
return continue_conversation
if '제습기 꺼' in myscript:
client.publish("home/livingroom_state/airdry", "0")
return continue_conversation
if '환풍기 켜' in myscript:
client.publish("home/livingroom_state/fan", "1")
return continue_conversation
if '환풍기 꺼' in myscript:
client.publish("home/livingroom_state/fan", "0")
return continue_conversation
# entrance_control
if '차고 문 열어' in myscript:
client.publish("home/entrance_state/garage_door", "1")
return continue_conversation
if '차고 문 닫아' in myscript:
client.publish("home/entrance_state/garage_door", "0")
return continue_conversation
if '현관문 열어' in myscript:
client.publish("home/entrance_state/door", "1")
return continue_conversation
if '현관문 닫아' in myscript:
client.publish("home/entrance_state/door", "0")
return continue_conversation
if '정원 불 켜' in myscript:
client.publish("home/entrance_state/garden_led", "1")
return continue_conversation
if '정원 불 꺼' in myscript:
client.publish("home/entrance_state/garden_led", "0")
return continue_conversation
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
global spokenAnswerOld
global spokenAnswer
global followUp
global followUpSentence
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
logging.info(
'Transcript of user request: "%s".',
' '.join(r.transcript for r in resp.speech_results))
#Function to store the complete spoken text to use as input for NLP model
self.storeAsText(resp.speech_results)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
self.conversation_state = conversation_state
if resp.dialog_state_out.supplemental_display_text:
display_text = resp.dialog_state_out.supplemental_display_text
spokenAnswer = display_text
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
self.spokenQuestion = str(self.spokenQuestion)
# I only want the text
self.spokenQuestion = re.findall(r'"(.+?)"',
self.spokenQuestion)
text = str(self.spokenQuestion[0]).lower()
# Sometimes the SDK can't recongize "Reachy"
matches = [
"reachy", "richie", "reach", "ritchie", "re", "read",
"rich", 'rici', 'robot', 'robots'
]
text = "\"" + text + "\""
if (text != ""):
results = predictor.predict(sentence=text)
if (followUp == True):
if (text == "\"yes\""):
if (followUpSentence == "cross"):
self.cross_arms(robot)
if (followUpSentence == "raise"):
self.raise_hand(robot)
followUp = False
else:
#If our NLP model can classify the sentence
if (results["verbs"]):
semantic_labeling = results["verbs"][0][
"description"]
print(semantic_labeling)
subject = self.find_between(
semantic_labeling, "[ARG0: ", "]")
verb = self.find_between(
semantic_labeling, "[V: ", "]")
part = self.find_between(
semantic_labeling, "[ARG1: ", "]")
print(part)
#If we recognize a sentence to move on
if (verb in dict_bewegen.keys()):
#If Reachy is the subject
if (subject or part in matches):
if (verb == "cross"):
self.cross_arms(robot)
if (verb == "raise"):
self.raise_hand(robot)
#But Reachy is not the subject...
else:
synthesizer.speak_text_async(
"Do you want me to %s %s" %
(verb, part))
followUpSentence = verb
followUp = True
#Just play the googla assistant SDK's answer
else:
if (spokenAnswer == ""):
synthesizer.speak_text_async(
"I can't answer to that question, i'm sorry."
)
else:
synthesizer.speak_text_async(
str(spokenAnswer))
logging.info('Playing assistant response.')
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
print("Recording...")
sys.stdout.flush()
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
print('end')
sys.stdout.flush()
self.conversation_stream.stop_recording()
looptime = 0
if resp.speech_results:
looptime = 1
print('~in ' + ' '.join(r.transcript
for r in resp.speech_results))
sys.stdout.flush()
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
#logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if looptime == 2:
data = '~out ' + resp.dialog_state_out.supplemental_display_text
dataencode = data.encode("utf-8")
print(dataencode)
sys.stdout.flush()
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
looptime = looptime + 1
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
#print('Finished playing assistant response')
sys.stdout.flush()
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self, text_query, flag):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
display_text = None
def iter_log_assist_requests(text_query, flag):
for c in self.gen_assist_requests(text_query, flag):
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(
iter_log_assist_requests(text_query, flag), self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.dialog_state_out.supplemental_display_text:
display_text = resp.dialog_state_out.supplemental_display_text
print(display_text)
if resp.speech_results:
voice_query = ' '.join(r.transcript
for r in resp.speech_results)
print(voice_query)
if '이게 뭐야' in voice_query:
# vision code
self.conversation_stream.stop_playback()
res_vision = myvision.getText(1)
return self.assist(res_vision, 1)
# self.assistant.assist('a',0)
if '이게 누구야' in voice_query:
# vision code
self.conversation_stream.stop_playback()
res_vision = myvision.getText(0)
return self.assist(res_vision, 1)
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
text_response = None
user_input = None
rthing = None
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
#print (resp.device_action.device_request_json)
#print (type(resp.device_action.device_request_json))
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
user_input = ' '.join(r.transcript for r in resp.speech_results)
print('Transcript of user request: "%s".',
' '.join(r.transcript
for r in resp.speech_results))
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
#self.conversation_stream.start_playback()
#print('Playing assistant response.')
#print (resp)
#self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
print('Updating conversation state.')
#self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
print('Setting volume to %s%%', volume_percentage)
#self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
if resp.dialog_state_out.supplemental_display_text:
text_response = resp.dialog_state_out.supplemental_display_text
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json
)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
print (device_actions_futures)
# if self.display and resp.screen_out.data:
# system_browser = browser_helpers.system_browser
# system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
#self.conversation_stream.stop_playback()
self.conversation_stream.close()
return {"continue" : continue_conversation, "response" : text_response, "input" : user_input}
def assist(self, text_query):
"""Send a request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
if self.MODE is 0 or self.MODE is 1:
def iter_log_assist_requests():
for c in self.gen_text_assist_requests(text_query):
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of text AssistRequest iteration.')
elif self.MODE is 2:
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_voice_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of audio AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
text_response = None
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
logging.info(
'Transcript of user request: "%s".',
' '.join(r.transcript for r in resp.speech_results))
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
#print(str(conversation_state))
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if resp.dialog_state_out.supplemental_display_text:
text_response = resp.dialog_state_out.supplemental_display_text
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return text_response, continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
# logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
# logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
recording = True
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
# print("message==|D" + len(resp))
# sys.stdout.flush()
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
# logging.info('End of audio request detected.')
# logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
recording = False
if resp.speech_results and recording:
print('transcript==|D',
' '.join(r.transcript
for r in resp.speech_results))
sys.stdout.flush()
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
# logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
# print('AUDIO==|D' + str(resp.audio_out.audio_data))
# sys.stdout.flush()
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
# logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
# logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
# logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.screen_out.data:
# print(resp.screen_out.data)
filePath = os.path.join(os.getcwd(),'modules/MMM-GoogleAssistant/results.html')
with open(filePath,'w') as doc:
try:
startIndex = re.search(r'<div class="popout-content" id="assistant-card-content"',resp.screen_out.data.decode('utf-8')).start()
endIndex = re.search(r'<div class="popout-overflow-shadow-down"',resp.screen_out.data.decode('utf-8')).start()
html_answer = resp.screen_out.data.decode('utf-8')[startIndex:endIndex]
except:
html_answer = resp.screen_out.data.decode('utf-8')
doc.write(html_answer)
print('HTML==|D' + filePath)
sys.stdout.flush()
# webbrowser.open_new_tab('file:///Users/pareekravi/Desktop/results.html')
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json
)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
pass
# system_browser = browser_helpers.system_browser
# system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
# logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
print('message==|DFinished playing assistant response.')
sys.stdout.flush()
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
GPIO.output(GPIO_LED, True)
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
logging.info(
'Transcript of user request: "%s".',
' '.join(r.transcript for r in resp.speech_results))
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
GPIO.output(GPIO_LED, False)
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
global dimstart
global pin_num
dimstart = 1 #Start dimming LED to signal ready status
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
t1 = threading.Thread(target=leddim,args=(5,)) #Dim second pin
t1.start()
def iter_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
self.conversation_stream.start_playback()
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected')
self.conversation_stream.stop_recording()
dimstart = 0 #Stop dimming
#speech text
if resp.speech_results:
logging.info('Transcript of user request: "%s".',
' '.join(r.transcript
for r in resp.speech_results))
logging.info('Playing assistant response.')
#Possible text from google
print(resp.dialog_state_out.supplemental_display_text)
if len(resp.audio_out.audio_data) > 0:
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
print('stop conversation')
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json
)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
print('stopped playback')
#Need to close stream in order to release audio back to speech recog program
if continue_conversation == False: self.conversation_stream.close()
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
user_query = ""
global updt_time, resp_text, mute, startMouth, mouthMove
t2status = False
def mouthMove():
#~ global startMouth
while True:
#~ print "mouth"
TezHead.SetMouth(0xFFF10E00)
time.sleep(0.1)
TezHead.SetMouth(0xFFF1F10E)
time.sleep(0.1)
TezHead.SetMouth(0xFFFF0E00)
time.sleep(0.1)
TezHead.SetMouth(0x00FF0000)
time.sleep(0.1)
t2 = Process(target=mouthMove)
continue_conversation = True
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if mute == False and user_query != "":
print user_query
socket.emit('event-user-message', user_query)
user_query = ""
if resp.speech_results:
for r in resp.speech_results:
user_query = r.transcript
#~ for r in resp.speech_results:
#~ user_query=r.transcript
if len(resp.audio_out.audio_data) > 0 and mute == False:
if not self.conversation_stream.playing:
resp.dialog_state_out.supplemental_display_text
self.conversation_stream.stop_recording()
t2.start()
t2status = True
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
updt_time = time.time()
#mouthMove(True)https://askubuntu.com/questions/895397/microphone-is-not-working-on-ubuntu-16-04
mute = False
#print text
#wf.write("/home/pi/hello.wav",44100,resp.audio_out.audio_data)
#print resp.audio_out.audio_data
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
#DialogStateOut.supplemental_display_text()
logging.debug(
'Updating convhttps://askubuntu.com/questions/895397/microphone-is-not-working-on-ubuntu-16-04ersation state.'
)
self.conversation_state = conversation_state
if resp.dialog_state_out.supplemental_display_text:
resp_text = resp.dialog_state_out.supplemental_display_text
updt_time = time.time()
#updt_time=time.time()
if mute == False:
socket.emit('event-robot-message', resp_text)
logging.info('Resp: %s', resp_text)
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = True
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
if t2status == True:
print "stopped"
time.sleep(0.5)
t2.terminate()
TezHead.SetMouth(0x110E00)
logging.info('Finished playing assistant response.')
#mute=False
self.conversation_stream.stop_playback()
time.sleep(0.5)
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
logging.info('Transcript of user request: "%s".',
' '.join(r.transcript
for r in resp.speech_results))
#해냈다...우리는 할 수 있어..
if '또또야' in ' '.join(r.transcript for r in resp.speech_results):
if '불 켜 줄래' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('led on')
GPIO.output(21,True)
GPIO.output(20,True)
return continue_conversation
if '불 꺼 줄래' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('led off')
GPIO.output(21,False)
GPIO.output(20,False)
return continue_conversation
if '창문 닫아 줄래' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('window open')
p = GPIO.PWM(servoPIN, 50)
p.start(0)
try:
p.ChangeDutyCycle(5)
time.sleep(1)
p.ChangeDutyCycle(7.5)
time.sleep(1)
p.ChangeDutyCycle(10)
time.sleep(1)
p.ChangeDutyCycle(12.5)
time.sleep(1)
p.ChangeDutyCycle(15)
except:
p.stop()
GPIO.cleanup()
return continue_conversation
if '창문 열어 줄래' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('window close')
p = GPIO.PWM(servoPIN, 50)
p.start(15)
try:
p.ChangeDutyCycle(12.5)
time.sleep(1)
p.ChangeDutyCycle(10)
time.sleep(1)
p.ChangeDutyCycle(7.5)
time.sleep(1)
p.ChangeDutyCycle(5)
time.sleep(1)
p.ChangeDutyCycle(2.5)
except:
p.stop()
return continue_conversation
if '집이야' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('home in')
GPIO.output(21,True)
GPIO.output(20,True)
p = GPIO.PWM(servoPIN, 50)
p.start(15)
try:
p.ChangeDutyCycle(12.5)
time.sleep(1)
p.ChangeDutyCycle(10)
time.sleep(1)
p.ChangeDutyCycle(7.5)
time.sleep(1)
p.ChangeDutyCycle(5)
time.sleep(1)
p.ChangeDutyCycle(2.5)
except:
p.stop()
return continue_conversation
if '다녀올게' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('home out')
GPIO.output(21,False)
GPIO.output(20,False)
p = GPIO.PWM(servoPIN, 50)
p.start(0)
try:
p.ChangeDutyCycle(5)
time.sleep(1)
p.ChangeDutyCycle(7.5)
time.sleep(1)
p.ChangeDutyCycle(10)
time.sleep(1)
p.ChangeDutyCycle(12.5)
time.sleep(1)
p.ChangeDutyCycle(15)
except:
p.stop()
GPIO.cleanup()
return continue_conversation
if '모터 돌려' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('motor on')
for pin in control_pins:
GPIO.setup(pin, GPIO.OUT)
halfstep_seq = [
[1,0,0,0],
[1,1,0,0],
[0,1,0,0],
[0,1,1,0],
[0,0,1,0],
[0,0,1,1],
[0,0,0,1],
[1,0,0,1]
]
for i in range(1024):
for halfstep in range(8):
for pin in range(4):
GPIO.output(control_pins[pin], halfstep_seq[halfstep][pin])
time.sleep(0.001)
return continue_conversation
if '鈴木さん' in ' '.join(r.transcript for r in resp.speech_results):
if '電気つけてくれ' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('led on')
GPIO.output(21,True)
GPIO.output(20,True)
return continue_conversation
if '電気けしていくれ' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('led off')
GPIO.output(21,False)
GPIO.output(20,False)
return continue_conversation
if '窓を閉めて' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('window close')
p = GPIO.PWM(servoPIN, 50)
p.start(0)
try:
p.ChangeDutyCycle(5)
time.sleep(1)
p.ChangeDutyCycle(7.5)
time.sleep(1)
p.ChangeDutyCycle(10)
time.sleep(1)
p.ChangeDutyCycle(12.5)
time.sleep(1)
p.ChangeDutyCycle(15)
except:
p.stop()
GPIO.cleanup()
return continue_conversation
if '窓を開けて' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('window open')
p = GPIO.PWM(servoPIN, 50)
p.start(15)
try:
p.ChangeDutyCycle(12.5)
time.sleep(1)
p.ChangeDutyCycle(10)
time.sleep(1)
p.ChangeDutyCycle(7.5)
time.sleep(1)
p.ChangeDutyCycle(5)
time.sleep(1)
p.ChangeDutyCycle(2.5)
except:
p.stop()
return continue_conversation
if 'ただいま' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('home in')
GPIO.output(21,True)
GPIO.output(20,True)
p = GPIO.PWM(servoPIN, 50)
p.start(15)
try:
p.ChangeDutyCycle(12.5)
time.sleep(1)
p.ChangeDutyCycle(10)
time.sleep(1)
p.ChangeDutyCycle(7.5)
time.sleep(1)
p.ChangeDutyCycle(5)
time.sleep(1)
p.ChangeDutyCycle(2.5)
except:
p.stop()
return continue_conversation
if '行ってくる' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('home out')
GPIO.output(21,False)
GPIO.output(20,False)
p = GPIO.PWM(servoPIN, 50)
p.start(0)
try:
p.ChangeDutyCycle(5)
time.sleep(1)
p.ChangeDutyCycle(7.5)
time.sleep(1)
p.ChangeDutyCycle(10)
time.sleep(1)
p.ChangeDutyCycle(12.5)
time.sleep(1)
p.ChangeDutyCycle(15)
except:
p.stop()
GPIO.cleanup()
return continue_conversation
if 'モーター回す' in ' '.join(r.transcript for r in resp.speech_results):
logging.info('motor on')
for pin in control_pins:
GPIO.setup(pin, GPIO.OUT)
halfstep_seq = [
[1,0,0,0],
[1,1,0,0],
[0,1,0,0],
[0,1,1,0],
[0,0,1,0],
[0,0,1,1],
[0,0,0,1],
[1,0,0,1]
]
for i in range(1024):
for halfstep in range(8):
for pin in range(4):
GPIO.output(control_pins[pin], halfstep_seq[halfstep][pin])
time.sleep(0.001)
return continue_conversation
#우리는...해낼꺼야..
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json
)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
global spokenAnswer
global followUp
global followUpSentence
global canPlay
global couldAnswer
couldAnswer = False
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
logging.info(
'Transcript of user request: "%s".',
' '.join(r.transcript for r in resp.speech_results))
#Functie om onze zelf gestelde vraag / mededeling op te slaan in een variabele: spokentext
self.storeAsText(resp.speech_results)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
self.conversation_state = conversation_state
if resp.dialog_state_out.supplemental_display_text:
display_text = resp.dialog_state_out.supplemental_display_text
#Manier om antwoord van google assistant op te vangen
spokenAnswer = display_text
#Enkel als hij hier geraakt is en dus een antwoord kan verzinnen wordt deze variabele op true gezet
couldAnswer = True
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
#De TTS gaat mogen spreken...
canPlay = True
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
speaking_text = ""
#Eigen gestelde vraag ophalen
self.spokenQuestion = re.findall(r'"(.+?)"',
str(self.spokenQuestion))
question = str(self.spokenQuestion[0]).lower()
#In welke taal gaat de assistant antwoorden?
language = detect_language(spokenAnswer)
#Is wat er gezegd wordt een mededeling of een vraag
type = selector.predict(vectorizer.transform([question]))
#De naam van het aanspreekpunt van de robot
matches = ['leonard', 'lennert']
#Als het geen lege audio opname is
if (question != ""):
#Laat ons SRL model onze vraag labelen
labeling = predictor.predict(sentence=question)
#Als de vraag een follow-up is van de vorige vraag
if (followUp == True):
#Als de gebruiker yes zegt
if (question == "yes"):
#Cross..
if (followUpSentence == "cross"):
self.cross_arms(robot)
#Hier zetten we canPlay op false omdat de robot anders 2 maal gaat spreken
canPlay = False
#Raise...
if (followUpSentence == "raise"):
self.raise_hand(robot)
# Hier zetten we canPlay op false omdat de robot anders 2 maal gaat spreken
canPlay = False
followUp = False
break
else:
break
followUp = False
else:
#Als ons SRL model structuren in de zin kan herkennen...
for item in labeling["verbs"]:
#If there is a verb in the question we want to know
if (item["verb"] in list_bewegen):
#Labelen via SRL
subject = self.find_between(
item["description"], "[ARG0: ", "]")
subject2 = self.find_between(
item["description"], "[ARGM-DIS: ",
"]")
verb = self.find_between(
item["description"], "[V: ", "]")
part = self.find_between(
item["description"], "[ARG1: ", "]")
#Are we talking to the robot?
if (subject in matches
or subject2 in matches):
if (verb == "cross"):
# Als het gelabelde part in onze lijst zit
if ([
ele for ele in list_parts
if (ele in part)
]):
self.cross_arms(robot)
if (verb == "raise"):
#Als het gelabelde part in onze lijst zit
if ([
ele for ele in list_parts
if (ele in part)
]):
self.raise_hand(robot)
else:
if (str(type[0]) == "ynQuestion" or
str(type[0]) == "whQuestion"):
speaking_text = "Do you want me to %s %s" % (
verb, part)
#We willen een antwoordo op deze vraag in volgende audio
followUpSentence = verb
#Daarom zetten we followup op true
followUp = True
else:
#Het is een gewone mededeling met een werkwoord die wij willen opvangen
speaking_text = "I can't understand that, i'm sorry"
else:
#Geen werkwoord dat we willen opvangen: google assistant antwoord met zijn zelf gevonden antwoord
speaking_text = str(spokenAnswer)
else:
#Mocht er een lege audio opname zijn
speaking_text = "I can't understand that, i'm sorry"
#Als de assistant mag spreken
if (canPlay == True):
self.speak_message(language, speaking_text)
logging.info('Playing assistant response.')
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
#logging.info('assist-sjxls')
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
counter = 0
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
resp_str = ' '.join(r.transcript for r in resp.speech_results)
logging.info('Transcript of user request: "%s".', resp_str)
try:
wiki_response = entitywiki.getWiki(resp_str)
except:
wiki_response = ''
if wiki_response is None:
logging.info('no wiki')
else:
if len(wiki_response) > 2:
if (counter == 0):
texttospeech.synthesize_ssml(
"<speak> {} </speak".format(wiki_response))
counter += 1
temp_data = resp.audio_out.audio_data
resp.audio_out.audio_data = open('output',
'rb').read()
#self.conversation_stream.stop_recording()
#self.conversation_stream.start_playback()
#logging.info('Playing assistant response.')
#self.conversation_stream.write(resp.audio_out.audio_data)#self.conversation_stream.stop_recording()
#self.conversation_stream.start_playback()
#logging.info('My test response')
#print(' '.join(r.transcript for r in resp.speech_results))
#print(entitywiki.getWiki(''.join(r.transcript for r in resp.speech_results)))
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
#logging.info('pointAAAAA')
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
#logging.info('pointA1A1A1')
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
#logging.info('pointB1B1B1')
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
#logging.info('pointB12B12B12')
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if (counter == 1):
break
#logging.info('pointBBBBB')
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
#logging.info('sssssss')
return continue_conversation
def assist(self, quiz, continue_quiz):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
end_of_user_request = False
#If a process mpv is in running, terminate it before run another one
if player:
player.terminate()
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
self.conversation_stream.start_playback()
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected')
self.conversation_stream.stop_recording()
end_of_user_request = True
if resp.speech_results:
# Intercepts with research on Youtube
if end_of_user_request:
request = ' '.join(r.transcript
for r in resp.speech_results)
#if we are in quiz game
if quiz:
logging.info('quiz : ' + str(quiz))
logging.info('continue_quiz : ' + str(continue_quiz))
if request.lower() == "un gland":
logging.debug('Gagné')
continue_quiz = True
else:
logging.debug('Perdu')
continue_quiz = True
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
continue_conversation = True
return continue_conversation, quiz, continue_quiz
#Search word from request
search_response = search.word_search(request)
if search_response:
#If a mpv command is called, call the mpv command
if search_response[1] == 'command':
logging.info('MPV command called : "%s"', search_response[0])
if search_response[0] == 'suivant':
player.playlist_next('weak')
#play_mpv_command(search_response[0])
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation, quiz, continue_quiz
#If a process mpv is in running, terminate it before run another one
#if player:
# player.terminate()
#If a quiz game is called
logging.info('quiz : ' + str(quiz))
logging.info('continue_quiz : ' + str(continue_quiz))
logging.info('search_response : ' + search_response[1].lower())
if search_response[1] == 'quiz':
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
quiz = True
if continue_quiz:
logging.info('Launch quiz game')
quiz_game.read_question(1)
quiz_game.read_reponses(1)
#player2 = mpv.MPV(vid=False)
#player2.play('/home/pi/GoogleEnv/lib/python3.5/site-packages/childassistant/quiz/animaux_celebres/debutant/audio/questions/question1.mp3')
#player2.wait_for_playback()
#player2.play('/home/pi/GoogleEnv/lib/python3.5/site-packages/childassistant/quiz/animaux_celebres/debutant/audio/reponses/reponses1.mp3')
#player2.wait_for_playback()
#player2.terminate()
#logging.info('Réponse : ' + response_quiz)
continue_quiz = False
else:
if search_response[1].lower() == "un gland":
logging.debug('Gagné')
playback.playback_audio_file('/home/pi/GoogleEnv/lib/python3.5/site-packages/childassistant/quiz/sons/applaude.mp3')
continue_quiz = True
else:
logging.debug('Perdu')
playback.playback_audio_file('/home/pi/GoogleEnv/lib/python3.5/site-packages/childassistant/quiz/sons/huee.mp3')
continue_quiz = True
continue_conversation = True
return continue_conversation, quiz, continue_quiz
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
play_audio(search_response[0],search_response[1])
return continue_conversation, quiz, continue_quiz
logging.info('Transcript of user request: "%s".',
' '.join(r.transcript
for r in resp.speech_results))
logging.info('Playing assistant response.')
if len(resp.audio_out.audio_data) > 0:
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json
)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
def set_everloop_color(red, green, blue, white):
color_array = bytearray()
for x in range(0, 35):
color_array += bytearray([red, green, blue, white])
with open('/dev/matrixio_everloop', 'wb') as bin_file:
bin_file.write(color_array)
bin_file.close()
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
set_everloop_color(0, 0, 10, 0) # blue
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
set_everloop_color(0, 0, 0, 0) # black
self.conversation_stream.stop_recording()
if resp.speech_results:
logging.info(
'Transcript of user request: "%s".',
' '.join(r.transcript for r in resp.speech_results))
set_everloop_color(0, 10, 0, 0) # green
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
set_everloop_color(0, 0, 0, 10) # white
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
set_everloop_color(0, 0, 0, 0) # black
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
last_request = None
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
last_request = ' '.join(r.transcript
for r in resp.speech_results)
logging.info('Transcript of user request: "%s".', last_request)
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
logging.info('Device request: %s', device_request)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
if len(device_actions_futures):
# Custom device actions triggered. Ignore the response from Assistant
continue
# Showing data from the assistant (for big screens)
system_browser = browser_helpers.system_browser
html = resp.screen_out.data.decode('utf-8')
system_browser.display(html.encode('utf-8'))
# # Showing formatted data for mobile devices
# if last_request:
# # Showing asynchronously
# threading.Thread(
# target=self.display_preview,
# args=[last_request]).start()
# # Showing synchronously (can be slow)
# # self.display_preview(last_request)
# last_request = None
if resp.dialog_state_out.supplemental_display_text:
text_response = resp.dialog_state_out.supplemental_display_text
logging.info('Supplemental text: %s', text_response)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
def assist(self):
"""Send a voice request to the Assistant and playback the response.
Returns: True if conversation should continue.
"""
continue_conversation = False
device_actions_futures = []
self.conversation_stream.start_recording()
logging.info('Recording audio request.')
def iter_log_assist_requests():
for c in self.gen_assist_requests():
assistant_helpers.log_assist_request_without_audio(c)
yield c
logging.debug('Reached end of AssistRequest iteration.')
# This generator yields AssistResponse proto messages
# received from the gRPC Google Assistant API.
for resp in self.assistant.Assist(iter_log_assist_requests(),
self.deadline):
assistant_helpers.log_assist_response_without_audio(resp)
if resp.event_type == END_OF_UTTERANCE:
logging.info('End of audio request detected.')
logging.info('Stopping recording.')
self.conversation_stream.stop_recording()
if resp.speech_results:
logging.info(
'Transcript of user request: "%s".',
' '.join(r.transcript for r in resp.speech_results))
myscript = ''.join(r.transcript for r in resp.speech_results)
print(myscript)
# 핀의 번호 할당 방법 지정
GPIO.setmode(GPIO.BCM)
# 사용할 GPIO 핀 설정
GPIO.setup(23, GPIO.OUT, initial=GPIO.LOW)
if ((myscript == '불 켜') or (myscript == '켜')
or (myscript == '불 켜 줘')):
GPIO.output(23, 1)
elif ((myscript == '불 꺼') or (myscript == '꺼')
or (myscript == '불 꺼 줘')):
GPIO.output(23, 0)
if len(resp.audio_out.audio_data) > 0:
if not self.conversation_stream.playing:
self.conversation_stream.stop_recording()
self.conversation_stream.start_playback()
logging.info('Playing assistant response.')
self.conversation_stream.write(resp.audio_out.audio_data)
if resp.dialog_state_out.conversation_state:
conversation_state = resp.dialog_state_out.conversation_state
logging.debug('Updating conversation state.')
self.conversation_state = conversation_state
if resp.dialog_state_out.volume_percentage != 0:
volume_percentage = resp.dialog_state_out.volume_percentage
logging.info('Setting volume to %s%%', volume_percentage)
self.conversation_stream.volume_percentage = volume_percentage
if resp.dialog_state_out.microphone_mode == DIALOG_FOLLOW_ON:
continue_conversation = True
logging.info('Expecting follow-on query from user.')
elif resp.dialog_state_out.microphone_mode == CLOSE_MICROPHONE:
continue_conversation = False
if resp.device_action.device_request_json:
device_request = json.loads(
resp.device_action.device_request_json)
fs = self.device_handler(device_request)
if fs:
device_actions_futures.extend(fs)
if self.display and resp.screen_out.data:
system_browser = browser_helpers.system_browser
system_browser.display(resp.screen_out.data)
if len(device_actions_futures):
logging.info('Waiting for device executions to complete.')
concurrent.futures.wait(device_actions_futures)
logging.info('Finished playing assistant response.')
self.conversation_stream.stop_playback()
return continue_conversation
