def test_makeRequest(self):
face = ['will']
object = ['snowboard']
data = OneFaceOneObjectFake()
results = [
"",
"I am 100 percent sure that I can see a snowboard behind me on my left.",
"Yes, I am 100 percent sure that I can see exactly one snowboard behind me on my left.",
"I am 84 percent sure that I can see will. Not the real one though. behind me on my left.",
"Yes, I am 84 percent sure that I can see will. Not the real one though. behind me on my left.",
"I am 100 percent confident that the snowboard belongs to Will. behind me on my left.",
"",
"I am 100 percent sure that the snowboard is here behind me on my left.",
"Yes, I am 84 percent sure that a fake will is here, behind me on my left."
]
for id in range(1, 9):
if id == 6:
continue
data.instruction_id = id
text_to_speech = TextToSpeech(data,
'en-EN',
input_faces=face,
input_objects=object)
out = text_to_speech.callback()
self.assertEqual(out, results[id])
def __init__(self, detect_model="data/andrew2.net",
lyrics_model="data/keras_model_1200.h5",
lyrics_chars="data/chars.pkl"):
# microphone
self.mic = ResumableMicrophoneStream(SAMPLE_RATE, CHUNK_SIZE)
# wake word detector
self.detector = TriggerDetector(detect_model)
# speech and language services
self.speech_client = SpeechToText()
self.luis = LangUnderstand()
self.tts = TextToSpeech()
# lyrics generator model
self.lyrics_gen = LyricsGenerator(lyrics_model, lyrics_chars)
self.pred_queue = DetectQueue(maxlen=5)
self.is_wakeup = False
# pytft display
self.tft = TFTDisplay()
self.tft_queue = queue.Queue()
self.tft_thread = threading.Thread(target=self.tft_manage, args=())
self.tft_thread.daemon = True
self.tft_thread.start()
self.notify("hi_there")
def test_makeRequest(self):
face = ['will']
object = ['snowboard']
data = ThreeFacesThreeObjectsThreeDifferent()
results = [
"", "I can see 1 sports ball, 1 snowboard, and 1 tennis racket.",
"Yes, I am 100 percent sure that I can see exactly one snowboard behind me on my left.",
"I can see will nico and eivinas.",
"Yes, I am 84 percent sure that I can see will behind me on my left.",
"I am 100 percent confident that the snowboard belongs to Will. behind me on my left.",
"",
"I am 100 percent sure that the snowboard is here behind me on my left.",
"I am 84 percent sure that will is here behind me on my left."
]
for id in range(1, 9):
if id == 6:
continue
data.instruction_id = id
text_to_speech = TextToSpeech(data,
'en-EN',
input_faces=face,
input_objects=object)
out = text_to_speech.callback()
self.assertEqual(out, results[id])
def test_makeRequest(self):
face = ['will']
object = ['snowboard']
data = ThreeFacesThreeObjectsTwoSameOneDifferent()
results = [
"", "I can see 2 snowboards, and 1 sports ball.",
"Yes, I can actually see more than one snowboard. I think there are 2. ",
"I can see will nico and eivinas.",
"Yes, I am 84 percent sure that I can see will behind me on my left.",
"I think that the following people have a snowboard: Will and Eivinas.",
"",
"I can actually see more than one snowboard. I think there are 2. One of them is here. ",
"I am 84 percent sure that will is here behind me on my left."
]
for id in range(1, 9):
if id == 6:
continue
data.instruction_id = id
text_to_speech = TextToSpeech(data,
'en-EN',
input_faces=face,
input_objects=object)
out = text_to_speech.callback()
self.assertEqual(out, results[id])
def __init__(self):
self.car = Vehicle()
# Verify that the Vehicle object was created
# print("Vehicle made!")
# self.find_own_pos("1362060585", "1362060062")
self.minute_warning = False
self.second_warning = False
self.meter_message = False
self.map = MapView()
self.text_to_speech = TextToSpeech()
def send_text(message):
if message.text.lower() == 'привет':
bot.send_message(message.chat.id, 'Привет')
elif message.text.lower() == 'пока':
bot.send_message(message.chat.id, 'Прощай, создатель')
elif message.text.lower() == 'я тебя люблю':
bot.send_sticker(message.chat.id, 'CAADAgADZgkAAnlc4gmfCor5YbYYRAI')
else:
tts = TextToSpeech(text=message.text, lang='ru')
ogg_file = tts.save()
with open(ogg_file, 'rb') as f:
bot.send_voice(message.chat.id, f)
def common_words_method(language_full, language):
welcome = (
"Welcome to the common words section. We are going to learn the %s translation of ten most common english words"
) % (language_full)
print(welcome)
f = open('data.txt', 'r')
random_indices = []
for i in range(5):
random_indices.append(random.randint(1, 3001))
#print(f[0],f[1])
line_number = 1
eng = []
esp = []
for line in f:
if (line_number in random_indices):
eng.append(line)
tran = translate_text_to_foreign(line, language)
esp.append(tran)
print(line + "->" + tran)
line_object_english = TextToSpeech(line, "en")
line_object = TextToSpeech(tran, language)
line_object_english.speech()
line_object.speech()
print(" ")
line_number += 1
print("Finally the words with their translations are- ")
for i in range(len(eng)):
print(eng[i], esp[i])
def start_rap(self):
media = vlc.MediaPlayer("assets/beat.mp3")
media.play()
media.set_time(20000)
for i in self.player_raps[self.current_player]:
TextToSpeech(i)
media.stop()
def __init__(self, config):
# X. Initialize announcer.
self.announcer = TextToSpeech()
self.announcer.add_speech_text(
"Initializing system. Please wait for a moment.")
# X. Tf
self.broadcaster = tf.TransformBroadcaster()
self.listener = tf.TransformListener()
# Prepare shared object.
self.data = SharedData()
# Publishers & Subscribers
self.pubs = Publishers()
self.subs = Subscribers(self.announcer)
# Store tf
if (not config.pose_init):
while (self.subs.odom.get_object() is None):
time.sleep(0.5)
odom = self.subs.odom.get_object()
odom_xyzrpy = create_xyzrpy_from_pose(odom.pose.pose)
pose_stamped = PoseStamped()
pose_stamped.header.stamp = rospy.Time.now()
pose_stamped.header.frame_id = 'map'
pose_stamped.pose = create_pose_from_xyzrpy(config.xyzrpy)
self.data.last_valid_ndt_pose = pose_stamped
self.data.last_valid_tf_odom_to_map = create_transform_from_pose(
config.xyzrpy, odom_xyzrpy)
# Initilize Pose
self.initialize_pose(config)
# Start checking thread
self.checking_thread = threading.Thread(target=self.run_checking)
self.checking_thread.start()
# Start control
self.run_control()
# Terminate thread.
self.announcer.terminate()
self.checking_thread.join()
def __init__(self):
self.car = Vehicle()
# Verify that the Vehicle object was created
# print("Vehicle made!")
# self.find_own_pos("1362060585", "1362060062")
self.minute_warning = False
self.second_warning = False
self.meter_message = False
self.map = MapView()
self.text_to_speech = TextToSpeech()
def execute(self, userdata):
"""
Execute function called in the state machine
Key arguments:
userdata -- state machine userdata object being passed around
"""
rospy.logdebug("Information")
rospy.loginfo(self.output_msg + "\n")
rospy.loginfo(userdata.recog_elements_in)
# Prepare sentence and get information on the camera
rospy.set_param(ACTIVE_CAMERA,
1) # Default front camera (changed by audio request)
text_to_speech = TextToSpeech(userdata.recog_elements_in,
userdata.lang_code_in,
userdata.action_in[1],
userdata.action_in[2])
self.output_msg = text_to_speech.callback()
# Indicating to the Display Controller that the display should show what the robot thinks
rospy.set_param(RECOG_UPPER_X, userdata.recog_elements_in.upper_x)
rospy.set_param(RECOG_UPPER_Y, userdata.recog_elements_in.upper_y)
rospy.set_param(RECOG_LOWER_X, userdata.recog_elements_in.lower_x)
rospy.set_param(RECOG_LOWER_Y, userdata.recog_elements_in.lower_y)
rospy.set_param(RECOG_LABEL, userdata.recog_elements_in.label)
rospy.set_param(RECOG_CAMERA, userdata.recog_elements_in.camera)
rospy.set_param(DISPLAY_INFORMATION, True)
# Maintain the thread for the duration of the speech out
audio_out_client.make_request(self.output_msg, userdata.lang_code_in)
# Resetting Parameters
rospy.set_param(RECOG_UPPER_X, [])
rospy.set_param(RECOG_UPPER_Y, [])
rospy.set_param(RECOG_LOWER_X, [])
rospy.set_param(RECOG_LOWER_Y, [])
rospy.set_param(RECOG_LABEL, [])
rospy.set_param(RECOG_CAMERA, [])
rospy.set_param(DISPLAY_INFORMATION, False)
return 'turning'
def greeting(self):
try:
chat_bot = ChatBot(None)
response = chat_bot.send_message(self.first_message)
output = response[0]
self.con_id = response[1]
self.browser.append("<p>Agent: " + output + "</p>")
self.first_message = False
TextToSpeech(output)
except Exception as e:
print(e)
def test_makeRequest(self):
face = ['will']
object = ['snowboard']
data = EmptyData()
results = [
"", "I can’t see any objects.", "No, I cannot see any snowboard.",
"I can’t see anybody.", "No, I cannot see will.",
"I don’t think anybody has a snowboard.", "",
"I cannot see any snowboard.", "I cannot see will."
]
for id in range(1, 9):
if id == 6:
continue
data.instruction_id = id
text_to_speech = TextToSpeech(data,
'en-EN',
input_faces=face,
input_objects=object)
out = text_to_speech.callback()
self.assertEqual(out, results[id])
def test_makeRequest(self):
face = ['will']
object = ['snowboard']
data = TwoFacesOneFakeOneReal()
results = [
"", "I can’t see any objects.", "No, I cannot see any snowboard.",
"I can see nico and will. But at least one of them is fake.",
"Yes, I am 84 percent sure that I can see will. Not the real one though. behind me on my left.",
"I don’t think anybody has a snowboard.", "",
"I cannot see any snowboard.",
"Yes, I am 84 percent sure that a fake will is here, behind me on my left."
]
for id in range(1, 9):
if id == 6:
continue
data.instruction_id = id
text_to_speech = TextToSpeech(data,
'en-EN',
input_faces=face,
input_objects=object)
out = text_to_speech.callback()
self.assertEqual(out, results[id])
def test_makeRequest(self):
face = ['will']
object = ['snowboard']
data = TwoFacesSameReal()
results = [
"", "I can’t see any objects.", "No, I cannot see any snowboard.",
"I can see will and will.",
"Yes, I can actually see will more than once. I knew I shouldn’t have had that last pint last night. ",
"I don’t think anybody has a snowboard.", "",
"I cannot see any snowboard.",
"Yes, I can actually see will more than once. I knew I shouldn’t have had that last pint last night. One of them is here. "
]
for id in range(1, 9):
if id == 6:
continue
data.instruction_id = id
text_to_speech = TextToSpeech(data,
'en-EN',
input_faces=face,
input_objects=object)
out = text_to_speech.callback()
self.assertEqual(out, results[id])
def test_makeRequest(self):
face = ['will']
object = ['snowboard']
data = OneFaceReal()
results = [
"", "I can’t see any objects.", "No, I cannot see any snowboard.",
"I am 100 percent sure that I can see will behind me on my left.",
"Yes, I am 100 percent sure that I can see will behind me on my left.",
"I don’t think anybody has a snowboard.", "",
"I cannot see any snowboard.",
"I am 100 percent sure that will is here behind me on my left."
]
for id in range(1, 9):
if id == 6:
continue
data.instruction_id = id
text_to_speech = TextToSpeech(data,
'en-EN',
input_faces=face,
input_objects=object)
out = text_to_speech.callback()
self.assertEqual(out, results[id])
def predict(speech_recognition=False, speech_synthesis=False):
''' Работа с обученной моделью seq2seq.
1. speech_recognition - включение распознавания речи с микрофона с помощью PocketSphinx
2. speech_synthesis - включение озвучивания ответов с помощью RHVoice '''
name_dataset = configure_file_names()
ttt = TextToText(f_name_w2v_model=f_name_w2v_model,
f_name_model=f_name_model,
f_name_model_weights=f_name_model_weights)
if speech_recognition:
print('[i] Загрузка языковой модели для распознавания речи...')
stt = SpeechToText('from_microphone', name_dataset)
if speech_synthesis:
print('[i] Загрузка синтезатора речи...')
tts = TextToSpeech('anna')
print()
question = ''
while (True):
if speech_recognition:
print('Слушаю...')
question = stt.get()
os.write(sys.stdout.fileno(), curses.tigetstr('cuu1'))
print('Вы: ' + question)
else:
question = input('Вы: ')
answer, lost_words = ttt.predict(question, True)
print('\t=> %s' % answer)
if len(lost_words) > 0:
print('[w] Потерянные слова: ' + ', '.join(lost_words) + '\n')
else:
print()
if speech_synthesis:
tts.get(answer)
def receive_message(self, message):
try:
chat_bot = ChatBot(message)
if self.con_id is None:
response = chat_bot.send_message(self.first_message)
else:
response = chat_bot.send_message(self.first_message,
self.con_id)
output = response[0]
self.con_id = response[1]
self.browser.append("<p>Agent: " + output + "</p>")
self.first_message = False
TextToSpeech(output)
except Exception as e:
print(e)
def __init__(self, config):
"""
Initialize
:param config: configuration
:type config: Config
"""
self.command_processor = CommandProcessor(
self._command_handlers(config.command_handlers))
self.robot = Robot(config.apiai.client_access_token,
config.apiai.language,
self.command_processor.commands)
self.speech_to_text = SpeechToText(
config.speechkit.key, "", config.speechkit.recognition.language)
self.text_to_speech = TextToSpeech(
config.speechkit.synthesis.cache_size, config.speechkit.key,
config.speechkit.synthesis.language,
config.speechkit.synthesis.speaker,
config.speechkit.synthesis.emotion,
config.speechkit.synthesis.speed)
self.record = SpeechCapture(config.record.silence_calculation_chunks,
config.record.speech_level_coefficient,
config.record.start_wait_chunks,
config.record.finish_wait_chunks)
from text_to_speech import TextToSpeech obj = TextToSpeech() obj.text_input() obj.create_gtts_object() obj.speech_save() obj.run()
print("language code is %s"%(language_to_be_learnt))
print("How do you want to learn today?")
print(" ")
selector=input(("Enter 1 for translation help. Enter 2 to see the most used words translated for you. Enter 3 for a fun quiz. Enter 4 to hear %s pronunciation of specific words: ")%(language_full_name))
if(selector=="1"):
fine_selector=input(("Enter 0 to translate from english to %s. Enter 1 to translate from %s to english: ")%(language_full_name,language_full_name))
if(fine_selector=="0"):
times=int(input("Number of sentences to be translated: "))
for i in range(times):
text=(input(("Enter text to be translated to %s: ")%(language_full_name)))
translation =translate_text_to_foreign(text,language_to_be_learnt)
print("The translation is %s"%(translation))
speech_object = TextToSpeech(translation,language_to_be_learnt)
speech_object.speech()
print(" ")
if(i==times-1):
print("Thanks!!")
else:
times=int(input("Number of sentences to be translated: "))
for i in range(times):
text=(input("Enter text to be translated to english: "))
#language_to_be_learnt="English"
translation =translate_text_to_english(text,language_to_be_learnt)
print("The translation is %s"%(translation))
speech_object = TextToSpeech(translation,"en")
def __init__(self):
self.queue = Queue()
self.alarm_running_queue = Queue()
self.media = MultimmediaController()
self.sched = Scheduler()
self.tts = TextToSpeech()
self.schedule = {}
#self.gcal = GoogleCalendar()
self.arduino = ArduinoController(0x08, self.queue)
self.gpio = RPiGPIO()
self.radio_db = RadioDB(
"/home/pi/Python/NetRadioAlarmClockPi/radio-settings.db")
self.webserver = webserver
self.webserver.add_station_cb = self.add_station
self.webserver.get_stations_cb = self.radio_db.get_stations
self.webserver.set_current_station_cb = self.set_current_station
self.webserver.get_current_station_cb = self.get_current_station
self.webserver.update_station_name_cb = self.radio_db.update_station_title
self.webserver.get_stream_playing_cb = self.media.get_playing
self.webserver.play_stream_cb = self.start_stream
self.webserver.stop_stream_cb = self.stop_stream
self.webserver.set_volume_cb = self.media.set_volume
self.webserver.get_schedule_cb = self.get_schedule
self.webserver.save_schedule_cb = self.save_schedule
self.webserver.alarm_running_queue = self.alarm_running_queue
self.alarm_running = False
self.update_interval = 60 * 5
self.state = "idle"
self.gpio.set_snooze_btn_callback(self.alarm_snooze_event)
self.webdev = False
self.stream_playing = False
self.alarm_running_queue.put(False)
self.last_snooze_time = None
self.button_press_count = 0
self.debouncing = False
# Set current station
url, station_title = self.get_current_station()
self.media.set_stream_url(url)
# Load radio alarm schedule and set up events if there are any
self.load_schedule()
self.create_events()
# Test event
test_time = datetime.datetime(2021, 2, 23, 20, 14, 0)
self.sched.add_event(test_time, self.alarm_event)
if len(sys.argv) > 1:
#print(sys.argv)
if sys.argv[1].replace("\r", "") == "webdev":
print("Web development mode")
self.webdev = True
if not self.webdev:
print("Resetting Arduino")
self.gpio.reset_arduino()
print("Done")
self.arduino.set_vol_change_callback(self.media.set_volume)
self.arduino.start_rot_enc_thread()
class NetRadioAlarmClock():
def __init__(self):
self.queue = Queue()
self.alarm_running_queue = Queue()
self.media = MultimmediaController()
self.sched = Scheduler()
self.tts = TextToSpeech()
self.schedule = {}
#self.gcal = GoogleCalendar()
self.arduino = ArduinoController(0x08, self.queue)
self.gpio = RPiGPIO()
self.radio_db = RadioDB(
"/home/pi/Python/NetRadioAlarmClockPi/radio-settings.db")
self.webserver = webserver
self.webserver.add_station_cb = self.add_station
self.webserver.get_stations_cb = self.radio_db.get_stations
self.webserver.set_current_station_cb = self.set_current_station
self.webserver.get_current_station_cb = self.get_current_station
self.webserver.update_station_name_cb = self.radio_db.update_station_title
self.webserver.get_stream_playing_cb = self.media.get_playing
self.webserver.play_stream_cb = self.start_stream
self.webserver.stop_stream_cb = self.stop_stream
self.webserver.set_volume_cb = self.media.set_volume
self.webserver.get_schedule_cb = self.get_schedule
self.webserver.save_schedule_cb = self.save_schedule
self.webserver.alarm_running_queue = self.alarm_running_queue
self.alarm_running = False
self.update_interval = 60 * 5
self.state = "idle"
self.gpio.set_snooze_btn_callback(self.alarm_snooze_event)
self.webdev = False
self.stream_playing = False
self.alarm_running_queue.put(False)
self.last_snooze_time = None
self.button_press_count = 0
self.debouncing = False
# Set current station
url, station_title = self.get_current_station()
self.media.set_stream_url(url)
# Load radio alarm schedule and set up events if there are any
self.load_schedule()
self.create_events()
# Test event
test_time = datetime.datetime(2021, 2, 23, 20, 14, 0)
self.sched.add_event(test_time, self.alarm_event)
if len(sys.argv) > 1:
#print(sys.argv)
if sys.argv[1].replace("\r", "") == "webdev":
print("Web development mode")
self.webdev = True
if not self.webdev:
print("Resetting Arduino")
self.gpio.reset_arduino()
print("Done")
self.arduino.set_vol_change_callback(self.media.set_volume)
self.arduino.start_rot_enc_thread()
# Set current station from web interface
def set_current_station(self, station_url):
self.radio_db.set_current_station(station_url)
self.media.set_stream_url(station_url)
def add_station(self, station_url):
title = self.media.get_title_from_any_url(station_url)
self.radio_db.add_station(station_url, title)
def get_current_station(self):
station_url = self.radio_db.get_current_station()
title = self.radio_db.get_title_for_station_url(station_url)
return station_url, title
def test_radio(self):
self.media.set_stream_url(radio_triplej_url)
self.media.play_stream()
def test_alarm(self):
pass
def start_stream(self, queue):
self.media.play_stream()
self.stream_playing = True
queue.put(True)
volume = self.media.get_volume()
url, station_name = self.get_current_station()
if not self.webdev:
self.update_lcd_playing("null")
self.webserver.emit_status(self.stream_playing, volume, station_name)
def stop_stream(self, queue):
self.media.stop_stream()
self.stream_playing = False
queue.put(False)
volume = self.media.get_volume()
url, station_name = self.get_current_station()
webserver.emit_status(self.stream_playing, volume, station_name)
def setup(self):
#print("Station: " + self.mpdc.get_station_name())
#print("Artist: " + self.mpdc.get_artist())
#print("Title: " + self.mpdc.get_title())
if not self.webdev:
#time.sleep(5)
# display idle screen
print("Updating arduino idle (with time)")
self.arduino.update_lcd_idle()
# read events from google calendar
#self.update_events("null")
#self.media.set_stream_url(radio_triplej_url)
# Testing
#self.test_radio()
#self.alarm_event(datetime.datetime.now())
pass
# update time on arduino
#self.update_lcd_idle("null")
def get_schedule(self):
events = self.radio_db.get_events()
cur_time = datetime.datetime.now()
schedule = {
"weekday_time": "",
"enabled_weekdays": [],
"weekend_time": "",
"enabled_weekend_days": [],
"next_alarm": ""
}
for event in events:
e_time = datetime.datetime.strptime(event, "%Y-%m-%d %H:%M:%S")
if (e_time - cur_time).hours < 24:
schedule["next_alarm"] = event
if e_time.weekday() >= 0 and e_time.weekday() < 5:
schedule["weekday_time"] = event
if e_time.weekday() >= 5 and e_time.weekday() < 7:
schedule["weekend_time"] = event
if e_time.weekday() == 0:
schedule["enabled_weekdays"].append("monday")
if e_time.weekday() == 1:
schedule["enabled_weekdays"].append("tuesday")
if e_time.weekday() == 2:
schedule["enabled_weekdays"].append("wednesday")
if e_time.weekday() == 3:
schedule["enabled_weekdays"].append("thursday")
if e_time.weekday() == 4:
schedule["enabled_weekdays"].append("friday")
if e_time.weekday() == 5:
schedule["enabled_weekend_days"].append("saturday")
if e_time.weekday() == 6:
schedule["enabled_weekend_days"].append("sunday")
def save_schedule(self, schedule):
print("Saving new schedule?")
self.schedule = schedule
self.radio_db.save_schedule(schedule)
self.create_events()
def load_schedule(self):
self.schedule = self.radio_db.load_schedule()
self.webserver.saved_schedule = self.schedule
print("Saved schedule: " + str(self.schedule))
def create_events(self):
days = [
"monday", "tuesday", "wednesday", "thursday", "friday", "saturday",
"sunday"
]
if self.schedule is None:
return
enabled_weekdays = self.schedule["enabled_weekdays"]
enabled_weekend_days = self.schedule["enabled_weekend_days"]
weekday_time = datetime.datetime.strptime(
self.schedule["weekday_time"], "%H:%M")
weekend_time = datetime.datetime.strptime(
self.schedule["weekend_time"], "%H:%M")
def get_day_integer(day_str):
for i in range(0, len(days)):
day_i_str = days[i]
if day_i_str == day_str:
return i
print("ERROR: No day for : '" + day_str + "'")
return -1
self.sched.remove_all_events()
for day in enabled_weekdays:
day_int = get_day_integer(day)
if day_int >= 0:
print("Adding alarm for day: " + days[day_int])
self.sched.add_weekday_event_day(day_int, weekday_time,
self.alarm_event)
else:
print("ERROR: weekday int not found for '" + day + "', " +
str(day_int))
for day in enabled_weekend_days:
day_int = get_day_integer(day)
if day_int > 0:
print("Adding alarm for weekend day: " + days[day_int])
self.sched.add_weekday_event_day(day_int, weekend_time,
self.alarm_event)
else:
print("ERROR: weekend day int not found for '" + day + "'")
def add_events_new(self, events, cb_func):
self.radio_db.clear_events()
self.sched.remove_all_events()
for event in events:
self.radio_db.add_event(event["time"])
self.sched.add_event(event["time"], cb_func)
def update_lcd_playing(self, time_str):
#station = self.mpdc.get_station_name()
#artist = self.mpdc.get_artist()
#title = self.mpdc.get_title()
station = self.media.get_station_title()
#print("station: " + station)
self.arduino.update_lcd_playing(station, "", "")
#one_second = datetime.datetime.now() + datetime.timedelta(seconds=1)
#if self.state == "playing":
# self.sched.schedule_event(one_second, self.update_lcd_playing)
def update_lcd_idle(self):
self.arduino.update_lcd_idle()
#one_second = datetime.datetime.now() + datetime.timedelta(seconds=1)
#if self.state == "idle":
# self.sched.schedule_event(one_second, self.update_lcd_idle)
def alarm_snooze_delay(self):
# If button pressed only once - snoozing
if self.button_press_count == 1:
print("SNOOZING!! ZZZZ")
now = datetime.datetime.now()
h = str(now.hour)
m = str(now.minute)
if now.minute < 10:
m = "0" + m
text = "Snoozing. The time is now {hr} {min}".format(hr=h, min=m)
self.tts.say(text)
self.state = "snooze"
self.media.stop_stream()
self.alarm_running = False
radio_rest_time = datetime.datetime.now() + datetime.timedelta(
minutes=7)
self.sched.add_event(radio_rest_time, self.alarm_event)
self.alarm_running_queue.put(False)
self.last_snooze_time = None
self.button_press_count = 0
if not self.webdev:
self.update_lcd_idle()
elif self.button_press_count > 1:
print("Alarm OFF! Time to get up!!")
self.tts.say("Alarm is now off. Time to get up!")
self.alarm_running_queue.put(False)
self.media.stop_stream()
# Clear snooze event
self.sched.remove_fixed_events()
self.button_press_count = 0
else:
print("Button pressed more than once! I see that.. Alarm stop?")
def debounce(self):
self.debouncing = False
print("Debounced?")
def alarm_snooze_event(self, channel):
if not self.debouncing:
self.button_press_count += 1
print("Btn count: " + str(self.button_press_count))
if self.button_press_count == 1:
self.last_snooze_time = datetime.datetime.now()
# Check in three seconds if the button has been pressed more than once
threading.Timer(3, self.alarm_snooze_delay).start()
self.debouncing = True
threading.Timer(0.2, self.debounce).start()
else:
print("Bounce!")
def alarm_event(self):
if self.state == "snooze":
now = datetime.datetime.now()
now_h = now.hour
now_m = now.minute
if self.last_snooze_time is not None:
last_m = self.last_snooze_time.minute
last_h = self.last_snooze_time.hour
# If we snoozed in the same minute, don't alarm again (bit of a hack)
if now_h == last_h and now_m == last_m:
return
self.state = "playing"
self.media.play_stream()
self.alarm_running = True
self.alarm_running_queue.put(True)
if not self.webdev:
self.update_lcd_playing("null")
def run_alarm_button(self, queue):
running = False
while True:
data = None
if not queue.empty():
data = queue.get()
if data is True:
running = True
elif data is False:
running = False
self.sched.process_events()
time.sleep(1)
if running:
#print("Running!")
self.update_lcd_playing("null")
self.gpio.snooze_button_led_on()
time.sleep(0.5)
self.gpio.snooze_button_lef_off()
time.sleep(0.5)
else:
#print("idle")
self.arduino.update_lcd_idle()
time.sleep(4)
def update_idle_thread(self, queue):
#self.arduino = ArduinoController(0x08)
idle = True
while True:
data = None
if not queue.empty():
data = queue.get()
if data is False:
idle = True
elif data is True:
idle = False
if idle:
#TODO: run main playing screen when playing
print("Idling")
self.arduino.update_lcd_idle()
time.sleep(5)
def run(self):
alarm_btn_thread = threading.Thread(target=self.run_alarm_button,
args=(self.alarm_running_queue, ))
alarm_btn_thread.daemon = True
alarm_btn_thread.start()
#update_idle_thread = threading.Thread(target=self.update_idle_thread, args=(self.alarm_running_queue,))
#update_idle_thread.daemon = True
#update_idle_thread.start()
#while True:
# time.sleep(1)
print("Start web server?")
self.webserver.run()
class ProcessData:
def __init__(self):
self.car = Vehicle()
# Verify that the Vehicle object was created
# print("Vehicle made!")
# self.find_own_pos("1362060585", "1362060062")
self.minute_warning = False
self.second_warning = False
self.meter_message = False
self.map = MapView()
self.text_to_speech = TextToSpeech()
def notify(self, ambulance_position_history):
'''Called by receiver to notify the car about new ambulance position'''
print('process_data notified')
car_pos = self.find_own_pos(ambulance_position_history[0]['timestamp'],
ambulance_position_history[1]['timestamp'])
if car_pos is not None:
if 2 == len(car_pos):
if car_pos[0] is not None and car_pos[1] is not None:
print(" ")
# TODO: Handle edge case when first message arrive
# Plot coordinates to map, updates everytime new data arrives. Run in Safari
self.map.plot_coordinates(car_pos[1]['longitude'],
car_pos[1]['latitude'], 'bs')
self.map.plot_coordinates(
ambulance_position_history[1]['longitude'],
ambulance_position_history[1]['latitude'], 'rs')
message = self.pick_message(car_pos[1], car_pos[0],
ambulance_position_history[1],
ambulance_position_history[0])
self.text_to_speech.play(message)
self.map.show_map()
time.sleep(0.75)
def find_own_pos(self, first_timestamp, second_timestamp):
'''Use timestamps from ambulance data to get car data'''
relevant_pos = []
old_car = None
new_car = None
for position in self.car.position_history:
#print(str(position['timestamp'])[:10])
if str(position['timestamp'])[:10] == str(first_timestamp +
300)[:10]:
relevant_pos.append(position)
if str(position['timestamp'])[:10] == str(second_timestamp +
300)[:10]:
relevant_pos.append(position)
if len(relevant_pos) > 1:
old_car = relevant_pos.pop()
new_car = relevant_pos.pop()
return new_car, old_car
# if first_car_pos is not None and second_car_pos is not None:
# self.car.position_history.clear() # Clear old data points
# self.car.position_history.append(first_car_pos)
# self.car.position_history.append(second_car_pos)
def pick_message(self, new_car, old_car, new_ambu, old_ambu):
'''Checks if the current data is relevant, and returns a integer
for each case
Keyword arguments:
new_car -- A tuple containing the cars current position
old_car -- A tuple containing the cars previous position
new_ambu -- A tuple containing the ambulance current position
old_ambu -- A tuple containing the ambulance previous position
returns
0 if the situation is not relevant
1 if the ambulance is less than 20 sec behind
2 if the ambulance is less than 2 minutes behind
3 if the ambulance just passed the car and is less than 50 meters ahead
'''
new_car_pos = (new_car['latitude'], new_car['longitude'])
new_car_time = new_car['timestamp']
old_car_pos = (old_car['latitude'], old_car['longitude'])
old_car_time = old_car['timestamp']
car_speed = Calculator.speed(new_car_pos[1], new_car_pos[0],
old_car_pos[1], old_car_pos[0],
(new_car_time - old_car_time))
print("Car time" + str(new_car_time - old_car_time))
print("Car speed: " + str(car_speed))
new_ambu_pos = (new_ambu['latitude'], new_ambu['longitude'])
new_ambu_time = new_car['timestamp']
old_ambu_pos = (old_ambu['latitude'], old_ambu['longitude'])
old_ambu_time = old_car['timestamp']
ambu_speed = Calculator.speed(new_ambu_pos[1], new_ambu_pos[0],
old_ambu_pos[1], old_ambu_pos[0],
(new_ambu_time - old_ambu_time))
print("Ambu speed: " + str(ambu_speed))
car_dir = self._find_direction(new_car_pos, old_car_pos)
ambu_dir = self._find_direction(new_ambu_pos, old_ambu_pos)
distance_km = Calculator.gps_to_kmeters(new_car_pos[1], new_car_pos[0],
new_ambu_pos[1],
new_ambu_pos[0])
print("Distance is : " + str(distance_km))
if not self._is_relevant(new_car_pos, car_speed, old_car_pos,
new_ambu_pos, ambu_speed, old_ambu_pos):
#0.05km is 50 meters.
if distance_km < 0.05\
and car_dir.name == ambu_dir.name:
if not self.meter_message:
self.meter_message = True
self.minute_warning = False
self.second_warning = False
return 3
else:
return 0
return 0
time_to_intersection = Calculator.time_to_intersection(
distance_km, ambu_speed, car_speed)
print('The vehicles are: ' + str(distance_km) +
' kms Appart. Time to intersect: ' + str(time_to_intersection))
#time to intersection is less than 20 sec, 1/3 of a minute
if time_to_intersection <= (1 / 3):
if not self.second_warning:
self.second_warning = True
return 1
return 0
if time_to_intersection <= 2:
if not self.minute_warning:
self.minute_warning = True
self.meter_message = False
return 2
return 0
def _is_relevant(self, new_car_pos, car_speed, old_car_pos, new_ambu_pos,
ambu_speed, old_ambu_pos):
'''Takes in the car and the ambulances current and previous postition.
Returns whether the car should be notified or not, as a boolean
Keyword arguments:
new_car_pos -- A tuple containing the cars current position
latitude first, longitude second
car_speed -- A float containing the cars current speed
old_car_pos -- A tuple containing the cars previous position
latitude first, longitude second
new_ambu_pos -- A tuple containing the ambulance current position
latitude first, longitude second
ambu_speed -- A float containing the ambulances current speed
old_ambu_pos -- A tuple containing the ambulance previous position
latitude first, longitude second
'''
car_dir = self._find_direction(old_car_pos, new_car_pos)
ambu_dir = self._find_direction(old_ambu_pos, new_ambu_pos)
if ambu_speed <= 0:
return False
if car_dir != ambu_dir:
print('Car not going the same direction as ambu')
return False
if not self._ambu_behind(new_car_pos, new_ambu_pos, car_dir):
print('Ambulance not behind car')
return False
distance_km = Calculator.gps_to_kmeters(float(new_car_pos[1]),
float(new_car_pos[0]),
float(new_ambu_pos[1]),
float(new_ambu_pos[0]))
time_to_intersection = Calculator.time_to_intersection(
distance_km, ambu_speed, car_speed)
print('The vehicles are: ' + str(distance_km) +
' kms apart. Time to intersect: ' + str(time_to_intersection))
if time_to_intersection == 0:
return False
if time_to_intersection > 2:
print('Ambulance is too far behind: ' + str(time_to_intersection))
return False
return True
def _find_direction(self, data1, data2):
'''Find direction for vehicle, returns Direction enum
Keyword arguments:
data1 -- tuple with latitude and longitude from newest data
data2 -- tuple with latitude and longitude from oldest data
'''
lat_change = float(data2[0]) - float(data1[0])
long_change = float(data2[1]) - float(data1[1])
if lat_change == 0 and long_change == 0:
return Direction.standing_still
if fabs(lat_change) > fabs(long_change):
if lat_change > 0:
return Direction.north
return Direction.south
if long_change > 0:
return Direction.east
return Direction.west
def _ambu_behind(self, car_pos, ambu_pos, direction):
'''Decide if the ambu is in front of, or behind the car
Keyword arguments:
car_pos -- tuple with latitude and longitude from newest data of car
ambu_pos -- tuple with latitude and longitude from newest data of ambu
direction -- Direction of the two vehicles. Must be the same after
comparing in is_relevant
'''
if direction.name == 'north':
return car_pos[0] > ambu_pos[0]
if direction.name == 'south':
return car_pos[0] < ambu_pos[0]
if direction.name == 'east':
return car_pos[1] > ambu_pos[1]
if direction.name == 'west':
return car_pos[1] < ambu_pos[1]
return True
def main():
rospy.init_node("cyborg_audio")
playback = Playback()
text_to_speech = TextToSpeech()
rospy.spin()
def run(host, port, wsgi=False, https_mode=False):
''' Автовыбор доступного порта (если указан порт 0), загрузка языковой модели и нейронной сети и запуск сервера.
1. wsgi - True: запуск WSGI сервера, False: запуск тестового Flask сервера
2. https - True: запуск в режиме https (сертификат и ключ должны быть в cert.pem и key.pem), False: запуск в режиме http
Самоподписанный сертификат можно получить, выполнив: openssl req -x509 -newkey rsa:4096 -nodes -out temp/cert.pem -keyout temp/key.pem -days 365 '''
if port == 0: # Если был введён порт 0, то автовыбор любого доступного порта
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind((host, 0))
port = sock.getsockname()[1]
log('выбран порт ' + str(port))
sock.close()
except socket.gaierror:
log('адрес ' + host + ':' + str(port) + ' некорректен',
level='error')
sock.close()
return
except OSError:
log('адрес ' + host + ':' + str(port) + ' недоступен',
level='error')
sock.close()
return
log('Flask v.' + flask_version + ', WSGIServer v.' + wsgi_version)
log('установлен максимальный размер принимаемых данных: {:.2f} Кб'.format(
max_content_length / 1024))
name_dataset = f_name_w2v_model_plays[
f_name_w2v_model_plays.rfind('w2v_model_') +
len('w2v_model_'):f_name_w2v_model_plays.rfind('.bin')]
log('загрузка обученной на наборе данных ' + name_dataset +
' модели seq2seq...')
global ttt
print()
ttt = TextToText(f_name_w2v_model=f_name_w2v_model_plays,
f_name_model=f_name_model_plays,
f_name_model_weights=f_name_model_weights_plays)
print()
log('загрузка языковой модели для распознавания речи...')
global stt
stt = SpeechToText('from_file', name_dataset)
log('загрузка синтезатора речи...')
global tts
tts = TextToSpeech('anna')
if wsgi:
global http_server
if https_mode:
log('WSGI сервер запущен на https://' + host + ':' + str(port) +
' (нажмите Ctrl+C или Ctrl+Z для выхода)')
else:
log('WSGI сервер запущен на http://' + host + ':' + str(port) +
' (нажмите Ctrl+C или Ctrl+Z для выхода)')
try:
if https_mode:
http_server = WSGIServer((host, port),
app,
log=app.logger,
error_log=app.logger,
keyfile='temp/key.pem',
certfile='temp/cert.pem')
else:
http_server = WSGIServer((host, port),
app,
log=app.logger,
error_log=app.logger)
http_server.serve_forever()
except OSError:
print()
log('адрес ' + host + ':' + str(port) + ' недоступен',
level='error')
else:
log('запуск тестового Flask сервера...')
try:
if https_mode:
app.run(host=host,
port=port,
ssl_context=('temp/cert.pem', 'temp/key.pem'),
threaded=True,
debug=False)
else:
app.run(host=host, port=port, threaded=True, debug=False)
except OSError:
print()
log('адрес ' + host + ':' + str(port) + ' недоступен',
level='error')
class ProcessData:
def __init__(self):
self.car = Vehicle()
# Verify that the Vehicle object was created
# print("Vehicle made!")
# self.find_own_pos("1362060585", "1362060062")
self.minute_warning = False
self.second_warning = False
self.meter_message = False
self.map = MapView()
self.text_to_speech = TextToSpeech()
def notify(self, ambulance_position_history):
'''Called by receiver to notify the car about new ambulance position'''
print('process_data notified')
car_pos = self.find_own_pos(ambulance_position_history[0]['timestamp'], ambulance_position_history[1]['timestamp'])
if car_pos is not None:
if 2 == len(car_pos):
if car_pos[0] is not None and car_pos[1] is not None:
print(" ")
# TODO: Handle edge case when first message arrive
# Plot coordinates to map, updates everytime new data arrives. Run in Safari
self.map.plot_coordinates(car_pos[1]['longitude'], car_pos[1]['latitude'], 'bs')
self.map.plot_coordinates(ambulance_position_history[1]['longitude'], ambulance_position_history[1]['latitude'], 'rs')
message = self.pick_message(car_pos[1], car_pos[0], ambulance_position_history[1], ambulance_position_history[0])
self.text_to_speech.play(message)
self.map.show_map()
time.sleep(0.75)
def find_own_pos(self, first_timestamp, second_timestamp):
'''Use timestamps from ambulance data to get car data'''
relevant_pos = []
old_car = None
new_car = None
for position in self.car.position_history:
#print(str(position['timestamp'])[:10])
if str(position['timestamp'])[:10] == str(first_timestamp+300)[:10]:
relevant_pos.append(position)
if str(position['timestamp'])[:10] == str(second_timestamp + 300)[:10]:
relevant_pos.append(position)
if len(relevant_pos) > 1:
old_car = relevant_pos.pop()
new_car = relevant_pos.pop()
return new_car, old_car
# if first_car_pos is not None and second_car_pos is not None:
# self.car.position_history.clear() # Clear old data points
# self.car.position_history.append(first_car_pos)
# self.car.position_history.append(second_car_pos)
def pick_message(self, new_car, old_car, new_ambu, old_ambu):
'''Checks if the current data is relevant, and returns a integer
for each case
Keyword arguments:
new_car -- A tuple containing the cars current position
old_car -- A tuple containing the cars previous position
new_ambu -- A tuple containing the ambulance current position
old_ambu -- A tuple containing the ambulance previous position
returns
0 if the situation is not relevant
1 if the ambulance is less than 20 sec behind
2 if the ambulance is less than 2 minutes behind
3 if the ambulance just passed the car and is less than 50 meters ahead
'''
new_car_pos = (new_car['latitude'], new_car['longitude'])
new_car_time = new_car['timestamp']
old_car_pos = (old_car['latitude'], old_car['longitude'])
old_car_time = old_car['timestamp']
car_speed = Calculator.speed(new_car_pos[1], new_car_pos[0], old_car_pos[1], old_car_pos[0], (new_car_time - old_car_time))
print("Car time" + str(new_car_time-old_car_time))
print("Car speed: " + str(car_speed))
new_ambu_pos = (new_ambu['latitude'], new_ambu['longitude'])
new_ambu_time = new_car['timestamp']
old_ambu_pos = (old_ambu['latitude'], old_ambu['longitude'])
old_ambu_time = old_car['timestamp']
ambu_speed = Calculator.speed(new_ambu_pos[1], new_ambu_pos[0], old_ambu_pos[1], old_ambu_pos[0], (new_ambu_time - old_ambu_time ))
print("Ambu speed: " + str(ambu_speed))
car_dir = self._find_direction(new_car_pos, old_car_pos)
ambu_dir = self._find_direction(new_ambu_pos, old_ambu_pos)
distance_km = Calculator.gps_to_kmeters(new_car_pos[1], new_car_pos[0],
new_ambu_pos[1], new_ambu_pos[0])
print("Distance is : " + str(distance_km ))
if not self._is_relevant(new_car_pos, car_speed, old_car_pos,
new_ambu_pos, ambu_speed, old_ambu_pos):
#0.05km is 50 meters.
if distance_km < 0.05\
and car_dir.name == ambu_dir.name:
if not self.meter_message:
self.meter_message = True
self.minute_warning = False
self.second_warning = False
return 3
else:
return 0
return 0
time_to_intersection = Calculator.time_to_intersection(
distance_km, ambu_speed, car_speed)
print ('The vehicles are: ' + str(distance_km) +
' kms Appart. Time to intersect: ' + str(time_to_intersection))
#time to intersection is less than 20 sec, 1/3 of a minute
if time_to_intersection <= (1/3):
if not self.second_warning:
self.second_warning = True
return 1
return 0
if time_to_intersection <= 2:
if not self.minute_warning:
self.minute_warning = True
self.meter_message = False
return 2
return 0
def _is_relevant(self, new_car_pos, car_speed, old_car_pos,
new_ambu_pos, ambu_speed, old_ambu_pos):
'''Takes in the car and the ambulances current and previous postition.
Returns whether the car should be notified or not, as a boolean
Keyword arguments:
new_car_pos -- A tuple containing the cars current position
latitude first, longitude second
car_speed -- A float containing the cars current speed
old_car_pos -- A tuple containing the cars previous position
latitude first, longitude second
new_ambu_pos -- A tuple containing the ambulance current position
latitude first, longitude second
ambu_speed -- A float containing the ambulances current speed
old_ambu_pos -- A tuple containing the ambulance previous position
latitude first, longitude second
'''
car_dir = self._find_direction(old_car_pos, new_car_pos)
ambu_dir = self._find_direction(old_ambu_pos, new_ambu_pos)
if ambu_speed <= 0:
return False
if car_dir != ambu_dir:
print('Car not going the same direction as ambu')
return False
if not self._ambu_behind(new_car_pos, new_ambu_pos, car_dir):
print ('Ambulance not behind car')
return False
distance_km = Calculator.gps_to_kmeters(float(new_car_pos[1]), float(new_car_pos[0]),
float(new_ambu_pos[1]), float(new_ambu_pos[0]))
time_to_intersection = Calculator.time_to_intersection(
distance_km, ambu_speed, car_speed)
print ('The vehicles are: ' + str(distance_km) +
' kms apart. Time to intersect: ' + str(time_to_intersection))
if time_to_intersection == 0:
return False
if time_to_intersection > 2:
print('Ambulance is too far behind: ' + str(time_to_intersection))
return False
return True
def _find_direction(self, data1, data2):
'''Find direction for vehicle, returns Direction enum
Keyword arguments:
data1 -- tuple with latitude and longitude from newest data
data2 -- tuple with latitude and longitude from oldest data
'''
lat_change = float(data2[0]) - float(data1[0])
long_change = float(data2[1]) - float(data1[1])
if lat_change == 0 and long_change == 0:
return Direction.standing_still
if fabs(lat_change) > fabs(long_change):
if lat_change > 0:
return Direction.north
return Direction.south
if long_change > 0:
return Direction.east
return Direction.west
def _ambu_behind(self, car_pos, ambu_pos, direction):
'''Decide if the ambu is in front of, or behind the car
Keyword arguments:
car_pos -- tuple with latitude and longitude from newest data of car
ambu_pos -- tuple with latitude and longitude from newest data of ambu
direction -- Direction of the two vehicles. Must be the same after
comparing in is_relevant
'''
if direction.name == 'north':
return car_pos[0] > ambu_pos[0]
if direction.name == 'south':
return car_pos[0] < ambu_pos[0]
if direction.name == 'east':
return car_pos[1] > ambu_pos[1]
if direction.name == 'west':
return car_pos[1] < ambu_pos[1]
return True
class Andrew(object):
"""the rap voice assisstant
"""
def __init__(self, detect_model="data/andrew2.net",
lyrics_model="data/keras_model_1200.h5",
lyrics_chars="data/chars.pkl"):
# microphone
self.mic = ResumableMicrophoneStream(SAMPLE_RATE, CHUNK_SIZE)
# wake word detector
self.detector = TriggerDetector(detect_model)
# speech and language services
self.speech_client = SpeechToText()
self.luis = LangUnderstand()
self.tts = TextToSpeech()
# lyrics generator model
self.lyrics_gen = LyricsGenerator(lyrics_model, lyrics_chars)
self.pred_queue = DetectQueue(maxlen=5)
self.is_wakeup = False
# pytft display
self.tft = TFTDisplay()
self.tft_queue = queue.Queue()
self.tft_thread = threading.Thread(target=self.tft_manage, args=())
self.tft_thread.daemon = True
self.tft_thread.start()
self.notify("hi_there")
def notify(self, topic="hi_there", is_async=False, audio_path="data/audio"):
# Notify with local preset audio files
from os.path import join, isfile
audio_file = join(audio_path, f"{topic}.wav")
if not isfile(audio_file):
return
self.tts.play_file(audio_file, is_async)
def generate_rap(self, topic="", beat_path="data/beat"):
"""Generate rap and play
"""
tts = self.tts
lyrics_gen = self.lyrics_gen
response = tts.generate_speech(f"hey, I can rap about {topic}")
tts.play(response, True)
# Generate based on topic
lyrics_output = lyrics_gen.generate(topic)
# Generate speech
lyrics_speech = tts.generate_speech(lyrics_output)
# Select beat
beat_index = random.randint(0, 20)
# Play beat and lyrics
tts.play_file(f'{beat_path}/beat_{beat_index}.wav', True)
tts.play(lyrics_speech)
def get_weather_message(self, city="Ithaca"):
import requests, json, os
api_key = os.getenv('WEATHER_APIKEY')
base_url = "https://api.openweathermap.org/data/2.5/weather?"
city_name = f"{city},us"
complete_url = f"{base_url}q={city_name}&units=imperial&APPID={api_key}"
try:
response = requests.get(complete_url)
res = response.json()
msg_weather = f"Today, it's {res['weather'][0]['description']} in {city}. "
msg_temp = f"The temperature is {int(res['main']['temp'])} degrees."
return msg_weather + msg_temp
except:
pass
return ""
def intent_recognize(self, text=""):
"""Recognize intent
"""
luis = self.luis
tts = self.tts
# Get result from language understanding engine
luis_result = luis.predict(text)
intent = luis_result.top_scoring_intent.intent
if intent == "Freestyle":
entities = luis_result.entities
entity_topic = "rap"
if (len(entities) > 0):
entity = entities[0]
cprint(f'The topic is {entity.entity}', 'cyan')
entity_topic = entity.entity
self.generate_rap(entity_topic)
elif intent == "Weather":
response = tts.generate_speech("I will tell you the weather in Ithaca.")
tts.play(response)
weather = self.get_weather_message()
response = tts.generate_speech(weather)
tts.play(response)
else:
self.notify("sorry")
def tft_manage(self):
"""Manage TFT display through state
"""
self.tft.display_text("Andrew is waking up")
status = {'state': 'None'}
while True:
if status['state'] is 'wait':
self.tft.display_wave()
elif status['state'] is 'listen':
self.tft.display_wave((0, 255, 0))
# Update the status
try:
update = self.tft_queue.get(block=False)
if update is not None:
status = update
except queue.Empty:
continue
def start(self):
"""Start listening and interacting
"""
tft = self.tft
tts = self.tts
# Init stream
with self.mic as stream:
self.tft_queue.put({'state': 'listen'})
while True:
if not self.is_wakeup:
stream.closed = False
while not stream.closed:
stream.audio_input = []
audio_gen = stream.generator()
for chunk in audio_gen:
if not self.is_wakeup:
prob = self.detector.get_prediction(chunk)
self.pred_queue.append(prob > 0.6)
print('!' if prob > 0.6 else '.', end='', flush=True)
if (self.pred_queue.count >= 2):
self.notify("hi")
cprint(' Trigger word detected! \n', 'magenta')
self.pred_queue.clear()
self.is_wakeup = True
stream.pause()
break
else:
cprint('Speech to text\n', 'green')
time.sleep(1)
stream.closed = False
try:
voice_command = self.speech_client.recognize(stream)
cprint(f'{voice_command}\n', 'yellow')
cprint('Recognition ended...\n', 'red')
stream.pause()
#tft.display_text(f'"{voice_command}"')
if ("goodbye" in voice_command):
self.notify("see_you")
exit()
if ("sorry" in voice_command):
self.notify("its_ok")
else:
cprint('Recognize intents...', 'cyan')
self.intent_recognize(voice_command)
except Exception as e:
cprint(f'Error: {e}', 'red')
self.is_wakeup = False
class StatusManager:
def __init__(self, config):
# X. Initialize announcer.
self.announcer = TextToSpeech()
self.announcer.add_speech_text(
"Initializing system. Please wait for a moment.")
# X. Tf
self.broadcaster = tf.TransformBroadcaster()
self.listener = tf.TransformListener()
# Prepare shared object.
self.data = SharedData()
# Publishers & Subscribers
self.pubs = Publishers()
self.subs = Subscribers(self.announcer)
# Store tf
if (not config.pose_init):
while (self.subs.odom.get_object() is None):
time.sleep(0.5)
odom = self.subs.odom.get_object()
odom_xyzrpy = create_xyzrpy_from_pose(odom.pose.pose)
pose_stamped = PoseStamped()
pose_stamped.header.stamp = rospy.Time.now()
pose_stamped.header.frame_id = 'map'
pose_stamped.pose = create_pose_from_xyzrpy(config.xyzrpy)
self.data.last_valid_ndt_pose = pose_stamped
self.data.last_valid_tf_odom_to_map = create_transform_from_pose(
config.xyzrpy, odom_xyzrpy)
# Initilize Pose
self.initialize_pose(config)
# Start checking thread
self.checking_thread = threading.Thread(target=self.run_checking)
self.checking_thread.start()
# Start control
self.run_control()
# Terminate thread.
self.announcer.terminate()
self.checking_thread.join()
def initialize_pose(self, config):
init_xyzrpy = config.xyzrpy
if (config.via_gnss):
# X. Wait update from gnss.
while (self.subs.gnss_pose.get_object() is None
and not rospy.is_shutdown()):
rospy.logwarn("Waiting for GNSS data to be received.")
time.sleep(0.5)
init_xyzrpy = create_xyzrpy_from_pose(
self.subs.gnss_pose.get_object().pose)
init_xyzrpy.roll = 0
init_xyzrpy.pitch = 0
init_xyzrpy.yaw = 0
# X. Start pose initializer if necessary.
if (config.pose_init):
self.announcer.add_speech_text(
"Start pose initializer. This will take about twenty seconds.")
# X. Switch to low resolution.
self.pubs.cfg_voxel.publish(CONFIG_VOXEL_FILT_SEARCH())
req = FULL_INIT_REQUEST(init_xyzrpy)
init_xyzrpy = call_pose_initialize_service(req)
# X. Set high resolution mode.
self.pubs.cfg_voxel.publish(CONFIG_VOXEL_FILT_RUN())
# X. Send for ndt localizer.
self.pubs.cfg_ntd.publish(DEFAULT_CONFIG_NDT(init_xyzrpy))
# X. Send for amcl localizer.
rospy.logwarn("amcl : {}".format(
create_pose_stamped_with_cov(init_xyzrpy)))
self.pubs.amcl_init.publish(create_pose_stamped_with_cov(init_xyzrpy))
# X. Notify
self.announcer.add_speech_text(
"Pose initialization is done. Start control.")
def run_pose_initialize_if_necessary(self):
if (self.data.last_pose_init_time is None \
or rospy.Time.now() - self.data.last_pose_init_time > rospy.Duration(LOCALIZATION_REINIT_MINIMUM_PERIOD)):
if (LOCALIZATION_CNT_REINIT <=
self.data.localization_not_reliable_cnt):
self.data.pose_initializing = True
INIT_POSE_STANDSTILL_COUNT = 10
wait_cnt = 0
while (self.data.standstill_count <
INIT_POSE_STANDSTILL_COUNT):
wait_cnt += 1
if (wait_cnt % 3):
self.announcer.add_speech_text("Wait robot to stop.")
time.sleep(1.0)
if (self.data.last_valid_ndt_pose is not None \
and rospy.Time.now() - self.data.last_valid_ndt_pose.header.stamp < rospy.Duration(LOCALIZATION_ODOM_TRUST_PERIOD)):
self.announcer.add_speech_text(
"Reinitialize localization based on odometry. Robot will stop."
)
config = StatusManagerConfig(
True, False,
create_xyzrpy_from_pose(
self.data.last_cur_pose.get_object().pose))
else:
self.announcer.add_speech_text(
"Reinitialize localization based on GNSS. Robot will stop."
)
config = StatusManagerConfig(
True, True, XYZRPY(0.0, 0.0, 0.0, 0.0, 0.0, 0.0))
# X. This function takes time.
self.initialize_pose(config)
self.data.last_pose_init_time = rospy.Time.now()
self.data.pose_initializing = False
self.data.localization_not_reliable_cnt \
= max(0, self.data.localization_not_reliable_cnt - LOCALIZATION_CNT_REINIT / 2)
def check_robot_stop_reason(self):
obs_idx = self.subs.obst_idx.get_object()
if (obs_idx is None or obs_idx.data == -1):
self.data.stopped_due_to_obstable_cnt = 0
return
twist_raw = self.subs.twist_raw.get_object()
if (twist_raw is None or \
not twist_is_almost_zero(twist_raw.twist, STANDSTILL_VX_THR, STANDSTILL_WX_THR)):
self.data.stopped_due_to_obstable_cnt = 0
return
fin_wps = self.subs.final_wps.get_object()
#ndt_pose = self.subs.ndt_pose.get_object()
last_cur_pose = self.data.last_cur_pose.get_object()
if (fin_wps is None or last_cur_pose is None):
self.data.stopped_due_to_obstable_cnt = 0
return
print("Distance to obstacle : {}".format(
compute_distance_to_obstacle_on_waypoint(obs_idx.data, fin_wps,
last_cur_pose.pose)))
if (compute_distance_to_obstacle_on_waypoint(
obs_idx.data, fin_wps, last_cur_pose.pose) < OBSTACLE_DIST):
if (self.data.stopped_due_to_obstable_cnt % 3 == 0):
self.announcer.add_speech_text(
"Robot stops due to obstacle in front.")
self.data.stopped_due_to_obstable_cnt += 1
if (WAIT_BEFORE_AVOID_OBST_SEC < \
self.data.stopped_due_to_obstable_cnt / STATUS_CHECK_HZ):
if (is_avoidance_ok_waypoint(obs_idx.data, fin_wps)):
self.announcer.add_speech_text(
"Rerouting for obstacle avoidance.")
header = Header()
header.stamp = rospy.Time.now()
self.pubs.astar_avoid.publish(header)
self.data.stopped_due_to_obstable_cnt = 0
else:
self.announcer.add_speech_text(
"Rerouting is not allowed here. Keep waiting.")
self.data.stopped_due_to_obstable_cnt = 0
def run_checking(self):
rate = rospy.Rate(STATUS_CHECK_HZ)
while not rospy.is_shutdown():
# X. Robot stuck reason.
self.check_robot_stop_reason()
# X. Localization status.
self.run_pose_initialize_if_necessary()
rate.sleep()
def run_control(self):
rate = rospy.Rate(CONTROL_LOOP_HZ)
while not rospy.is_shutdown():
cur_time = rospy.Time.now()
# X. Control command.
twist_st = self.subs.twist_raw.get_object()
if (twist_st is not None and not self.data.pose_initializing):
self.pubs.cmd_vel.publish(
self.subs.twist_raw.get_object().twist)
else:
self.pubs.cmd_vel.publish(
create_twist(0.0, 0.0, 0.0, 0.0, 0.0, 0.0))
# X. Velocity routing.
odom = self.subs.odom.get_object()
if (odom is not None):
twist_stamped = TwistStamped()
twist_stamped.header.stamp = cur_time
twist_stamped.twist = odom.twist.twist
self.pubs.cur_vel.publish(twist_stamped)
if (twist_is_almost_zero(odom.twist.twist, STANDSTILL_VX_THR,
STANDSTILL_WX_THR)):
self.data.standstill_count += 1
else:
self.data.standstill_count = 0
# X. Check pose validity
if (not self.data.pose_initializing):
self.check_pose_validity()
# X. Current pose
self.publish_curpose(cur_time)
rate.sleep()
def check_pose_validity(self):
# X. When map pose is unstable.
ndt_pose = self.subs.ndt_pose.get_object()
amcl_pose = self.subs.amcl_pose.get_object()
if (ndt_pose is not None and amcl_pose is not None and \
not pose_is_almost_same(ndt_pose, amcl_pose.pose, \
LOCALIZATION_TRANS_THRESH, LOCALIZATION_YAW_THRESH)):
self.data.localization_not_reliable_cnt = \
min(self.data.localization_not_reliable_cnt + 1, LOCALIZATION_CNT_REINIT)
# X. When map pose is stable.
else:
# X. Extract last valid pose.
if (ndt_pose is not None and amcl_pose is not None and \
pose_is_almost_same(ndt_pose, amcl_pose.pose, \
LOCALIZATION_TRANS_RELIABLE_THRESH, LOCALIZATION_YAW_RELIABLE_THRESH)):
try:
transform = self.listener.lookupTransform(
'map', 'odom', ndt_pose.header.stamp)
self.data.last_valid_ndt_pose_queue.append(
[ndt_pose, transform])
except (tf.LookupException, tf.ConnectivityException,
tf.ExtrapolationException):
pass
# If reaches stable count, treat the first one as valid pose.
if (len(self.data.last_valid_ndt_pose_queue) ==
LOCALIZATION_RELIABLE_CNT):
self.data.last_valid_ndt_pose = self.data.last_valid_ndt_pose_queue[
0][0]
self.data.last_valid_tf_odom_to_map = self.data.last_valid_ndt_pose_queue[
0][1]
else:
self.data.last_valid_ndt_pose_queue.clear()
self.data.localization_not_reliable_cnt \
= max(0, self.data.localization_not_reliable_cnt - 1)
def publish_curpose(self, cur_time):
odom = self.subs.odom.get_object()
if (odom is not None):
pose_stamped = PoseStamped()
pose_stamped.header.stamp = cur_time
pose_stamped.header.frame_id = "map"
pose_st = self.subs.ndt_pose.get_object()
if (pose_st is not None and \
self.data.localization_not_reliable_cnt == 0 and \
not self.data.pose_initializing):
# X. Use ndt pose as cur_pose
pose_stamped.pose = pose_st.pose
transform = create_tf_transform_from_pose(
pose_st.pose, odom.pose.pose)
# X. Send tf
self.broadcaster.sendTransform(transform[0], transform[1],
cur_time, 'odom', 'map')
self.pubs.cur_pose.publish(pose_stamped)
self.data.last_cur_pose.set_object(pose_stamped)
self.data.ndt_pose_unreliable_cnt = 0
else:
# X. Use odometry as cur_pose
if (self.data.last_valid_tf_odom_to_map is not None):
trans = self.data.last_valid_tf_odom_to_map[0]
rot = self.data.last_valid_tf_odom_to_map[1]
# X. Send tf
self.broadcaster.sendTransform(trans, rot, cur_time,
'odom', 'map')
trans_pose = transform_pose(odom.pose.pose, trans, rot)
pose_stamped.pose = trans_pose
self.pubs.cur_pose.publish(pose_stamped)
self.data.last_cur_pose.set_object(pose_stamped)
self.data.ndt_pose_unreliable_cnt += 1
if (self.data.ndt_pose_unreliable_cnt %
(5 * CONTROL_LOOP_HZ) == 0):
self.announcer.add_speech_text(
"Ndt pose is unstable. ")
def main():
CHUNK = 1024
CHANNELS = 1
RATE = 16000
FORMAT = 'S16LE'
BYTE_RATE = 32000
STATUS_KWS = 0
STATUS_ASR = 1
STATUS_TTS = 2
IP = '203.113.152.90'
IP = '10.30.154.10'
URI = 'ws://{}:8892/client/ws/speech'.format(IP)
content_type = "audio/x-raw, layout=(string)interleaved, rate=(int){}, format=(string){}, channels=(int){}".format(
RATE, FORMAT, CHANNELS)
IS_ACTIVE = False
p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paInt16,
channels=CHANNELS,
rate=RATE,
input=True,
frames_per_buffer=CHUNK)
sr = None
sentence = None
kws = KeywordSpotting(graph='graph/conv_kws.pb',
labels='graph/conv_kws_labels.txt')
tts = TextToSpeech(url='http://{}/hmm-stream/syn'.format(IP),
voices='doanngocle.htsvoice')
brain = aiml.Kernel()
brain.learn("std-startup.xml")
brain.respond("load aiml b")
status = STATUS_KWS
print("===============================")
print("[INFO] Waiting keyword [xin chào | chào bot | hi bot] ...")
while True:
block = stream.read(CHUNK)
if status == STATUS_KWS:
if kws.spotting(block):
status = STATUS_ASR
print("[INFO] Keyword detected! Start recognize ...")
sr = SpeechRecognize(
url=URI + '?%s' %
(urllib.parse.urlencode([("content-type", content_type)])),
byterate=BYTE_RATE,
one_sentence=False)
sr_response = sr.get_response_queue()
elif status == STATUS_ASR:
if sr.is_alive():
sr.push_audio(block)
while sr_response.qsize() > 0:
msg = sr_response.get()
if msg == "EOS":
print("\rHuman: {}".format(sentence))
text = brain.respond(sentence)
audio = tts.get_speech(text)
status = STATUS_TTS
tts.play_audio(audio)
status = STATUS_ASR
if "tạm biệt" in sentence:
sr.close()
else:
sentence = msg
print("\r-----: {}".format(msg), end='')
else:
status = STATUS_KWS
print("===============================")
print("========= GOOD BYE!!! =========")
print("===============================")
print("[INFO] Waiting keyword ...")
elif status == STATUS_TTS:
if sr.is_alive():
sr.push_audio(bytearray(CHUNK))
time.sleep(1. * CHUNK / BYTE_RATE)
