def test_makeRequest(self): face = ['will'] object = ['snowboard'] data = OneFaceOneObjectFake() results = [ "", "I am 100 percent sure that I can see a snowboard behind me on my left.", "Yes, I am 100 percent sure that I can see exactly one snowboard behind me on my left.", "I am 84 percent sure that I can see will. Not the real one though. behind me on my left.", "Yes, I am 84 percent sure that I can see will. Not the real one though. behind me on my left.", "I am 100 percent confident that the snowboard belongs to Will. behind me on my left.", "", "I am 100 percent sure that the snowboard is here behind me on my left.", "Yes, I am 84 percent sure that a fake will is here, behind me on my left." ] for id in range(1, 9): if id == 6: continue data.instruction_id = id text_to_speech = TextToSpeech(data, 'en-EN', input_faces=face, input_objects=object) out = text_to_speech.callback() self.assertEqual(out, results[id])
def __init__(self, detect_model="data/andrew2.net", lyrics_model="data/keras_model_1200.h5", lyrics_chars="data/chars.pkl"): # microphone self.mic = ResumableMicrophoneStream(SAMPLE_RATE, CHUNK_SIZE) # wake word detector self.detector = TriggerDetector(detect_model) # speech and language services self.speech_client = SpeechToText() self.luis = LangUnderstand() self.tts = TextToSpeech() # lyrics generator model self.lyrics_gen = LyricsGenerator(lyrics_model, lyrics_chars) self.pred_queue = DetectQueue(maxlen=5) self.is_wakeup = False # pytft display self.tft = TFTDisplay() self.tft_queue = queue.Queue() self.tft_thread = threading.Thread(target=self.tft_manage, args=()) self.tft_thread.daemon = True self.tft_thread.start() self.notify("hi_there")
def test_makeRequest(self): face = ['will'] object = ['snowboard'] data = ThreeFacesThreeObjectsThreeDifferent() results = [ "", "I can see 1 sports ball, 1 snowboard, and 1 tennis racket.", "Yes, I am 100 percent sure that I can see exactly one snowboard behind me on my left.", "I can see will nico and eivinas.", "Yes, I am 84 percent sure that I can see will behind me on my left.", "I am 100 percent confident that the snowboard belongs to Will. behind me on my left.", "", "I am 100 percent sure that the snowboard is here behind me on my left.", "I am 84 percent sure that will is here behind me on my left." ] for id in range(1, 9): if id == 6: continue data.instruction_id = id text_to_speech = TextToSpeech(data, 'en-EN', input_faces=face, input_objects=object) out = text_to_speech.callback() self.assertEqual(out, results[id])
def test_makeRequest(self): face = ['will'] object = ['snowboard'] data = ThreeFacesThreeObjectsTwoSameOneDifferent() results = [ "", "I can see 2 snowboards, and 1 sports ball.", "Yes, I can actually see more than one snowboard. I think there are 2. ", "I can see will nico and eivinas.", "Yes, I am 84 percent sure that I can see will behind me on my left.", "I think that the following people have a snowboard: Will and Eivinas.", "", "I can actually see more than one snowboard. I think there are 2. One of them is here. ", "I am 84 percent sure that will is here behind me on my left." ] for id in range(1, 9): if id == 6: continue data.instruction_id = id text_to_speech = TextToSpeech(data, 'en-EN', input_faces=face, input_objects=object) out = text_to_speech.callback() self.assertEqual(out, results[id])
def __init__(self): self.car = Vehicle() # Verify that the Vehicle object was created # print("Vehicle made!") # self.find_own_pos("1362060585", "1362060062") self.minute_warning = False self.second_warning = False self.meter_message = False self.map = MapView() self.text_to_speech = TextToSpeech()
def send_text(message): if message.text.lower() == 'привет': bot.send_message(message.chat.id, 'Привет') elif message.text.lower() == 'пока': bot.send_message(message.chat.id, 'Прощай, создатель') elif message.text.lower() == 'я тебя люблю': bot.send_sticker(message.chat.id, 'CAADAgADZgkAAnlc4gmfCor5YbYYRAI') else: tts = TextToSpeech(text=message.text, lang='ru') ogg_file = tts.save() with open(ogg_file, 'rb') as f: bot.send_voice(message.chat.id, f)
def common_words_method(language_full, language): welcome = ( "Welcome to the common words section. We are going to learn the %s translation of ten most common english words" ) % (language_full) print(welcome) f = open('data.txt', 'r') random_indices = [] for i in range(5): random_indices.append(random.randint(1, 3001)) #print(f[0],f[1]) line_number = 1 eng = [] esp = [] for line in f: if (line_number in random_indices): eng.append(line) tran = translate_text_to_foreign(line, language) esp.append(tran) print(line + "->" + tran) line_object_english = TextToSpeech(line, "en") line_object = TextToSpeech(tran, language) line_object_english.speech() line_object.speech() print(" ") line_number += 1 print("Finally the words with their translations are- ") for i in range(len(eng)): print(eng[i], esp[i])
def start_rap(self): media = vlc.MediaPlayer("assets/beat.mp3") media.play() media.set_time(20000) for i in self.player_raps[self.current_player]: TextToSpeech(i) media.stop()
def __init__(self, config): # X. Initialize announcer. self.announcer = TextToSpeech() self.announcer.add_speech_text( "Initializing system. Please wait for a moment.") # X. Tf self.broadcaster = tf.TransformBroadcaster() self.listener = tf.TransformListener() # Prepare shared object. self.data = SharedData() # Publishers & Subscribers self.pubs = Publishers() self.subs = Subscribers(self.announcer) # Store tf if (not config.pose_init): while (self.subs.odom.get_object() is None): time.sleep(0.5) odom = self.subs.odom.get_object() odom_xyzrpy = create_xyzrpy_from_pose(odom.pose.pose) pose_stamped = PoseStamped() pose_stamped.header.stamp = rospy.Time.now() pose_stamped.header.frame_id = 'map' pose_stamped.pose = create_pose_from_xyzrpy(config.xyzrpy) self.data.last_valid_ndt_pose = pose_stamped self.data.last_valid_tf_odom_to_map = create_transform_from_pose( config.xyzrpy, odom_xyzrpy) # Initilize Pose self.initialize_pose(config) # Start checking thread self.checking_thread = threading.Thread(target=self.run_checking) self.checking_thread.start() # Start control self.run_control() # Terminate thread. self.announcer.terminate() self.checking_thread.join()
def execute(self, userdata): """ Execute function called in the state machine Key arguments: userdata -- state machine userdata object being passed around """ rospy.logdebug("Information") rospy.loginfo(self.output_msg + "\n") rospy.loginfo(userdata.recog_elements_in) # Prepare sentence and get information on the camera rospy.set_param(ACTIVE_CAMERA, 1) # Default front camera (changed by audio request) text_to_speech = TextToSpeech(userdata.recog_elements_in, userdata.lang_code_in, userdata.action_in[1], userdata.action_in[2]) self.output_msg = text_to_speech.callback() # Indicating to the Display Controller that the display should show what the robot thinks rospy.set_param(RECOG_UPPER_X, userdata.recog_elements_in.upper_x) rospy.set_param(RECOG_UPPER_Y, userdata.recog_elements_in.upper_y) rospy.set_param(RECOG_LOWER_X, userdata.recog_elements_in.lower_x) rospy.set_param(RECOG_LOWER_Y, userdata.recog_elements_in.lower_y) rospy.set_param(RECOG_LABEL, userdata.recog_elements_in.label) rospy.set_param(RECOG_CAMERA, userdata.recog_elements_in.camera) rospy.set_param(DISPLAY_INFORMATION, True) # Maintain the thread for the duration of the speech out audio_out_client.make_request(self.output_msg, userdata.lang_code_in) # Resetting Parameters rospy.set_param(RECOG_UPPER_X, []) rospy.set_param(RECOG_UPPER_Y, []) rospy.set_param(RECOG_LOWER_X, []) rospy.set_param(RECOG_LOWER_Y, []) rospy.set_param(RECOG_LABEL, []) rospy.set_param(RECOG_CAMERA, []) rospy.set_param(DISPLAY_INFORMATION, False) return 'turning'
def greeting(self): try: chat_bot = ChatBot(None) response = chat_bot.send_message(self.first_message) output = response[0] self.con_id = response[1] self.browser.append("<p>Agent: " + output + "</p>") self.first_message = False TextToSpeech(output) except Exception as e: print(e)
def test_makeRequest(self): face = ['will'] object = ['snowboard'] data = EmptyData() results = [ "", "I can’t see any objects.", "No, I cannot see any snowboard.", "I can’t see anybody.", "No, I cannot see will.", "I don’t think anybody has a snowboard.", "", "I cannot see any snowboard.", "I cannot see will." ] for id in range(1, 9): if id == 6: continue data.instruction_id = id text_to_speech = TextToSpeech(data, 'en-EN', input_faces=face, input_objects=object) out = text_to_speech.callback() self.assertEqual(out, results[id])
def test_makeRequest(self): face = ['will'] object = ['snowboard'] data = TwoFacesOneFakeOneReal() results = [ "", "I can’t see any objects.", "No, I cannot see any snowboard.", "I can see nico and will. But at least one of them is fake.", "Yes, I am 84 percent sure that I can see will. Not the real one though. behind me on my left.", "I don’t think anybody has a snowboard.", "", "I cannot see any snowboard.", "Yes, I am 84 percent sure that a fake will is here, behind me on my left." ] for id in range(1, 9): if id == 6: continue data.instruction_id = id text_to_speech = TextToSpeech(data, 'en-EN', input_faces=face, input_objects=object) out = text_to_speech.callback() self.assertEqual(out, results[id])
def test_makeRequest(self): face = ['will'] object = ['snowboard'] data = TwoFacesSameReal() results = [ "", "I can’t see any objects.", "No, I cannot see any snowboard.", "I can see will and will.", "Yes, I can actually see will more than once. I knew I shouldn’t have had that last pint last night. ", "I don’t think anybody has a snowboard.", "", "I cannot see any snowboard.", "Yes, I can actually see will more than once. I knew I shouldn’t have had that last pint last night. One of them is here. " ] for id in range(1, 9): if id == 6: continue data.instruction_id = id text_to_speech = TextToSpeech(data, 'en-EN', input_faces=face, input_objects=object) out = text_to_speech.callback() self.assertEqual(out, results[id])
def test_makeRequest(self): face = ['will'] object = ['snowboard'] data = OneFaceReal() results = [ "", "I can’t see any objects.", "No, I cannot see any snowboard.", "I am 100 percent sure that I can see will behind me on my left.", "Yes, I am 100 percent sure that I can see will behind me on my left.", "I don’t think anybody has a snowboard.", "", "I cannot see any snowboard.", "I am 100 percent sure that will is here behind me on my left." ] for id in range(1, 9): if id == 6: continue data.instruction_id = id text_to_speech = TextToSpeech(data, 'en-EN', input_faces=face, input_objects=object) out = text_to_speech.callback() self.assertEqual(out, results[id])
def predict(speech_recognition=False, speech_synthesis=False): ''' Работа с обученной моделью seq2seq. 1. speech_recognition - включение распознавания речи с микрофона с помощью PocketSphinx 2. speech_synthesis - включение озвучивания ответов с помощью RHVoice ''' name_dataset = configure_file_names() ttt = TextToText(f_name_w2v_model=f_name_w2v_model, f_name_model=f_name_model, f_name_model_weights=f_name_model_weights) if speech_recognition: print('[i] Загрузка языковой модели для распознавания речи...') stt = SpeechToText('from_microphone', name_dataset) if speech_synthesis: print('[i] Загрузка синтезатора речи...') tts = TextToSpeech('anna') print() question = '' while (True): if speech_recognition: print('Слушаю...') question = stt.get() os.write(sys.stdout.fileno(), curses.tigetstr('cuu1')) print('Вы: ' + question) else: question = input('Вы: ') answer, lost_words = ttt.predict(question, True) print('\t=> %s' % answer) if len(lost_words) > 0: print('[w] Потерянные слова: ' + ', '.join(lost_words) + '\n') else: print() if speech_synthesis: tts.get(answer)
def receive_message(self, message): try: chat_bot = ChatBot(message) if self.con_id is None: response = chat_bot.send_message(self.first_message) else: response = chat_bot.send_message(self.first_message, self.con_id) output = response[0] self.con_id = response[1] self.browser.append("<p>Agent: " + output + "</p>") self.first_message = False TextToSpeech(output) except Exception as e: print(e)
def __init__(self, config): """ Initialize :param config: configuration :type config: Config """ self.command_processor = CommandProcessor( self._command_handlers(config.command_handlers)) self.robot = Robot(config.apiai.client_access_token, config.apiai.language, self.command_processor.commands) self.speech_to_text = SpeechToText( config.speechkit.key, "", config.speechkit.recognition.language) self.text_to_speech = TextToSpeech( config.speechkit.synthesis.cache_size, config.speechkit.key, config.speechkit.synthesis.language, config.speechkit.synthesis.speaker, config.speechkit.synthesis.emotion, config.speechkit.synthesis.speed) self.record = SpeechCapture(config.record.silence_calculation_chunks, config.record.speech_level_coefficient, config.record.start_wait_chunks, config.record.finish_wait_chunks)
from text_to_speech import TextToSpeech obj = TextToSpeech() obj.text_input() obj.create_gtts_object() obj.speech_save() obj.run()
print("language code is %s"%(language_to_be_learnt)) print("How do you want to learn today?") print(" ") selector=input(("Enter 1 for translation help. Enter 2 to see the most used words translated for you. Enter 3 for a fun quiz. Enter 4 to hear %s pronunciation of specific words: ")%(language_full_name)) if(selector=="1"): fine_selector=input(("Enter 0 to translate from english to %s. Enter 1 to translate from %s to english: ")%(language_full_name,language_full_name)) if(fine_selector=="0"): times=int(input("Number of sentences to be translated: ")) for i in range(times): text=(input(("Enter text to be translated to %s: ")%(language_full_name))) translation =translate_text_to_foreign(text,language_to_be_learnt) print("The translation is %s"%(translation)) speech_object = TextToSpeech(translation,language_to_be_learnt) speech_object.speech() print(" ") if(i==times-1): print("Thanks!!") else: times=int(input("Number of sentences to be translated: ")) for i in range(times): text=(input("Enter text to be translated to english: ")) #language_to_be_learnt="English" translation =translate_text_to_english(text,language_to_be_learnt) print("The translation is %s"%(translation)) speech_object = TextToSpeech(translation,"en")
def __init__(self): self.queue = Queue() self.alarm_running_queue = Queue() self.media = MultimmediaController() self.sched = Scheduler() self.tts = TextToSpeech() self.schedule = {} #self.gcal = GoogleCalendar() self.arduino = ArduinoController(0x08, self.queue) self.gpio = RPiGPIO() self.radio_db = RadioDB( "/home/pi/Python/NetRadioAlarmClockPi/radio-settings.db") self.webserver = webserver self.webserver.add_station_cb = self.add_station self.webserver.get_stations_cb = self.radio_db.get_stations self.webserver.set_current_station_cb = self.set_current_station self.webserver.get_current_station_cb = self.get_current_station self.webserver.update_station_name_cb = self.radio_db.update_station_title self.webserver.get_stream_playing_cb = self.media.get_playing self.webserver.play_stream_cb = self.start_stream self.webserver.stop_stream_cb = self.stop_stream self.webserver.set_volume_cb = self.media.set_volume self.webserver.get_schedule_cb = self.get_schedule self.webserver.save_schedule_cb = self.save_schedule self.webserver.alarm_running_queue = self.alarm_running_queue self.alarm_running = False self.update_interval = 60 * 5 self.state = "idle" self.gpio.set_snooze_btn_callback(self.alarm_snooze_event) self.webdev = False self.stream_playing = False self.alarm_running_queue.put(False) self.last_snooze_time = None self.button_press_count = 0 self.debouncing = False # Set current station url, station_title = self.get_current_station() self.media.set_stream_url(url) # Load radio alarm schedule and set up events if there are any self.load_schedule() self.create_events() # Test event test_time = datetime.datetime(2021, 2, 23, 20, 14, 0) self.sched.add_event(test_time, self.alarm_event) if len(sys.argv) > 1: #print(sys.argv) if sys.argv[1].replace("\r", "") == "webdev": print("Web development mode") self.webdev = True if not self.webdev: print("Resetting Arduino") self.gpio.reset_arduino() print("Done") self.arduino.set_vol_change_callback(self.media.set_volume) self.arduino.start_rot_enc_thread()
class NetRadioAlarmClock(): def __init__(self): self.queue = Queue() self.alarm_running_queue = Queue() self.media = MultimmediaController() self.sched = Scheduler() self.tts = TextToSpeech() self.schedule = {} #self.gcal = GoogleCalendar() self.arduino = ArduinoController(0x08, self.queue) self.gpio = RPiGPIO() self.radio_db = RadioDB( "/home/pi/Python/NetRadioAlarmClockPi/radio-settings.db") self.webserver = webserver self.webserver.add_station_cb = self.add_station self.webserver.get_stations_cb = self.radio_db.get_stations self.webserver.set_current_station_cb = self.set_current_station self.webserver.get_current_station_cb = self.get_current_station self.webserver.update_station_name_cb = self.radio_db.update_station_title self.webserver.get_stream_playing_cb = self.media.get_playing self.webserver.play_stream_cb = self.start_stream self.webserver.stop_stream_cb = self.stop_stream self.webserver.set_volume_cb = self.media.set_volume self.webserver.get_schedule_cb = self.get_schedule self.webserver.save_schedule_cb = self.save_schedule self.webserver.alarm_running_queue = self.alarm_running_queue self.alarm_running = False self.update_interval = 60 * 5 self.state = "idle" self.gpio.set_snooze_btn_callback(self.alarm_snooze_event) self.webdev = False self.stream_playing = False self.alarm_running_queue.put(False) self.last_snooze_time = None self.button_press_count = 0 self.debouncing = False # Set current station url, station_title = self.get_current_station() self.media.set_stream_url(url) # Load radio alarm schedule and set up events if there are any self.load_schedule() self.create_events() # Test event test_time = datetime.datetime(2021, 2, 23, 20, 14, 0) self.sched.add_event(test_time, self.alarm_event) if len(sys.argv) > 1: #print(sys.argv) if sys.argv[1].replace("\r", "") == "webdev": print("Web development mode") self.webdev = True if not self.webdev: print("Resetting Arduino") self.gpio.reset_arduino() print("Done") self.arduino.set_vol_change_callback(self.media.set_volume) self.arduino.start_rot_enc_thread() # Set current station from web interface def set_current_station(self, station_url): self.radio_db.set_current_station(station_url) self.media.set_stream_url(station_url) def add_station(self, station_url): title = self.media.get_title_from_any_url(station_url) self.radio_db.add_station(station_url, title) def get_current_station(self): station_url = self.radio_db.get_current_station() title = self.radio_db.get_title_for_station_url(station_url) return station_url, title def test_radio(self): self.media.set_stream_url(radio_triplej_url) self.media.play_stream() def test_alarm(self): pass def start_stream(self, queue): self.media.play_stream() self.stream_playing = True queue.put(True) volume = self.media.get_volume() url, station_name = self.get_current_station() if not self.webdev: self.update_lcd_playing("null") self.webserver.emit_status(self.stream_playing, volume, station_name) def stop_stream(self, queue): self.media.stop_stream() self.stream_playing = False queue.put(False) volume = self.media.get_volume() url, station_name = self.get_current_station() webserver.emit_status(self.stream_playing, volume, station_name) def setup(self): #print("Station: " + self.mpdc.get_station_name()) #print("Artist: " + self.mpdc.get_artist()) #print("Title: " + self.mpdc.get_title()) if not self.webdev: #time.sleep(5) # display idle screen print("Updating arduino idle (with time)") self.arduino.update_lcd_idle() # read events from google calendar #self.update_events("null") #self.media.set_stream_url(radio_triplej_url) # Testing #self.test_radio() #self.alarm_event(datetime.datetime.now()) pass # update time on arduino #self.update_lcd_idle("null") def get_schedule(self): events = self.radio_db.get_events() cur_time = datetime.datetime.now() schedule = { "weekday_time": "", "enabled_weekdays": [], "weekend_time": "", "enabled_weekend_days": [], "next_alarm": "" } for event in events: e_time = datetime.datetime.strptime(event, "%Y-%m-%d %H:%M:%S") if (e_time - cur_time).hours < 24: schedule["next_alarm"] = event if e_time.weekday() >= 0 and e_time.weekday() < 5: schedule["weekday_time"] = event if e_time.weekday() >= 5 and e_time.weekday() < 7: schedule["weekend_time"] = event if e_time.weekday() == 0: schedule["enabled_weekdays"].append("monday") if e_time.weekday() == 1: schedule["enabled_weekdays"].append("tuesday") if e_time.weekday() == 2: schedule["enabled_weekdays"].append("wednesday") if e_time.weekday() == 3: schedule["enabled_weekdays"].append("thursday") if e_time.weekday() == 4: schedule["enabled_weekdays"].append("friday") if e_time.weekday() == 5: schedule["enabled_weekend_days"].append("saturday") if e_time.weekday() == 6: schedule["enabled_weekend_days"].append("sunday") def save_schedule(self, schedule): print("Saving new schedule?") self.schedule = schedule self.radio_db.save_schedule(schedule) self.create_events() def load_schedule(self): self.schedule = self.radio_db.load_schedule() self.webserver.saved_schedule = self.schedule print("Saved schedule: " + str(self.schedule)) def create_events(self): days = [ "monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday" ] if self.schedule is None: return enabled_weekdays = self.schedule["enabled_weekdays"] enabled_weekend_days = self.schedule["enabled_weekend_days"] weekday_time = datetime.datetime.strptime( self.schedule["weekday_time"], "%H:%M") weekend_time = datetime.datetime.strptime( self.schedule["weekend_time"], "%H:%M") def get_day_integer(day_str): for i in range(0, len(days)): day_i_str = days[i] if day_i_str == day_str: return i print("ERROR: No day for : '" + day_str + "'") return -1 self.sched.remove_all_events() for day in enabled_weekdays: day_int = get_day_integer(day) if day_int >= 0: print("Adding alarm for day: " + days[day_int]) self.sched.add_weekday_event_day(day_int, weekday_time, self.alarm_event) else: print("ERROR: weekday int not found for '" + day + "', " + str(day_int)) for day in enabled_weekend_days: day_int = get_day_integer(day) if day_int > 0: print("Adding alarm for weekend day: " + days[day_int]) self.sched.add_weekday_event_day(day_int, weekend_time, self.alarm_event) else: print("ERROR: weekend day int not found for '" + day + "'") def add_events_new(self, events, cb_func): self.radio_db.clear_events() self.sched.remove_all_events() for event in events: self.radio_db.add_event(event["time"]) self.sched.add_event(event["time"], cb_func) def update_lcd_playing(self, time_str): #station = self.mpdc.get_station_name() #artist = self.mpdc.get_artist() #title = self.mpdc.get_title() station = self.media.get_station_title() #print("station: " + station) self.arduino.update_lcd_playing(station, "", "") #one_second = datetime.datetime.now() + datetime.timedelta(seconds=1) #if self.state == "playing": # self.sched.schedule_event(one_second, self.update_lcd_playing) def update_lcd_idle(self): self.arduino.update_lcd_idle() #one_second = datetime.datetime.now() + datetime.timedelta(seconds=1) #if self.state == "idle": # self.sched.schedule_event(one_second, self.update_lcd_idle) def alarm_snooze_delay(self): # If button pressed only once - snoozing if self.button_press_count == 1: print("SNOOZING!! ZZZZ") now = datetime.datetime.now() h = str(now.hour) m = str(now.minute) if now.minute < 10: m = "0" + m text = "Snoozing. The time is now {hr} {min}".format(hr=h, min=m) self.tts.say(text) self.state = "snooze" self.media.stop_stream() self.alarm_running = False radio_rest_time = datetime.datetime.now() + datetime.timedelta( minutes=7) self.sched.add_event(radio_rest_time, self.alarm_event) self.alarm_running_queue.put(False) self.last_snooze_time = None self.button_press_count = 0 if not self.webdev: self.update_lcd_idle() elif self.button_press_count > 1: print("Alarm OFF! Time to get up!!") self.tts.say("Alarm is now off. Time to get up!") self.alarm_running_queue.put(False) self.media.stop_stream() # Clear snooze event self.sched.remove_fixed_events() self.button_press_count = 0 else: print("Button pressed more than once! I see that.. Alarm stop?") def debounce(self): self.debouncing = False print("Debounced?") def alarm_snooze_event(self, channel): if not self.debouncing: self.button_press_count += 1 print("Btn count: " + str(self.button_press_count)) if self.button_press_count == 1: self.last_snooze_time = datetime.datetime.now() # Check in three seconds if the button has been pressed more than once threading.Timer(3, self.alarm_snooze_delay).start() self.debouncing = True threading.Timer(0.2, self.debounce).start() else: print("Bounce!") def alarm_event(self): if self.state == "snooze": now = datetime.datetime.now() now_h = now.hour now_m = now.minute if self.last_snooze_time is not None: last_m = self.last_snooze_time.minute last_h = self.last_snooze_time.hour # If we snoozed in the same minute, don't alarm again (bit of a hack) if now_h == last_h and now_m == last_m: return self.state = "playing" self.media.play_stream() self.alarm_running = True self.alarm_running_queue.put(True) if not self.webdev: self.update_lcd_playing("null") def run_alarm_button(self, queue): running = False while True: data = None if not queue.empty(): data = queue.get() if data is True: running = True elif data is False: running = False self.sched.process_events() time.sleep(1) if running: #print("Running!") self.update_lcd_playing("null") self.gpio.snooze_button_led_on() time.sleep(0.5) self.gpio.snooze_button_lef_off() time.sleep(0.5) else: #print("idle") self.arduino.update_lcd_idle() time.sleep(4) def update_idle_thread(self, queue): #self.arduino = ArduinoController(0x08) idle = True while True: data = None if not queue.empty(): data = queue.get() if data is False: idle = True elif data is True: idle = False if idle: #TODO: run main playing screen when playing print("Idling") self.arduino.update_lcd_idle() time.sleep(5) def run(self): alarm_btn_thread = threading.Thread(target=self.run_alarm_button, args=(self.alarm_running_queue, )) alarm_btn_thread.daemon = True alarm_btn_thread.start() #update_idle_thread = threading.Thread(target=self.update_idle_thread, args=(self.alarm_running_queue,)) #update_idle_thread.daemon = True #update_idle_thread.start() #while True: # time.sleep(1) print("Start web server?") self.webserver.run()
class ProcessData: def __init__(self): self.car = Vehicle() # Verify that the Vehicle object was created # print("Vehicle made!") # self.find_own_pos("1362060585", "1362060062") self.minute_warning = False self.second_warning = False self.meter_message = False self.map = MapView() self.text_to_speech = TextToSpeech() def notify(self, ambulance_position_history): '''Called by receiver to notify the car about new ambulance position''' print('process_data notified') car_pos = self.find_own_pos(ambulance_position_history[0]['timestamp'], ambulance_position_history[1]['timestamp']) if car_pos is not None: if 2 == len(car_pos): if car_pos[0] is not None and car_pos[1] is not None: print(" ") # TODO: Handle edge case when first message arrive # Plot coordinates to map, updates everytime new data arrives. Run in Safari self.map.plot_coordinates(car_pos[1]['longitude'], car_pos[1]['latitude'], 'bs') self.map.plot_coordinates( ambulance_position_history[1]['longitude'], ambulance_position_history[1]['latitude'], 'rs') message = self.pick_message(car_pos[1], car_pos[0], ambulance_position_history[1], ambulance_position_history[0]) self.text_to_speech.play(message) self.map.show_map() time.sleep(0.75) def find_own_pos(self, first_timestamp, second_timestamp): '''Use timestamps from ambulance data to get car data''' relevant_pos = [] old_car = None new_car = None for position in self.car.position_history: #print(str(position['timestamp'])[:10]) if str(position['timestamp'])[:10] == str(first_timestamp + 300)[:10]: relevant_pos.append(position) if str(position['timestamp'])[:10] == str(second_timestamp + 300)[:10]: relevant_pos.append(position) if len(relevant_pos) > 1: old_car = relevant_pos.pop() new_car = relevant_pos.pop() return new_car, old_car # if first_car_pos is not None and second_car_pos is not None: # self.car.position_history.clear() # Clear old data points # self.car.position_history.append(first_car_pos) # self.car.position_history.append(second_car_pos) def pick_message(self, new_car, old_car, new_ambu, old_ambu): '''Checks if the current data is relevant, and returns a integer for each case Keyword arguments: new_car -- A tuple containing the cars current position old_car -- A tuple containing the cars previous position new_ambu -- A tuple containing the ambulance current position old_ambu -- A tuple containing the ambulance previous position returns 0 if the situation is not relevant 1 if the ambulance is less than 20 sec behind 2 if the ambulance is less than 2 minutes behind 3 if the ambulance just passed the car and is less than 50 meters ahead ''' new_car_pos = (new_car['latitude'], new_car['longitude']) new_car_time = new_car['timestamp'] old_car_pos = (old_car['latitude'], old_car['longitude']) old_car_time = old_car['timestamp'] car_speed = Calculator.speed(new_car_pos[1], new_car_pos[0], old_car_pos[1], old_car_pos[0], (new_car_time - old_car_time)) print("Car time" + str(new_car_time - old_car_time)) print("Car speed: " + str(car_speed)) new_ambu_pos = (new_ambu['latitude'], new_ambu['longitude']) new_ambu_time = new_car['timestamp'] old_ambu_pos = (old_ambu['latitude'], old_ambu['longitude']) old_ambu_time = old_car['timestamp'] ambu_speed = Calculator.speed(new_ambu_pos[1], new_ambu_pos[0], old_ambu_pos[1], old_ambu_pos[0], (new_ambu_time - old_ambu_time)) print("Ambu speed: " + str(ambu_speed)) car_dir = self._find_direction(new_car_pos, old_car_pos) ambu_dir = self._find_direction(new_ambu_pos, old_ambu_pos) distance_km = Calculator.gps_to_kmeters(new_car_pos[1], new_car_pos[0], new_ambu_pos[1], new_ambu_pos[0]) print("Distance is : " + str(distance_km)) if not self._is_relevant(new_car_pos, car_speed, old_car_pos, new_ambu_pos, ambu_speed, old_ambu_pos): #0.05km is 50 meters. if distance_km < 0.05\ and car_dir.name == ambu_dir.name: if not self.meter_message: self.meter_message = True self.minute_warning = False self.second_warning = False return 3 else: return 0 return 0 time_to_intersection = Calculator.time_to_intersection( distance_km, ambu_speed, car_speed) print('The vehicles are: ' + str(distance_km) + ' kms Appart. Time to intersect: ' + str(time_to_intersection)) #time to intersection is less than 20 sec, 1/3 of a minute if time_to_intersection <= (1 / 3): if not self.second_warning: self.second_warning = True return 1 return 0 if time_to_intersection <= 2: if not self.minute_warning: self.minute_warning = True self.meter_message = False return 2 return 0 def _is_relevant(self, new_car_pos, car_speed, old_car_pos, new_ambu_pos, ambu_speed, old_ambu_pos): '''Takes in the car and the ambulances current and previous postition. Returns whether the car should be notified or not, as a boolean Keyword arguments: new_car_pos -- A tuple containing the cars current position latitude first, longitude second car_speed -- A float containing the cars current speed old_car_pos -- A tuple containing the cars previous position latitude first, longitude second new_ambu_pos -- A tuple containing the ambulance current position latitude first, longitude second ambu_speed -- A float containing the ambulances current speed old_ambu_pos -- A tuple containing the ambulance previous position latitude first, longitude second ''' car_dir = self._find_direction(old_car_pos, new_car_pos) ambu_dir = self._find_direction(old_ambu_pos, new_ambu_pos) if ambu_speed <= 0: return False if car_dir != ambu_dir: print('Car not going the same direction as ambu') return False if not self._ambu_behind(new_car_pos, new_ambu_pos, car_dir): print('Ambulance not behind car') return False distance_km = Calculator.gps_to_kmeters(float(new_car_pos[1]), float(new_car_pos[0]), float(new_ambu_pos[1]), float(new_ambu_pos[0])) time_to_intersection = Calculator.time_to_intersection( distance_km, ambu_speed, car_speed) print('The vehicles are: ' + str(distance_km) + ' kms apart. Time to intersect: ' + str(time_to_intersection)) if time_to_intersection == 0: return False if time_to_intersection > 2: print('Ambulance is too far behind: ' + str(time_to_intersection)) return False return True def _find_direction(self, data1, data2): '''Find direction for vehicle, returns Direction enum Keyword arguments: data1 -- tuple with latitude and longitude from newest data data2 -- tuple with latitude and longitude from oldest data ''' lat_change = float(data2[0]) - float(data1[0]) long_change = float(data2[1]) - float(data1[1]) if lat_change == 0 and long_change == 0: return Direction.standing_still if fabs(lat_change) > fabs(long_change): if lat_change > 0: return Direction.north return Direction.south if long_change > 0: return Direction.east return Direction.west def _ambu_behind(self, car_pos, ambu_pos, direction): '''Decide if the ambu is in front of, or behind the car Keyword arguments: car_pos -- tuple with latitude and longitude from newest data of car ambu_pos -- tuple with latitude and longitude from newest data of ambu direction -- Direction of the two vehicles. Must be the same after comparing in is_relevant ''' if direction.name == 'north': return car_pos[0] > ambu_pos[0] if direction.name == 'south': return car_pos[0] < ambu_pos[0] if direction.name == 'east': return car_pos[1] > ambu_pos[1] if direction.name == 'west': return car_pos[1] < ambu_pos[1] return True
def main(): rospy.init_node("cyborg_audio") playback = Playback() text_to_speech = TextToSpeech() rospy.spin()
def run(host, port, wsgi=False, https_mode=False): ''' Автовыбор доступного порта (если указан порт 0), загрузка языковой модели и нейронной сети и запуск сервера. 1. wsgi - True: запуск WSGI сервера, False: запуск тестового Flask сервера 2. https - True: запуск в режиме https (сертификат и ключ должны быть в cert.pem и key.pem), False: запуск в режиме http Самоподписанный сертификат можно получить, выполнив: openssl req -x509 -newkey rsa:4096 -nodes -out temp/cert.pem -keyout temp/key.pem -days 365 ''' if port == 0: # Если был введён порт 0, то автовыбор любого доступного порта try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.bind((host, 0)) port = sock.getsockname()[1] log('выбран порт ' + str(port)) sock.close() except socket.gaierror: log('адрес ' + host + ':' + str(port) + ' некорректен', level='error') sock.close() return except OSError: log('адрес ' + host + ':' + str(port) + ' недоступен', level='error') sock.close() return log('Flask v.' + flask_version + ', WSGIServer v.' + wsgi_version) log('установлен максимальный размер принимаемых данных: {:.2f} Кб'.format( max_content_length / 1024)) name_dataset = f_name_w2v_model_plays[ f_name_w2v_model_plays.rfind('w2v_model_') + len('w2v_model_'):f_name_w2v_model_plays.rfind('.bin')] log('загрузка обученной на наборе данных ' + name_dataset + ' модели seq2seq...') global ttt print() ttt = TextToText(f_name_w2v_model=f_name_w2v_model_plays, f_name_model=f_name_model_plays, f_name_model_weights=f_name_model_weights_plays) print() log('загрузка языковой модели для распознавания речи...') global stt stt = SpeechToText('from_file', name_dataset) log('загрузка синтезатора речи...') global tts tts = TextToSpeech('anna') if wsgi: global http_server if https_mode: log('WSGI сервер запущен на https://' + host + ':' + str(port) + ' (нажмите Ctrl+C или Ctrl+Z для выхода)') else: log('WSGI сервер запущен на http://' + host + ':' + str(port) + ' (нажмите Ctrl+C или Ctrl+Z для выхода)') try: if https_mode: http_server = WSGIServer((host, port), app, log=app.logger, error_log=app.logger, keyfile='temp/key.pem', certfile='temp/cert.pem') else: http_server = WSGIServer((host, port), app, log=app.logger, error_log=app.logger) http_server.serve_forever() except OSError: print() log('адрес ' + host + ':' + str(port) + ' недоступен', level='error') else: log('запуск тестового Flask сервера...') try: if https_mode: app.run(host=host, port=port, ssl_context=('temp/cert.pem', 'temp/key.pem'), threaded=True, debug=False) else: app.run(host=host, port=port, threaded=True, debug=False) except OSError: print() log('адрес ' + host + ':' + str(port) + ' недоступен', level='error')
class ProcessData: def __init__(self): self.car = Vehicle() # Verify that the Vehicle object was created # print("Vehicle made!") # self.find_own_pos("1362060585", "1362060062") self.minute_warning = False self.second_warning = False self.meter_message = False self.map = MapView() self.text_to_speech = TextToSpeech() def notify(self, ambulance_position_history): '''Called by receiver to notify the car about new ambulance position''' print('process_data notified') car_pos = self.find_own_pos(ambulance_position_history[0]['timestamp'], ambulance_position_history[1]['timestamp']) if car_pos is not None: if 2 == len(car_pos): if car_pos[0] is not None and car_pos[1] is not None: print(" ") # TODO: Handle edge case when first message arrive # Plot coordinates to map, updates everytime new data arrives. Run in Safari self.map.plot_coordinates(car_pos[1]['longitude'], car_pos[1]['latitude'], 'bs') self.map.plot_coordinates(ambulance_position_history[1]['longitude'], ambulance_position_history[1]['latitude'], 'rs') message = self.pick_message(car_pos[1], car_pos[0], ambulance_position_history[1], ambulance_position_history[0]) self.text_to_speech.play(message) self.map.show_map() time.sleep(0.75) def find_own_pos(self, first_timestamp, second_timestamp): '''Use timestamps from ambulance data to get car data''' relevant_pos = [] old_car = None new_car = None for position in self.car.position_history: #print(str(position['timestamp'])[:10]) if str(position['timestamp'])[:10] == str(first_timestamp+300)[:10]: relevant_pos.append(position) if str(position['timestamp'])[:10] == str(second_timestamp + 300)[:10]: relevant_pos.append(position) if len(relevant_pos) > 1: old_car = relevant_pos.pop() new_car = relevant_pos.pop() return new_car, old_car # if first_car_pos is not None and second_car_pos is not None: # self.car.position_history.clear() # Clear old data points # self.car.position_history.append(first_car_pos) # self.car.position_history.append(second_car_pos) def pick_message(self, new_car, old_car, new_ambu, old_ambu): '''Checks if the current data is relevant, and returns a integer for each case Keyword arguments: new_car -- A tuple containing the cars current position old_car -- A tuple containing the cars previous position new_ambu -- A tuple containing the ambulance current position old_ambu -- A tuple containing the ambulance previous position returns 0 if the situation is not relevant 1 if the ambulance is less than 20 sec behind 2 if the ambulance is less than 2 minutes behind 3 if the ambulance just passed the car and is less than 50 meters ahead ''' new_car_pos = (new_car['latitude'], new_car['longitude']) new_car_time = new_car['timestamp'] old_car_pos = (old_car['latitude'], old_car['longitude']) old_car_time = old_car['timestamp'] car_speed = Calculator.speed(new_car_pos[1], new_car_pos[0], old_car_pos[1], old_car_pos[0], (new_car_time - old_car_time)) print("Car time" + str(new_car_time-old_car_time)) print("Car speed: " + str(car_speed)) new_ambu_pos = (new_ambu['latitude'], new_ambu['longitude']) new_ambu_time = new_car['timestamp'] old_ambu_pos = (old_ambu['latitude'], old_ambu['longitude']) old_ambu_time = old_car['timestamp'] ambu_speed = Calculator.speed(new_ambu_pos[1], new_ambu_pos[0], old_ambu_pos[1], old_ambu_pos[0], (new_ambu_time - old_ambu_time )) print("Ambu speed: " + str(ambu_speed)) car_dir = self._find_direction(new_car_pos, old_car_pos) ambu_dir = self._find_direction(new_ambu_pos, old_ambu_pos) distance_km = Calculator.gps_to_kmeters(new_car_pos[1], new_car_pos[0], new_ambu_pos[1], new_ambu_pos[0]) print("Distance is : " + str(distance_km )) if not self._is_relevant(new_car_pos, car_speed, old_car_pos, new_ambu_pos, ambu_speed, old_ambu_pos): #0.05km is 50 meters. if distance_km < 0.05\ and car_dir.name == ambu_dir.name: if not self.meter_message: self.meter_message = True self.minute_warning = False self.second_warning = False return 3 else: return 0 return 0 time_to_intersection = Calculator.time_to_intersection( distance_km, ambu_speed, car_speed) print ('The vehicles are: ' + str(distance_km) + ' kms Appart. Time to intersect: ' + str(time_to_intersection)) #time to intersection is less than 20 sec, 1/3 of a minute if time_to_intersection <= (1/3): if not self.second_warning: self.second_warning = True return 1 return 0 if time_to_intersection <= 2: if not self.minute_warning: self.minute_warning = True self.meter_message = False return 2 return 0 def _is_relevant(self, new_car_pos, car_speed, old_car_pos, new_ambu_pos, ambu_speed, old_ambu_pos): '''Takes in the car and the ambulances current and previous postition. Returns whether the car should be notified or not, as a boolean Keyword arguments: new_car_pos -- A tuple containing the cars current position latitude first, longitude second car_speed -- A float containing the cars current speed old_car_pos -- A tuple containing the cars previous position latitude first, longitude second new_ambu_pos -- A tuple containing the ambulance current position latitude first, longitude second ambu_speed -- A float containing the ambulances current speed old_ambu_pos -- A tuple containing the ambulance previous position latitude first, longitude second ''' car_dir = self._find_direction(old_car_pos, new_car_pos) ambu_dir = self._find_direction(old_ambu_pos, new_ambu_pos) if ambu_speed <= 0: return False if car_dir != ambu_dir: print('Car not going the same direction as ambu') return False if not self._ambu_behind(new_car_pos, new_ambu_pos, car_dir): print ('Ambulance not behind car') return False distance_km = Calculator.gps_to_kmeters(float(new_car_pos[1]), float(new_car_pos[0]), float(new_ambu_pos[1]), float(new_ambu_pos[0])) time_to_intersection = Calculator.time_to_intersection( distance_km, ambu_speed, car_speed) print ('The vehicles are: ' + str(distance_km) + ' kms apart. Time to intersect: ' + str(time_to_intersection)) if time_to_intersection == 0: return False if time_to_intersection > 2: print('Ambulance is too far behind: ' + str(time_to_intersection)) return False return True def _find_direction(self, data1, data2): '''Find direction for vehicle, returns Direction enum Keyword arguments: data1 -- tuple with latitude and longitude from newest data data2 -- tuple with latitude and longitude from oldest data ''' lat_change = float(data2[0]) - float(data1[0]) long_change = float(data2[1]) - float(data1[1]) if lat_change == 0 and long_change == 0: return Direction.standing_still if fabs(lat_change) > fabs(long_change): if lat_change > 0: return Direction.north return Direction.south if long_change > 0: return Direction.east return Direction.west def _ambu_behind(self, car_pos, ambu_pos, direction): '''Decide if the ambu is in front of, or behind the car Keyword arguments: car_pos -- tuple with latitude and longitude from newest data of car ambu_pos -- tuple with latitude and longitude from newest data of ambu direction -- Direction of the two vehicles. Must be the same after comparing in is_relevant ''' if direction.name == 'north': return car_pos[0] > ambu_pos[0] if direction.name == 'south': return car_pos[0] < ambu_pos[0] if direction.name == 'east': return car_pos[1] > ambu_pos[1] if direction.name == 'west': return car_pos[1] < ambu_pos[1] return True
class Andrew(object): """the rap voice assisstant """ def __init__(self, detect_model="data/andrew2.net", lyrics_model="data/keras_model_1200.h5", lyrics_chars="data/chars.pkl"): # microphone self.mic = ResumableMicrophoneStream(SAMPLE_RATE, CHUNK_SIZE) # wake word detector self.detector = TriggerDetector(detect_model) # speech and language services self.speech_client = SpeechToText() self.luis = LangUnderstand() self.tts = TextToSpeech() # lyrics generator model self.lyrics_gen = LyricsGenerator(lyrics_model, lyrics_chars) self.pred_queue = DetectQueue(maxlen=5) self.is_wakeup = False # pytft display self.tft = TFTDisplay() self.tft_queue = queue.Queue() self.tft_thread = threading.Thread(target=self.tft_manage, args=()) self.tft_thread.daemon = True self.tft_thread.start() self.notify("hi_there") def notify(self, topic="hi_there", is_async=False, audio_path="data/audio"): # Notify with local preset audio files from os.path import join, isfile audio_file = join(audio_path, f"{topic}.wav") if not isfile(audio_file): return self.tts.play_file(audio_file, is_async) def generate_rap(self, topic="", beat_path="data/beat"): """Generate rap and play """ tts = self.tts lyrics_gen = self.lyrics_gen response = tts.generate_speech(f"hey, I can rap about {topic}") tts.play(response, True) # Generate based on topic lyrics_output = lyrics_gen.generate(topic) # Generate speech lyrics_speech = tts.generate_speech(lyrics_output) # Select beat beat_index = random.randint(0, 20) # Play beat and lyrics tts.play_file(f'{beat_path}/beat_{beat_index}.wav', True) tts.play(lyrics_speech) def get_weather_message(self, city="Ithaca"): import requests, json, os api_key = os.getenv('WEATHER_APIKEY') base_url = "https://api.openweathermap.org/data/2.5/weather?" city_name = f"{city},us" complete_url = f"{base_url}q={city_name}&units=imperial&APPID={api_key}" try: response = requests.get(complete_url) res = response.json() msg_weather = f"Today, it's {res['weather'][0]['description']} in {city}. " msg_temp = f"The temperature is {int(res['main']['temp'])} degrees." return msg_weather + msg_temp except: pass return "" def intent_recognize(self, text=""): """Recognize intent """ luis = self.luis tts = self.tts # Get result from language understanding engine luis_result = luis.predict(text) intent = luis_result.top_scoring_intent.intent if intent == "Freestyle": entities = luis_result.entities entity_topic = "rap" if (len(entities) > 0): entity = entities[0] cprint(f'The topic is {entity.entity}', 'cyan') entity_topic = entity.entity self.generate_rap(entity_topic) elif intent == "Weather": response = tts.generate_speech("I will tell you the weather in Ithaca.") tts.play(response) weather = self.get_weather_message() response = tts.generate_speech(weather) tts.play(response) else: self.notify("sorry") def tft_manage(self): """Manage TFT display through state """ self.tft.display_text("Andrew is waking up") status = {'state': 'None'} while True: if status['state'] is 'wait': self.tft.display_wave() elif status['state'] is 'listen': self.tft.display_wave((0, 255, 0)) # Update the status try: update = self.tft_queue.get(block=False) if update is not None: status = update except queue.Empty: continue def start(self): """Start listening and interacting """ tft = self.tft tts = self.tts # Init stream with self.mic as stream: self.tft_queue.put({'state': 'listen'}) while True: if not self.is_wakeup: stream.closed = False while not stream.closed: stream.audio_input = [] audio_gen = stream.generator() for chunk in audio_gen: if not self.is_wakeup: prob = self.detector.get_prediction(chunk) self.pred_queue.append(prob > 0.6) print('!' if prob > 0.6 else '.', end='', flush=True) if (self.pred_queue.count >= 2): self.notify("hi") cprint(' Trigger word detected! \n', 'magenta') self.pred_queue.clear() self.is_wakeup = True stream.pause() break else: cprint('Speech to text\n', 'green') time.sleep(1) stream.closed = False try: voice_command = self.speech_client.recognize(stream) cprint(f'{voice_command}\n', 'yellow') cprint('Recognition ended...\n', 'red') stream.pause() #tft.display_text(f'"{voice_command}"') if ("goodbye" in voice_command): self.notify("see_you") exit() if ("sorry" in voice_command): self.notify("its_ok") else: cprint('Recognize intents...', 'cyan') self.intent_recognize(voice_command) except Exception as e: cprint(f'Error: {e}', 'red') self.is_wakeup = False
class StatusManager: def __init__(self, config): # X. Initialize announcer. self.announcer = TextToSpeech() self.announcer.add_speech_text( "Initializing system. Please wait for a moment.") # X. Tf self.broadcaster = tf.TransformBroadcaster() self.listener = tf.TransformListener() # Prepare shared object. self.data = SharedData() # Publishers & Subscribers self.pubs = Publishers() self.subs = Subscribers(self.announcer) # Store tf if (not config.pose_init): while (self.subs.odom.get_object() is None): time.sleep(0.5) odom = self.subs.odom.get_object() odom_xyzrpy = create_xyzrpy_from_pose(odom.pose.pose) pose_stamped = PoseStamped() pose_stamped.header.stamp = rospy.Time.now() pose_stamped.header.frame_id = 'map' pose_stamped.pose = create_pose_from_xyzrpy(config.xyzrpy) self.data.last_valid_ndt_pose = pose_stamped self.data.last_valid_tf_odom_to_map = create_transform_from_pose( config.xyzrpy, odom_xyzrpy) # Initilize Pose self.initialize_pose(config) # Start checking thread self.checking_thread = threading.Thread(target=self.run_checking) self.checking_thread.start() # Start control self.run_control() # Terminate thread. self.announcer.terminate() self.checking_thread.join() def initialize_pose(self, config): init_xyzrpy = config.xyzrpy if (config.via_gnss): # X. Wait update from gnss. while (self.subs.gnss_pose.get_object() is None and not rospy.is_shutdown()): rospy.logwarn("Waiting for GNSS data to be received.") time.sleep(0.5) init_xyzrpy = create_xyzrpy_from_pose( self.subs.gnss_pose.get_object().pose) init_xyzrpy.roll = 0 init_xyzrpy.pitch = 0 init_xyzrpy.yaw = 0 # X. Start pose initializer if necessary. if (config.pose_init): self.announcer.add_speech_text( "Start pose initializer. This will take about twenty seconds.") # X. Switch to low resolution. self.pubs.cfg_voxel.publish(CONFIG_VOXEL_FILT_SEARCH()) req = FULL_INIT_REQUEST(init_xyzrpy) init_xyzrpy = call_pose_initialize_service(req) # X. Set high resolution mode. self.pubs.cfg_voxel.publish(CONFIG_VOXEL_FILT_RUN()) # X. Send for ndt localizer. self.pubs.cfg_ntd.publish(DEFAULT_CONFIG_NDT(init_xyzrpy)) # X. Send for amcl localizer. rospy.logwarn("amcl : {}".format( create_pose_stamped_with_cov(init_xyzrpy))) self.pubs.amcl_init.publish(create_pose_stamped_with_cov(init_xyzrpy)) # X. Notify self.announcer.add_speech_text( "Pose initialization is done. Start control.") def run_pose_initialize_if_necessary(self): if (self.data.last_pose_init_time is None \ or rospy.Time.now() - self.data.last_pose_init_time > rospy.Duration(LOCALIZATION_REINIT_MINIMUM_PERIOD)): if (LOCALIZATION_CNT_REINIT <= self.data.localization_not_reliable_cnt): self.data.pose_initializing = True INIT_POSE_STANDSTILL_COUNT = 10 wait_cnt = 0 while (self.data.standstill_count < INIT_POSE_STANDSTILL_COUNT): wait_cnt += 1 if (wait_cnt % 3): self.announcer.add_speech_text("Wait robot to stop.") time.sleep(1.0) if (self.data.last_valid_ndt_pose is not None \ and rospy.Time.now() - self.data.last_valid_ndt_pose.header.stamp < rospy.Duration(LOCALIZATION_ODOM_TRUST_PERIOD)): self.announcer.add_speech_text( "Reinitialize localization based on odometry. Robot will stop." ) config = StatusManagerConfig( True, False, create_xyzrpy_from_pose( self.data.last_cur_pose.get_object().pose)) else: self.announcer.add_speech_text( "Reinitialize localization based on GNSS. Robot will stop." ) config = StatusManagerConfig( True, True, XYZRPY(0.0, 0.0, 0.0, 0.0, 0.0, 0.0)) # X. This function takes time. self.initialize_pose(config) self.data.last_pose_init_time = rospy.Time.now() self.data.pose_initializing = False self.data.localization_not_reliable_cnt \ = max(0, self.data.localization_not_reliable_cnt - LOCALIZATION_CNT_REINIT / 2) def check_robot_stop_reason(self): obs_idx = self.subs.obst_idx.get_object() if (obs_idx is None or obs_idx.data == -1): self.data.stopped_due_to_obstable_cnt = 0 return twist_raw = self.subs.twist_raw.get_object() if (twist_raw is None or \ not twist_is_almost_zero(twist_raw.twist, STANDSTILL_VX_THR, STANDSTILL_WX_THR)): self.data.stopped_due_to_obstable_cnt = 0 return fin_wps = self.subs.final_wps.get_object() #ndt_pose = self.subs.ndt_pose.get_object() last_cur_pose = self.data.last_cur_pose.get_object() if (fin_wps is None or last_cur_pose is None): self.data.stopped_due_to_obstable_cnt = 0 return print("Distance to obstacle : {}".format( compute_distance_to_obstacle_on_waypoint(obs_idx.data, fin_wps, last_cur_pose.pose))) if (compute_distance_to_obstacle_on_waypoint( obs_idx.data, fin_wps, last_cur_pose.pose) < OBSTACLE_DIST): if (self.data.stopped_due_to_obstable_cnt % 3 == 0): self.announcer.add_speech_text( "Robot stops due to obstacle in front.") self.data.stopped_due_to_obstable_cnt += 1 if (WAIT_BEFORE_AVOID_OBST_SEC < \ self.data.stopped_due_to_obstable_cnt / STATUS_CHECK_HZ): if (is_avoidance_ok_waypoint(obs_idx.data, fin_wps)): self.announcer.add_speech_text( "Rerouting for obstacle avoidance.") header = Header() header.stamp = rospy.Time.now() self.pubs.astar_avoid.publish(header) self.data.stopped_due_to_obstable_cnt = 0 else: self.announcer.add_speech_text( "Rerouting is not allowed here. Keep waiting.") self.data.stopped_due_to_obstable_cnt = 0 def run_checking(self): rate = rospy.Rate(STATUS_CHECK_HZ) while not rospy.is_shutdown(): # X. Robot stuck reason. self.check_robot_stop_reason() # X. Localization status. self.run_pose_initialize_if_necessary() rate.sleep() def run_control(self): rate = rospy.Rate(CONTROL_LOOP_HZ) while not rospy.is_shutdown(): cur_time = rospy.Time.now() # X. Control command. twist_st = self.subs.twist_raw.get_object() if (twist_st is not None and not self.data.pose_initializing): self.pubs.cmd_vel.publish( self.subs.twist_raw.get_object().twist) else: self.pubs.cmd_vel.publish( create_twist(0.0, 0.0, 0.0, 0.0, 0.0, 0.0)) # X. Velocity routing. odom = self.subs.odom.get_object() if (odom is not None): twist_stamped = TwistStamped() twist_stamped.header.stamp = cur_time twist_stamped.twist = odom.twist.twist self.pubs.cur_vel.publish(twist_stamped) if (twist_is_almost_zero(odom.twist.twist, STANDSTILL_VX_THR, STANDSTILL_WX_THR)): self.data.standstill_count += 1 else: self.data.standstill_count = 0 # X. Check pose validity if (not self.data.pose_initializing): self.check_pose_validity() # X. Current pose self.publish_curpose(cur_time) rate.sleep() def check_pose_validity(self): # X. When map pose is unstable. ndt_pose = self.subs.ndt_pose.get_object() amcl_pose = self.subs.amcl_pose.get_object() if (ndt_pose is not None and amcl_pose is not None and \ not pose_is_almost_same(ndt_pose, amcl_pose.pose, \ LOCALIZATION_TRANS_THRESH, LOCALIZATION_YAW_THRESH)): self.data.localization_not_reliable_cnt = \ min(self.data.localization_not_reliable_cnt + 1, LOCALIZATION_CNT_REINIT) # X. When map pose is stable. else: # X. Extract last valid pose. if (ndt_pose is not None and amcl_pose is not None and \ pose_is_almost_same(ndt_pose, amcl_pose.pose, \ LOCALIZATION_TRANS_RELIABLE_THRESH, LOCALIZATION_YAW_RELIABLE_THRESH)): try: transform = self.listener.lookupTransform( 'map', 'odom', ndt_pose.header.stamp) self.data.last_valid_ndt_pose_queue.append( [ndt_pose, transform]) except (tf.LookupException, tf.ConnectivityException, tf.ExtrapolationException): pass # If reaches stable count, treat the first one as valid pose. if (len(self.data.last_valid_ndt_pose_queue) == LOCALIZATION_RELIABLE_CNT): self.data.last_valid_ndt_pose = self.data.last_valid_ndt_pose_queue[ 0][0] self.data.last_valid_tf_odom_to_map = self.data.last_valid_ndt_pose_queue[ 0][1] else: self.data.last_valid_ndt_pose_queue.clear() self.data.localization_not_reliable_cnt \ = max(0, self.data.localization_not_reliable_cnt - 1) def publish_curpose(self, cur_time): odom = self.subs.odom.get_object() if (odom is not None): pose_stamped = PoseStamped() pose_stamped.header.stamp = cur_time pose_stamped.header.frame_id = "map" pose_st = self.subs.ndt_pose.get_object() if (pose_st is not None and \ self.data.localization_not_reliable_cnt == 0 and \ not self.data.pose_initializing): # X. Use ndt pose as cur_pose pose_stamped.pose = pose_st.pose transform = create_tf_transform_from_pose( pose_st.pose, odom.pose.pose) # X. Send tf self.broadcaster.sendTransform(transform[0], transform[1], cur_time, 'odom', 'map') self.pubs.cur_pose.publish(pose_stamped) self.data.last_cur_pose.set_object(pose_stamped) self.data.ndt_pose_unreliable_cnt = 0 else: # X. Use odometry as cur_pose if (self.data.last_valid_tf_odom_to_map is not None): trans = self.data.last_valid_tf_odom_to_map[0] rot = self.data.last_valid_tf_odom_to_map[1] # X. Send tf self.broadcaster.sendTransform(trans, rot, cur_time, 'odom', 'map') trans_pose = transform_pose(odom.pose.pose, trans, rot) pose_stamped.pose = trans_pose self.pubs.cur_pose.publish(pose_stamped) self.data.last_cur_pose.set_object(pose_stamped) self.data.ndt_pose_unreliable_cnt += 1 if (self.data.ndt_pose_unreliable_cnt % (5 * CONTROL_LOOP_HZ) == 0): self.announcer.add_speech_text( "Ndt pose is unstable. ")
def main(): CHUNK = 1024 CHANNELS = 1 RATE = 16000 FORMAT = 'S16LE' BYTE_RATE = 32000 STATUS_KWS = 0 STATUS_ASR = 1 STATUS_TTS = 2 IP = '203.113.152.90' IP = '10.30.154.10' URI = 'ws://{}:8892/client/ws/speech'.format(IP) content_type = "audio/x-raw, layout=(string)interleaved, rate=(int){}, format=(string){}, channels=(int){}".format( RATE, FORMAT, CHANNELS) IS_ACTIVE = False p = pyaudio.PyAudio() stream = p.open(format=pyaudio.paInt16, channels=CHANNELS, rate=RATE, input=True, frames_per_buffer=CHUNK) sr = None sentence = None kws = KeywordSpotting(graph='graph/conv_kws.pb', labels='graph/conv_kws_labels.txt') tts = TextToSpeech(url='http://{}/hmm-stream/syn'.format(IP), voices='doanngocle.htsvoice') brain = aiml.Kernel() brain.learn("std-startup.xml") brain.respond("load aiml b") status = STATUS_KWS print("===============================") print("[INFO] Waiting keyword [xin chào | chào bot | hi bot] ...") while True: block = stream.read(CHUNK) if status == STATUS_KWS: if kws.spotting(block): status = STATUS_ASR print("[INFO] Keyword detected! Start recognize ...") sr = SpeechRecognize( url=URI + '?%s' % (urllib.parse.urlencode([("content-type", content_type)])), byterate=BYTE_RATE, one_sentence=False) sr_response = sr.get_response_queue() elif status == STATUS_ASR: if sr.is_alive(): sr.push_audio(block) while sr_response.qsize() > 0: msg = sr_response.get() if msg == "EOS": print("\rHuman: {}".format(sentence)) text = brain.respond(sentence) audio = tts.get_speech(text) status = STATUS_TTS tts.play_audio(audio) status = STATUS_ASR if "tạm biệt" in sentence: sr.close() else: sentence = msg print("\r-----: {}".format(msg), end='') else: status = STATUS_KWS print("===============================") print("========= GOOD BYE!!! =========") print("===============================") print("[INFO] Waiting keyword ...") elif status == STATUS_TTS: if sr.is_alive(): sr.push_audio(bytearray(CHUNK)) time.sleep(1. * CHUNK / BYTE_RATE)