def __init__(self, participant_id, experimenter_name, experiment_phase): self.state_machine = Machine(self, states=self.states, transitions=self.transitions, initial='GAME_START') self.participant_id = participant_id self.experimenter_name = experimenter_name self.experiment_phase = experiment_phase if not self.experiment_phase == 'experiment': print(str(self.experiment_phase) + " was not 'experiment'") exit() # alternate between which model makes active learning choices based on participant ID if int(self.participant_id[2]) % 2 == 1: print("USING WORD MODEL FOR ACTIVE LEARNING") self.active_student_model = self.student_word_model else: print("USING PHONEME MODEL FOR ACTIVE LEARNING") self.active_student_model = self.student_phoneme_model # Initializes a new audio recorder object if one hasn't been created self.recorder = AudioRecorder(self.participant_id, self.experimenter_name, self.experiment_phase) # Tell robot to look at Tablet self.ros_node_mgr.init_ros_node()
def __init__(self, participant_id, experimenter_name, experiment_phase): self.state_machine = Machine(self, states=self.states, transitions=self.transitions, initial='GAME_START') self.participant_id = participant_id self.experimenter_name = experimenter_name self.experiment_phase = experiment_phase if not self.experiment_phase == 'posttest': print(str(self.experiment_phase) + " was not 'posttest'") exit() # Initializes a new audio recorder object if one hasn't been created self.recorder = AudioRecorder(self.participant_id, self.experimenter_name, self.experiment_phase) # Tell robot to look at Tablet self.ros_node_mgr.init_ros_node()
def msg_evaluator(ispy_action_msg): """Calls the respective functions for each part of the action msg """ #Removes object position from ispy_action_msg if ispy_action_msg.clickedObjectName != "": object_name = "" for letter in ispy_action_msg.clickedObjectName: if letter == "-": break object_name += letter self.origText = object_name #Initializes a new audio recorder object if one hasn't been created if self.recorder == None: self.recorder = AudioRecorder() speakingStage(ispy_action_msg.speakingStage)
class TapGamePosttestFSM: # pylint: disable=no-member, too-many-instance-attributes """ An FSM for the Tap Game. Contains Game Logic and some nodes for interacting w the Unity "View" """ round_index = 0 player_score = 0 robot_score = 0 student_word_model = StudentWordModel() student_phoneme_model = StudentPhonemeModel() agent_model = AgentModel() pronunciation_utils = PronunciationUtils() ros_node_mgr = ROSNodeMgr() current_round_word = "" current_round_action = None audio_file = None letters = None # the graphemes/letters under consideration this rd passed = None # whether each letter corresponded to a phoneme above the threshold scores = None # the actual score of the phoneme each letter corresponds to player_won_round_tap = None # did the player buzz in first player_passed_round = None # was the players score high enough to pass the round states = [ 'GAME_START', 'ROUND_START', 'ROUND_ACTIVE', 'PLAYER_PRONOUNCE', 'ROBOT_PRONOUNCE', 'SHOW_RESULTS', 'ROUND_RESOLVE', 'GAME_FINISHED' ] transitions = [ { 'trigger': 'init_first_round', 'source': 'GAME_START', 'dest': 'ROUND_START', 'after': 'on_init_first_round' }, { 'trigger': 'start_round', 'source': 'ROUND_START', 'dest': 'ROUND_ACTIVE', 'after': 'on_start_round' }, { 'trigger': 'robot_ring_in', 'source': 'ROUND_ACTIVE', 'dest': 'ROBOT_PRONOUNCE', 'after': 'on_robot_ring_in' }, { 'trigger': 'player_ring_in', 'source': 'ROUND_ACTIVE', 'dest': 'PLAYER_PRONOUNCE', 'after': 'on_player_ring_in' }, { 'trigger': 'player_pronounce_eval', 'source': 'PLAYER_PRONOUNCE', 'dest': 'SHOW_RESULTS', 'after': 'on_player_pronounce_eval' }, { 'trigger': 'robot_pronounce_eval', 'source': 'ROBOT_PRONOUNCE', 'dest': 'SHOW_RESULTS', 'after': 'on_robot_pronounce_eval' }, { 'trigger': 'resolve_round', 'source': 'SHOW_RESULTS', 'dest': 'ROUND_RESOLVE', 'after': 'on_round_resolve' }, { 'trigger': 'handle_round_end', 'source': 'ROUND_RESOLVE', 'dest': 'ROUND_START', 'conditions': 'is_not_last_round', 'after': 'on_reset_round' }, { 'trigger': 'handle_round_end', 'source': 'ROUND_RESOLVE', 'dest': 'GAME_FINISHED', 'conditions': 'is_last_round', 'after': 'on_game_finished' }, { 'trigger': 'replay_game', 'source': 'GAME_FINISHED', 'dest': 'GAME_START', 'after': 'on_game_replay' }, ] def __init__(self, participant_id, experimenter_name, experiment_phase): self.state_machine = Machine(self, states=self.states, transitions=self.transitions, initial='GAME_START') self.participant_id = participant_id self.experimenter_name = experimenter_name self.experiment_phase = experiment_phase if not self.experiment_phase == 'posttest': print(str(self.experiment_phase) + " was not 'posttest'") exit() # Initializes a new audio recorder object if one hasn't been created self.recorder = AudioRecorder(self.participant_id, self.experimenter_name, self.experiment_phase) # Tell robot to look at Tablet self.ros_node_mgr.init_ros_node() #self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_AT_TABLET) def on_init_first_round(self): """ Called when the game registers with the controller Should send msg to Unity game telling it the word to load for the first round """ print("got to init_first round!") # get the next robot action self.current_round_action = ActionSpace.DONT_RING #send message every 2s in case it gets dropped def send_msg_til_received(): while (self.state == "ROUND_START"): self.current_round_word = self.student_word_model.curriculum[ self.round_index] self.ros_node_mgr.send_game_cmd( TapGameCommand.INIT_ROUND, json.dumps(self.current_round_word)) print('sent command!') print(self.state) time.sleep(5) thread.start_new_thread(send_msg_til_received, ()) def on_start_round(self): """ Called when the game registers that the round initialization is done Should send msg to Unity game telling it to begin countdown and make buzzers active """ print('got to start round cb') self.ros_node_mgr.send_game_cmd(TapGameCommand.START_ROUND) def on_player_ring_in(self): """ Called when the human player has tapped their buzzer to ring in Should send msg to Unity game telling it to load the pronunciation screen And also start recording from the phone for 5 seconds + writing to wav """ print('got to player ring in cb') self.player_won_round_tap = True #SEND SHOW_PRONUNCIATION_PAGE MSG self.recorder.start_recording(self.current_round_word, self.round_index, RECORD_TIME_MS) #time.sleep(RECORD_TIME_MS / 1000.0) self.recorder.stop_recording() ##Evaluates the action message ## If given a word to evaluate and done recording send the information to speechace if self.current_round_word and \ self.recorder.has_recorded % 2 == 0 and\ self.recorder.has_recorded != 0: # If you couldn't find the android audio topic, automatically pass # instead of using the last audio recording if not self.recorder.valid_recording: self.letters = list(self.current_round_word) self.passed = ['0'] * len(self.letters) self.scores = [0] * len(self.letters) print("NO RECORDING SO YOU AUTOMATICALLY FAIL") self.audio_file = 'None' else: self.audio_file = self.recorder.WAV_OUTPUT_FILENAME_PREFIX + self.current_round_word + '_' + str( self.recorder.recording_index) + '.wav' print("SENDING TO SPEECHACE") word_score_list = self.recorder.speechace(self.audio_file) print("WORD SCORE LIST") print(word_score_list) # if we didn't record, there will be no word score list if word_score_list: for word_results in word_score_list: print("Message for ROS") self.letters, self.passed, self.scores = \ self.pronunciation_utils.process_speechace_word_results(word_results) print(self.letters) print(self.passed) print(self.scores) else: self.letters = list(self.current_round_word) self.passed = ['0'] * len(self.letters) self.scores = [0] * len(self.letters) print('NO RECORDING, SO YOU AUTO-FAIL!!') self.player_pronounce_eval() else: print('THIS SHOULD NEVER HAPPEN') def on_player_pronounce_eval(self): """ Called after the human player has pronounced their buzzer to ring in send wav from previous step to speech ace, get results, update model, and send message to game to display results """ print('got to player pronounce eval cb') # Get the actual results here avg_phone_score = (sum(self.scores) / len(self.scores) ) #TODO: do this over phonemes, not letters! print("AVG PHONE SCORE WAS" + str(avg_phone_score)) means, variances = self.student_word_model.train_and_compute_posterior( [self.current_round_word], [avg_phone_score / 100.0]) phonemes_raw = pronouncing.phones_for_word( self.current_round_word.lower())[0].split(' ') arpabet_phonemes = [ ''.join(filter(lambda c: not c.isdigit(), pho)) for pho in phonemes_raw ] print("ARPABET PHONEME FOR ROUND") print(arpabet_phonemes) print(self.scores) self.student_phoneme_model.words_so_far.append( self.current_round_word ) #tell the phoneme model which word for active learning phoneme_means, phoneme_vars = self.student_phoneme_model.train_and_compute_posterior( arpabet_phonemes, [x / 100.0 for x in self.scores]) print("LATEST MEANS / VARS") print(self.student_word_model.curriculum) print(means) print(variances) results_params = {} results_params['letters'] = self.letters results_params['passed'] = self.passed results_params['scores'] = [ str(x) for x in self.scores ] #convert to string before sending over json (ask Sam why) self.ros_node_mgr.send_game_cmd(TapGameCommand.SHOW_RESULTS, json.dumps(results_params)) time.sleep(SHOW_RESULTS_TIME_MS / 1000.0) self.resolve_round() def on_robot_pronounce_eval(self): """ Called after the robot has 'pronounced' a word. Should send mesage to Game telling it to show results handle_round_end() to transition to next round """ time.sleep(SIMULATED_ROBOT_RESULTS_TIME_MS / 1000.0) results_params = {} results_params['letters'] = self.letters results_params['passed'] = self.passed results_params['scores'] = [ str(x) for x in self.scores ] #convert to string before sending over json (ask Sam why) self.ros_node_mgr.send_game_cmd(TapGameCommand.SHOW_RESULTS, json.dumps(results_params)) time.sleep(SHOW_RESULTS_TIME_MS / 1000.0) self.resolve_round() def on_round_resolve(self): """ Called after finishing a round in the game Should optionally send command for robot to react to result, then send cmd to game to reset for the next round """ print('got to round reset') time.sleep(.5) #wait for half a second so the lookat can go through #ROBOT REACTION LOGIC - consider using fidget text? avg_phoneme_score = (sum(self.scores) / len(self.scores)) print('AVG PHONEME SCORE WAS') print(avg_phoneme_score) if avg_phoneme_score >= PASSING_SCORE_THRESHOLD: #if child got the word right self.player_score += 1 self.player_passed_round = True else: self.player_passed_round = False self.handle_round_end() def on_reset_round(self): self.ros_node_mgr.send_game_cmd(TapGameCommand.RESET_NEXT_ROUND) def on_game_finished(self): """ Called when we have completed 'max_rounds' rounds in a game. Sends msg to the Unity game to load the game end screen """ print('got to game finished') self.ros_node_mgr.send_game_cmd(TapGameCommand.SHOW_GAME_END) def on_game_replay(self): """ Called when the player wants to replay the game after finishing. Sends msg to the Unity game to reset the game and start over """ # reset all state variables (rounds, score) self.player_score = 0 self.robot_score = 0 self.init_first_round() #reset student model here if needed #self.ros_node_mgr.send_game_cmd(TapGameCommand.RESTART_GAME) #START GAME OVER #self.ros_node_mgr.send_game_cmd() def on_log_received(self, data): """ Rospy Callback for when we get log messages """ if data.message in FSM_LOG_MESSAGES: if data.message == TapGameLog.CHECK_IN: print('Game Checked in!') if data.message == TapGameLog.GAME_START_PRESSED: time.sleep(500 / 1000.0) self.init_first_round( ) # makes state transition + calls self.on_init_first_round() if data.message == TapGameLog.INIT_ROUND_DONE: print('done initializing') self.start_round() if data.message == TapGameLog.START_ROUND_DONE: print('I heard Start Round DONE. Waiting for player input') if data.message == TapGameLog.PLAYER_RING_IN: print('Player Rang in!') self.player_ring_in() if data.message == TapGameLog.ROBOT_RING_IN: print('Robot Rang in!') self.robot_ring_in() if data.message == TapGameLog.PLAYER_BEAT_ROBOT: self.player_beat_robot = True if data.message == TapGameLog.RESET_NEXT_ROUND_DONE: print('Game Done Resetting Round! Now initing new round') self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_AT_TABLET) self.ros_node_mgr.publish_round_summary( self.round_index, self.current_round_action, self.current_round_word, self.player_won_round_tap, self.player_passed_round, self.audio_file, self.letters, self.scores, self.passed, self.player_score, self.robot_score, self.student_word_model.curriculum, self.student_word_model.means, self.student_word_model.variances, self.student_phoneme_model.curriculum, self.student_phoneme_model.means, self.student_phoneme_model.variances) self.round_index += 1 self.player_beat_robot = False self.current_round_word = self.student_word_model.curriculum[ self.round_index] self.ros_node_mgr.send_game_cmd( TapGameCommand.INIT_ROUND, json.dumps(self.current_round_word)) if data.message == TapGameLog.SHOW_GAME_END_DONE: print('GAME OVER! WAIT FOR RESET SINAL') if data.message == TapGameLog.RESTART_GAME: self.replay_game() else: print('NOT A REAL MESSAGE?!?!?!?') def is_last_round(self): """ used by FSM to determine whether to start next round or end game """ return (self.round_index == len(self.student_word_model.curriculum) - 1) def is_not_last_round(self): """ used by FSM to determine whether to start next round or end game """ return not self.is_last_round()
class TapGameFSM: # pylint: disable=no-member, too-many-instance-attributes """ An FSM for the Tap Game. Contains Game Logic and some nodes for interacting w the Unity "View" """ round_index = 0 #total number of rounds player_round_index = 0 # number of rounds player buzzed in first (ie how much data acquired) max_rounds = 20 #game ends when someone gets to this score max_player_rounds = 20 #game ends when we have acquired this many observations of player data #max_score = 5 #game ends when someone gets to this score player_score = 0 robot_score = 0 active_student_model = None student_word_model = StudentWordModel() student_phoneme_model = StudentPhonemeModel() agent_model = AgentModel() pronunciation_utils = PronunciationUtils() ros_node_mgr = ROSNodeMgr() current_round_word = "" current_round_action = None audio_file = None letters = None # the graphemes/letters under consideration this rd passed = None # whether each letter corresponded to a phoneme above the threshold scores = None # the actual score of the phoneme each letter corresponds to player_won_round_tap = None # did the player buzz in first player_passed_round = None # was the players score high enough to pass the round states = ['GAME_START', 'ROUND_START', 'ROUND_ACTIVE', 'PLAYER_PRONOUNCE', 'ROBOT_PRONOUNCE', 'SHOW_RESULTS', 'ROUND_RESOLVE', 'GAME_FINISHED'] transitions = [ {'trigger': 'init_first_round', 'source': 'GAME_START', 'dest': 'ROUND_START', 'after': 'on_init_first_round'}, {'trigger': 'start_round', 'source': 'ROUND_START', 'dest': 'ROUND_ACTIVE', 'after': 'on_start_round'}, {'trigger': 'robot_ring_in', 'source': 'ROUND_ACTIVE', 'dest': 'ROBOT_PRONOUNCE', 'after': 'on_robot_ring_in'}, {'trigger': 'player_ring_in', 'source': 'ROUND_ACTIVE', 'dest': 'PLAYER_PRONOUNCE', 'after': 'on_player_ring_in'}, {'trigger': 'player_pronounce_eval', 'source': 'PLAYER_PRONOUNCE', 'dest': 'SHOW_RESULTS', 'after': 'on_player_pronounce_eval'}, {'trigger': 'robot_pronounce_eval', 'source': 'ROBOT_PRONOUNCE', 'dest': 'SHOW_RESULTS', 'after': 'on_robot_pronounce_eval'}, {'trigger': 'resolve_round', 'source': 'SHOW_RESULTS', 'dest': 'ROUND_RESOLVE', 'after': 'on_round_resolve'}, {'trigger': 'handle_round_end', 'source': 'ROUND_RESOLVE', 'dest': 'ROUND_START', 'conditions': 'is_not_last_round', 'after': 'on_reset_round'}, {'trigger': 'handle_round_end', 'source': 'ROUND_RESOLVE', 'dest': 'GAME_FINISHED', 'conditions': 'is_last_round', 'after': 'on_game_finished'}, {'trigger': 'replay_game', 'source': 'GAME_FINISHED', 'dest': 'GAME_START', 'after': 'on_game_replay'}, ] def __init__(self, participant_id, experimenter_name, experiment_phase): self.state_machine = Machine(self, states=self.states, transitions=self.transitions, initial='GAME_START') self.participant_id = participant_id self.experimenter_name = experimenter_name self.experiment_phase = experiment_phase if not self.experiment_phase == 'experiment': print(str(self.experiment_phase) + " was not 'experiment'") exit() # alternate between which model makes active learning choices based on participant ID if int(self.participant_id[2]) % 2 == 1: print("USING WORD MODEL FOR ACTIVE LEARNING") self.active_student_model = self.student_word_model else: print("USING PHONEME MODEL FOR ACTIVE LEARNING") self.active_student_model = self.student_phoneme_model # Initializes a new audio recorder object if one hasn't been created self.recorder = AudioRecorder(self.participant_id, self.experimenter_name, self.experiment_phase) # Tell robot to look at Tablet self.ros_node_mgr.init_ros_node() #self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_AT_TABLET) def on_init_first_round(self): """ Called when the game registers with the controller Should send msg to Unity game telling it the word to load for the first round """ print("got to init_first round!") # get the next robot action self.current_round_action = self.agent_model.get_next_action() #send message every 2s in case it gets dropped def send_msg_til_received(): while(self.state == "ROUND_START"): self.current_round_word = self.active_student_model.get_next_best_word(self.current_round_action) self.ros_node_mgr.send_game_cmd(TapGameCommand.INIT_ROUND, json.dumps(self.current_round_word)) print('sent command!') print(self.state) time.sleep(5) thread.start_new_thread(send_msg_til_received, ()) def on_start_round(self): """ Called when the game registers that the round initialization is done Should send msg to Unity game telling it to begin countdown and make buzzers active """ print('got to start round cb') self.ros_node_mgr.send_game_cmd(TapGameCommand.START_ROUND) if self.current_round_action == ActionSpace.RING_ANSWER_CORRECT: time.sleep(WAIT_TO_BUZZ_TIME_MS / 1000.0) self.ros_node_mgr.send_robot_cmd(RobotBehaviors.RING_ANSWER_CORRECT) self.ros_node_mgr.send_game_cmd(TapGameCommand.ROBOT_RING_IN) def on_robot_ring_in(self): """ Called when the game registers that the robot 'buzzed in' Should send msg to Unity game telling it to load robot pronunciation screen """ print('got to robot ring in cb') self.player_won_round_tap = False self.audio_file = 'None' # Send message to robot telling it to pronounce # Wait a few seconds, pronounce word, then wait again time.sleep((RECORD_TIME_MS / 2) / 1000.0) if self.current_round_action == ActionSpace.RING_ANSWER_CORRECT: self.ros_node_mgr.send_robot_cmd(RobotBehaviors.PRONOUNCE_CORRECT, self.current_round_word) self.letters = list(self.current_round_word) self.passed = ['1'] * len(self.letters) #robot always gets it right if intentional ring self.scores = [1] * len(self.letters) #robot always gets it right if intentional ring elif self.current_round_action == ActionSpace.LATE_RING: self.ros_node_mgr.send_robot_cmd(RobotBehaviors.PRONOUNCE_WRONG_SOUND) time.sleep(.5) self.ros_node_mgr.send_robot_cmd(RobotBehaviors.PRONOUNCE_WRONG_SPEECH) self.letters = list(self.current_round_word) self.passed = ['0'] * len(self.letters) #robot always gets it wrong if late ring self.scores = [0] * len(self.letters) #robot always gets it right if intentional ring time.sleep((RECORD_TIME_MS / 2) / 1000.0) # Move to evaluation phase self.robot_pronounce_eval() def player_beat_robot(self): """ This function details what happens when the robot wants to buzz, but gets beat by the human """ # print("PLAYER BEAT ROBOT TO THE PUNCH!") # tmp = [int(x) for x in self.passed] # passed_ratio = (sum(tmp) / len(tmp)) #TODO: do this over phonemes, not letters! # print("ROUND PASSED RATIO WAS" + str(passed_ratio)) def on_player_ring_in(self): """ Called when the human player has tapped their buzzer to ring in Should send msg to Unity game telling it to load the pronunciation screen And also start recording from the phone for 5 seconds + writing to wav """ print('got to player ring in cb') self.player_won_round_tap = True self.recorder.start_recording(self.current_round_word, self.round_index, RECORD_TIME_MS) #time.sleep(RECORD_TIME_MS / 1000.0) self.recorder.stop_recording() ## If given a word to evaluate and done recording send the information to speechace if self.current_round_word and \ self.recorder.has_recorded % 2 == 0 and\ self.recorder.has_recorded != 0: # If you couldn't actually record, automatically fail if not self.recorder.valid_recording: self.letters = list(self.current_round_word) self.passed = ['1'] * len(self.letters) self.scores = [100] * len(self.letters) print ("NO RECORDING SO YOU AUTOMATICALLY FAIL") self.audio_file = 'None' else: self.audio_file = self.recorder.WAV_OUTPUT_FILENAME_PREFIX + self.current_round_word + '_' + str(self.recorder.recording_index) + '.wav' print("SENDING TO SPEECHACE") word_score_list = self.recorder.speechace(self.audio_file) print("WORD SCORE LIST") print(word_score_list) # if we didn't get a good response from speechace, there will be no word score list if word_score_list: for word_results in word_score_list: print("Message for ROS") self.letters, self.passed, self.scores = \ self.pronunciation_utils.process_speechace_word_results(word_results) print(self.letters) print(self.passed) print(self.scores) else: self.letters = list(self.current_round_word) self.passed = ['1'] * len(self.letters) self.scores = [100] * len(self.letters) print('NO RECORDING, SO YOU AUTO-FAIL!!') self.player_pronounce_eval() else: print('THIS SHOULD NEVER HAPPEN') def on_player_pronounce_eval(self): """ Called after the human player has pronounced their buzzer to ring in send wav from previous step to speech ace, get results, update model, and send message to game to display results """ print('got to player pronounce eval cb') # Get the actual results here avg_phone_score = (sum(self.scores) / len(self.scores)) print("AVG PHONE SCORE WAS" + str(avg_phone_score)) word_means, word_variances = self.student_word_model.train_and_compute_posterior([self.current_round_word], [avg_phone_score / 100.0]) phonemes_raw = pronouncing.phones_for_word(self.current_round_word.lower())[0].split(' ') arpabet_phonemes = [''.join(filter(lambda c: not c.isdigit(), pho)) for pho in phonemes_raw] print("ARPABET PHONEME FOR ROUND") print(arpabet_phonemes) print(self.scores) self.student_phoneme_model.words_so_far.append(self.current_round_word) #tell the phoneme model which word for active learning try: phoneme_means, phoneme_vars = self.student_phoneme_model.train_and_compute_posterior(arpabet_phonemes, [x / 100.0 for x in self.scores]) except: print("COULD NOT UPDATE MODEL!!!") # [avg_phone_score / 100.0]) # print("LATEST WORD MEANS / VARS") # print(self.student_word_model.curriculum) # print(word_means) # print(word_variances) # # print(self.student_phoneme_model.curriculum) # print(word_means) # print(word_variances) results_params = {} results_params['letters'] = self.letters results_params['passed'] = self.passed results_params['scores'] = [str(x) for x in self.scores] #convert to string before sending over json (ask Sam why) self.ros_node_mgr.send_game_cmd(TapGameCommand.SHOW_RESULTS, json.dumps(results_params)) time.sleep(SHOW_RESULTS_TIME_MS / 1000.0) self.resolve_round() def on_robot_pronounce_eval(self): """ Called after the robot has 'pronounced' a word. Should send mesage to Game telling it to show results handle_round_end() to transition to next round """ time.sleep(SIMULATED_ROBOT_RESULTS_TIME_MS / 1000.0) results_params = {} results_params['letters'] = self.letters results_params['passed'] = self.passed results_params['scores'] = [str(x) for x in self.scores] #convert to string before sending over json (ask Sam why) self.ros_node_mgr.send_game_cmd(TapGameCommand.SHOW_RESULTS, json.dumps(results_params)) time.sleep(SHOW_RESULTS_TIME_MS / 1000.0) self.resolve_round() def on_round_resolve(self): """ Called after finishing a round in the game Should optionally send command for robot to react to result, then send cmd to game to reset for the next round """ print('got to round reset') self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_CENTER) time.sleep(.5) #wait for half a second so the lookat can go through #ROBOT REACTION LOGIC avg_phoneme_score = (sum(self.scores) / len(self.scores)) print('AVG PHONEME SCORE WAS') print(avg_phoneme_score) self.round_index += 1 #increment total round count if self.player_won_round_tap: #reactions to player winning tap self.player_round_index += 1 #increment player round count if (not self.current_round_action == ActionSpace.DONT_RING): # if the robot intended to ring but got beaten, if avg_phoneme_score >= PASSING_SCORE_THRESHOLD: #if the robot was beaten and the child got the word right self.player_score += 1 self.player_passed_round = True self.ros_node_mgr.send_robot_cmd(RobotBehaviors.REACT_TO_BEAT_CORRECT) else: self.player_passed_round = False self.ros_node_mgr.send_robot_cmd(RobotBehaviors.REACT_TO_BEAT_WRONG) else: #regular response to human ring-in if avg_phoneme_score >= PASSING_SCORE_THRESHOLD: #if child got the word right self.player_score += 1 self.player_passed_round = True self.ros_node_mgr.send_robot_cmd(RobotBehaviors.REACT_PLAYER_CORRECT) else: self.player_passed_round = False self.ros_node_mgr.send_robot_cmd(RobotBehaviors.REACT_PLAYER_WRONG) else: #robot rang in if avg_phoneme_score >= PASSING_SCORE_THRESHOLD: #if robot got the word right self.robot_score += 1 self.ros_node_mgr.send_robot_cmd(RobotBehaviors.REACT_ROBOT_CORRECT) self.player_passed_round = False else: self.player_passed_round = False self.ros_node_mgr.send_robot_cmd(RobotBehaviors.REACT_ROBOT_WRONG) self.handle_round_end() def on_reset_round(self): self.ros_node_mgr.send_game_cmd(TapGameCommand.RESET_NEXT_ROUND) def on_game_finished(self): """ Called when we have completed 'max_rounds' rounds in a game. Sends msg to the Unity game to load the game end screen """ print('got to game finished') time.sleep(1) if self.player_score >= self.robot_score: self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOSE_MOTION) self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOSE_SPEECH) else: self.ros_node_mgr.send_robot_cmd(RobotBehaviors.WIN_MOTION) self.ros_node_mgr.send_robot_cmd(RobotBehaviors.WIN_SPEECH) self.ros_node_mgr.send_game_cmd(TapGameCommand.SHOW_GAME_END) def on_game_replay(self): """ Called when the player wants to replay the game after finishing. Sends msg to the Unity game to reset the game and start over """ # reset all state variables (rounds, score) self.player_score = 0 self.robot_score = 0 self.init_first_round() #reset student model here if needed def on_log_received(self, data): """ Rospy Callback for when we get log messages """ if data.message in FSM_LOG_MESSAGES: if data.message == TapGameLog.CHECK_IN: print('Game Checked in!') self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_CENTER) self.ros_node_mgr.send_robot_cmd(RobotBehaviors.SAY_HI) if data.message == TapGameLog.GAME_START_PRESSED: self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_AT_TABLET) time.sleep(500 / 1000.0) self.init_first_round() # makes state transition + calls self.on_init_first_round() if data.message == TapGameLog.INIT_ROUND_DONE: print('done initializing') self.start_round() if data.message == TapGameLog.START_ROUND_DONE: print('I heard Start Round DONE. Waiting for player input') #send various prompts to elicit response from player if self.current_round_action == ActionSpace.DONT_RING: thread.start_new_thread(self.send_player_prompts_til_input_received, ()) if data.message == TapGameLog.PLAYER_RING_IN: print('Player Rang in!') self.player_ring_in() if data.message == TapGameLog.ROBOT_RING_IN: print('Robot Rang in!') self.robot_ring_in() if data.message == TapGameLog.PLAYER_BEAT_ROBOT: self.player_beat_robot = True if data.message == TapGameLog.RESET_NEXT_ROUND_DONE: print('Game Done Resetting Round! Now initing new round') self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_AT_TABLET) self.ros_node_mgr.publish_round_summary(self.round_index, self.current_round_action, self.current_round_word, self.player_won_round_tap, self.player_passed_round, self.audio_file, self.letters, self.scores, self.passed, self.player_score, self.robot_score, self.student_word_model.curriculum, self.student_word_model.means, self.student_word_model.variances, self.student_phoneme_model.curriculum, self.student_phoneme_model.means, self.student_phoneme_model.variances) self.player_beat_robot = False self.current_round_action = self.agent_model.get_next_action() self.current_round_word = self.active_student_model.get_next_best_word(self.current_round_action) self.ros_node_mgr.send_game_cmd(TapGameCommand.INIT_ROUND, json.dumps(self.current_round_word)) if data.message == TapGameLog.SHOW_GAME_END_DONE: print('GAME OVER! WAIT FOR RESET SINAL') if data.message == TapGameLog.RESTART_GAME: self.replay_game() else: print('NOT A REAL MESSAGE?!?!?!?') def is_last_round(self): """ used by FSM to determine whether to start next round or end game """ #return (self.robot_score >= self.max_score or self.player_score >= self.max_score) #return (self.round_index >= self.max_rounds) return (self.player_round_index >= self.max_player_rounds) def is_not_last_round(self): """ used by FSM to determine whether to start next round or end game """ return not self.is_last_round() #send message every 2s in case it gets dropped def send_player_prompts_til_input_received(self): """ after starting the round, wait for some time and make some noises + behaviors to prompt human to ring in. Ring in and do something if enough time goes by """ start_time = time.time() rang_in = False time.sleep(5) while(self.state == "ROUND_ACTIVE"): time.sleep(3 + randint(1,3)) if time.time() - start_time > 12 and not rang_in: self.current_round_action = ActionSpace.LATE_RING # Change to "LATE_RING" self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LATE_RING) self.ros_node_mgr.send_game_cmd(TapGameCommand.ROBOT_RING_IN) rang_in = True elif self.state == "ROUND_ACTIVE": self.ros_node_mgr.send_robot_cmd(RobotBehaviors.PLAYER_RING_PROMPT) self.ros_node_mgr.send_robot_cmd(RobotBehaviors.EYE_FIDGET) print('sent command!') print(self.state)
class TapGamePracticeFSM: # pylint: disable=no-member, too-many-instance-attributes """ An FSM for the Tap Game. Contains Game Logic and some nodes for interacting w the Unity "View" """ round_index = 0 max_rounds = 6 #game ends after this many rounds player_score = 0 robot_score = 0 agent_model = AgentModel() pronunciation_utils = PronunciationUtils() ros_node_mgr = ROSNodeMgr() current_round_word = "" current_round_action = None audio_file = None letters = None # the graphemes/letters under consideration this rd passed = None # whether each letter corresponded to a phoneme above the threshold scores = None # the actual score of the phoneme each letter corresponds to player_won_round_tap = None # did the player buzz in first player_passed_round = None # was the players score high enough to pass the round states = [ 'GAME_START', 'ROUND_START', 'ROUND_ACTIVE', 'PLAYER_PRONOUNCE', 'ROBOT_PRONOUNCE', 'SHOW_RESULTS', 'ROUND_RESOLVE', 'GAME_FINISHED' ] # Scripted Sequence of Actions and Words to use in Game practice_actions = [ ActionSpace.DONT_RING, ActionSpace.DONT_RING, ActionSpace.DONT_RING, ActionSpace.RING_ANSWER_CORRECT, ActionSpace.RING_ANSWER_CORRECT, ActionSpace.DONT_RING, ActionSpace.RING_ANSWER_CORRECT ] practice_words = ["FORK", "FROG", "FISH", "DUCK", "SUN", "RABBIT", "DAD"] transitions = [ { 'trigger': 'init_first_round', 'source': 'GAME_START', 'dest': 'ROUND_START', 'after': 'on_init_first_round' }, { 'trigger': 'start_round', 'source': 'ROUND_START', 'dest': 'ROUND_ACTIVE', 'after': 'on_start_round' }, { 'trigger': 'robot_ring_in', 'source': 'ROUND_ACTIVE', 'dest': 'ROBOT_PRONOUNCE', 'after': 'on_robot_ring_in' }, { 'trigger': 'player_ring_in', 'source': 'ROUND_ACTIVE', 'dest': 'PLAYER_PRONOUNCE', 'after': 'on_player_ring_in' }, { 'trigger': 'player_pronounce_eval', 'source': 'PLAYER_PRONOUNCE', 'dest': 'SHOW_RESULTS', 'after': 'on_player_pronounce_eval' }, { 'trigger': 'robot_pronounce_eval', 'source': 'ROBOT_PRONOUNCE', 'dest': 'SHOW_RESULTS', 'after': 'on_robot_pronounce_eval' }, { 'trigger': 'resolve_round', 'source': 'SHOW_RESULTS', 'dest': 'ROUND_RESOLVE', 'after': 'on_round_resolve' }, { 'trigger': 'handle_round_end', 'source': 'ROUND_RESOLVE', 'dest': 'ROUND_START', 'conditions': 'is_not_last_round', 'after': 'on_reset_round' }, { 'trigger': 'handle_round_end', 'source': 'ROUND_RESOLVE', 'dest': 'GAME_FINISHED', 'conditions': 'is_last_round', 'after': 'on_game_finished' }, { 'trigger': 'replay_game', 'source': 'GAME_FINISHED', 'dest': 'GAME_START', 'after': 'on_game_replay' }, ] def __init__(self, participant_id, experimenter_name, experiment_phase): self.state_machine = Machine(self, states=self.states, transitions=self.transitions, initial='GAME_START') self.participant_id = participant_id self.experimenter_name = experimenter_name self.experiment_phase = experiment_phase if not self.experiment_phase == 'practice': print(str(self.experiment_phase) + " was not 'practice'") exit() # Initializes a new audio recorder object if one hasn't been created self.recorder = AudioRecorder(self.participant_id, self.experimenter_name, self.experiment_phase) # Tell robot to look at Tablet self.ros_node_mgr.init_ros_node() def on_init_first_round(self): """ Called when the game registers with the controller Should send msg to Unity game telling it the word to load for the first round """ print("got to init_first round!") # get the next robot action self.current_round_action = self.practice_actions[self.round_index] #send message every 2s in case it gets dropped def send_msg_til_received(): while (self.state == "ROUND_START"): self.current_round_word = self.practice_words[self.round_index] self.ros_node_mgr.send_game_cmd( TapGameCommand.INIT_ROUND, json.dumps(self.current_round_word)) print('sent command!') print(self.state) time.sleep(5) thread.start_new_thread(send_msg_til_received, ()) def on_start_round(self): """ Called when the game registers that the round initialization is done Should send msg to Unity game telling it to begin countdown and make buzzers active """ print('got to start round cb') self.ros_node_mgr.send_game_cmd(TapGameCommand.START_ROUND) if self.current_round_action == ActionSpace.RING_ANSWER_CORRECT: time.sleep(WAIT_TO_BUZZ_TIME_MS / 1000.0) self.ros_node_mgr.send_robot_cmd( RobotBehaviors.RING_ANSWER_CORRECT) self.ros_node_mgr.send_game_cmd(TapGameCommand.ROBOT_RING_IN) def on_robot_ring_in(self): """ Called when the game registers that the robot 'buzzed in' Should send msg to Unity game telling it to load robot pronunciation screen """ print('got to robot ring in cb') self.player_won_round_tap = False self.audio_file = 'None' # Send message to robot telling it to pronounce # Wait a few seconds, pronounce word, then wait again time.sleep((RECORD_TIME_MS / 2) / 1000.0) if self.current_round_action == ActionSpace.RING_ANSWER_CORRECT: self.ros_node_mgr.send_robot_cmd(RobotBehaviors.PRONOUNCE_CORRECT, self.current_round_word) self.letters = list(self.current_round_word) self.passed = ['1'] * len( self.letters) #robot always gets it right if intentional ring self.scores = [1] * len( self.letters) #robot always gets it right if intentional ring elif self.current_round_action == ActionSpace.LATE_RING: self.ros_node_mgr.send_robot_cmd( RobotBehaviors.PRONOUNCE_WRONG_SOUND) self.ros_node_mgr.send_robot_cmd( RobotBehaviors.PRONOUNCE_WRONG_SPEECH) self.letters = list(self.current_round_word) self.passed = ['0'] * len( self.letters) #robot always gets it wrong if late ring self.scores = [0] * len( self.letters) #robot always gets it right if intentional ring time.sleep((RECORD_TIME_MS / 2) / 1000.0) # Move to evaluation phase self.robot_pronounce_eval() def player_beat_robot(self): """ This function details what happens when the robot wants to buzz, but gets beat by the human """ # print("PLAYER BEAT ROBOT TO THE PUNCH!") # tmp = [int(x) for x in self.passed] # passed_ratio = (sum(tmp) / len(tmp)) #TODO: do this over phonemes, not letters! # print("ROUND PASSED RATIO WAS" + str(passed_ratio)) def on_player_ring_in(self): """ Called when the human player has tapped their buzzer to ring in Should send msg to Unity game telling it to load the pronunciation screen And also start recording from the phone for 5 seconds + writing to wav """ print('got to player ring in cb') self.player_won_round_tap = True #SEND SHOW_PRONUNCIATION_PAGE MSG self.recorder.start_recording(self.current_round_word, self.round_index, RECORD_TIME_MS) #time.sleep(RECORD_TIME_MS / 1000.0) self.recorder.stop_recording() ##Evaluates the action message ## If given a word to evaluate and done recording send the information to speechace if self.current_round_word and \ self.recorder.has_recorded % 2 == 0 and\ self.recorder.has_recorded != 0: # If you couldn't find the android audio topic, automatically pass # instead of using the last audio recording if not self.recorder.valid_recording: self.letters = list(self.current_round_word) self.passed = ['0'] * len(self.letters) self.scores = [0] * len(self.letters) print("NO RECORDING SO YOU AUTOMATICALLY FAIL") self.audio_file = 'None' else: self.audio_file = self.recorder.WAV_OUTPUT_FILENAME_PREFIX + self.current_round_word + '_' + str( self.recorder.recording_index) + '.wav' print("SENDING TO SPEECHACE") word_score_list = self.recorder.speechace(self.audio_file) print("WORD SCORE LIST") print(word_score_list) # if we didn't record, there will be no word score list if word_score_list: for word_results in word_score_list: print("Message for ROS") self.letters, self.passed, self.scores = \ self.pronunciation_utils.process_speechace_word_results(word_results) print(self.letters) print(self.passed) print(self.scores) else: self.letters = list(self.current_round_word) self.passed = ['0'] * len(self.letters) self.scores = [0] * len(self.letters) print('NO RECORDING, SO YOU AUTO-FAIL!!') self.player_pronounce_eval() else: print('THIS SHOULD NEVER HAPPEN') def on_player_pronounce_eval(self): """ Called after the human player has pronounced their buzzer to ring in send wav from previous step to speech ace, get results, update model, and send message to game to display results """ print('got to player pronounce eval cb') # Get the actual results here avg_phone_score = (sum(self.scores) / len(self.scores) ) #TODO: do this over phonemes, not letters! print("AVG PHONE SCORE WAS" + str(avg_phone_score)) results_params = {} results_params['letters'] = self.letters results_params['passed'] = self.passed results_params['scores'] = [ str(x) for x in self.scores ] #convert to string before sending over json (ask Sam why) self.ros_node_mgr.send_game_cmd(TapGameCommand.SHOW_RESULTS, json.dumps(results_params)) time.sleep(SHOW_RESULTS_TIME_MS / 1000.0) self.resolve_round() def on_robot_pronounce_eval(self): """ Called after the robot has 'pronounced' a word. Should send mesage to Game telling it to show results handle_round_end() to transition to next round """ time.sleep(SIMULATED_ROBOT_RESULTS_TIME_MS / 1000.0) results_params = {} results_params['letters'] = self.letters results_params['passed'] = self.passed results_params['scores'] = [ str(x) for x in self.scores ] #convert to string before sending over json (ask Sam why) self.ros_node_mgr.send_game_cmd(TapGameCommand.SHOW_RESULTS, json.dumps(results_params)) time.sleep(SHOW_RESULTS_TIME_MS / 1000.0) self.resolve_round() def on_round_resolve(self): """ Called after finishing a round in the game Should optionally send command for robot to react to result, then send cmd to game to reset for the next round """ print('got to round reset') self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_CENTER) time.sleep(.5) #wait for half a second so the lookat can go through #ROBOT REACTION LOGIC - consider using fidget text? avg_phoneme_score = (sum(self.scores) / len(self.scores)) print('AVG PHONEME SCORE WAS') print(avg_phoneme_score) if self.player_won_round_tap: #reactions to player winning tap if (not self.current_round_action == ActionSpace.DONT_RING ): # if the robot intended to ring but got beaten, if avg_phoneme_score >= PASSING_SCORE_THRESHOLD: #if the robot was beaten and the child got the word right self.player_score += 1 self.player_passed_round = True self.ros_node_mgr.send_robot_cmd( RobotBehaviors.REACT_TO_BEAT_CORRECT) else: self.player_passed_round = False self.ros_node_mgr.send_robot_cmd( RobotBehaviors.REACT_TO_BEAT_WRONG) else: #regular response to human ring-in if avg_phoneme_score >= PASSING_SCORE_THRESHOLD: #if child got the word right self.player_score += 1 self.player_passed_round = True self.ros_node_mgr.send_robot_cmd( RobotBehaviors.REACT_PLAYER_CORRECT) else: self.player_passed_round = False self.ros_node_mgr.send_robot_cmd( RobotBehaviors.REACT_PLAYER_WRONG) else: #robot rang in if avg_phoneme_score >= PASSING_SCORE_THRESHOLD: #if robot got the word right self.robot_score += 1 self.ros_node_mgr.send_robot_cmd( RobotBehaviors.REACT_ROBOT_CORRECT) self.player_passed_round = False else: self.player_passed_round = False self.ros_node_mgr.send_robot_cmd( RobotBehaviors.REACT_ROBOT_WRONG) self.handle_round_end() def on_reset_round(self): self.ros_node_mgr.send_game_cmd(TapGameCommand.RESET_NEXT_ROUND) def on_game_finished(self): """ Called when we have completed 'max_rounds' rounds in a game. Sends msg to the Unity game to load the game end screen """ print('got to game finished') time.sleep(1) if self.player_score >= self.robot_score: self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOSE_MOTION) self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOSE_SPEECH) else: self.ros_node_mgr.send_robot_cmd(RobotBehaviors.WIN_MOTION) self.ros_node_mgr.send_robot_cmd(RobotBehaviors.WIN_SPEECH) self.ros_node_mgr.send_game_cmd(TapGameCommand.SHOW_GAME_END) def on_game_replay(self): """ Called when the player wants to replay the game after finishing. Sends msg to the Unity game to reset the game and start over """ # reset all state variables (rounds, score) self.player_score = 0 self.robot_score = 0 self.init_first_round() #reset student model here if needed #self.ros_node_mgr.send_game_cmd(TapGameCommand.RESTART_GAME) #START GAME OVER #self.ros_node_mgr.send_game_cmd() def on_log_received(self, data): """ Rospy Callback for when we get log messages """ if data.message in FSM_LOG_MESSAGES: if data.message == TapGameLog.CHECK_IN: print('Game Checked in!') self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_CENTER) self.ros_node_mgr.send_robot_cmd(RobotBehaviors.SAY_HI) if data.message == TapGameLog.GAME_START_PRESSED: self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_AT_TABLET) time.sleep(500 / 1000.0) self.init_first_round( ) # makes state transition + calls self.on_init_first_round() if data.message == TapGameLog.INIT_ROUND_DONE: print('done initializing') self.start_round() if data.message == TapGameLog.START_ROUND_DONE: print('I heard Start Round DONE. Waiting for player input') if data.message == TapGameLog.PLAYER_RING_IN: print('Player Rang in!') self.player_ring_in() if data.message == TapGameLog.ROBOT_RING_IN: print('Robot Rang in!') self.robot_ring_in() if data.message == TapGameLog.PLAYER_BEAT_ROBOT: self.player_beat_robot = True if data.message == TapGameLog.RESET_NEXT_ROUND_DONE: print('Game Done Resetting Round! Now initing new round') self.ros_node_mgr.send_robot_cmd(RobotBehaviors.LOOK_AT_TABLET) #self.ros_node_mgr.publish_round_summary(self.round_index, self.current_round_action, self.current_round_word, # self.player_won_round_tap, self.player_passed_round, self.audio_file, self.letters, self.scores, # self.passed,self.player_score, self.robot_score, self.practice_words, # [.5 * len(self.practice_words)],[.3 * len(self.practice_words)]) self.round_index += 1 self.player_beat_robot = False self.current_round_action = self.practice_actions[ self.round_index] self.current_round_word = self.practice_words[self.round_index] self.ros_node_mgr.send_game_cmd( TapGameCommand.INIT_ROUND, json.dumps(self.current_round_word)) if data.message == TapGameLog.SHOW_GAME_END_DONE: print('GAME OVER! WAIT FOR RESET SINAL') if data.message == TapGameLog.RESTART_GAME: self.replay_game() else: print('NOT A REAL MESSAGE?!?!?!?') def is_last_round(self): """ used by FSM to determine whether to start next round or end game """ return (self.round_index >= self.max_rounds) def is_not_last_round(self): """ used by FSM to determine whether to start next round or end game """ return not self.is_last_round()