def extract_minimal_nlu(dialogue_manager): turn = dialogue_manager.turn if 'trans' not in turn: user_utterance = turn['user']['utt'] else: user_utterance = turn['trans'] if utils.remove_punctuation(user_utterance) in ['yes', 'yeah', 'okay']: return {'sem': ['acknowledgment']} elif utils.remove_punctuation(user_utterance) in ['no']: return {'sem': ['reject']} elif utils.remove_punctuation(user_utterance) in ['uav']: #forcing nlu for the uav case return { 'd_act': ['instruction'], 'robot': rig_utils.parse_sentence_for_robots( user_utterance, dialogue_manager.rig_db.dynamic_mission_db) } elif turn['user']['nlu'] == 'no_nlu': if DEBUG: utils.print_dict(turn) logging.error('No NLU input') input() else: nlu_dacts = extract_user_dialogue_acts(turn['user']['nlu']) if nlu_dacts == ['ACKNOWLEDGMENT']: return {'sem': ['acknowledgment']} else: return extract_semantic_frames(turn['user']['nlu'], dialogue_manager) return None
def on_data(self, data): if 'results' in data: for result in data['results']: routing_key = 'asr.data' if result['final'] else 'asr.incremental_data' trans, final = self._parse_speech_data(data) if DEBUG: utils.print_dict({'transcript': trans, 'final': final}) asr_pub.send((json.dumps({'transcript': trans, 'final': final}),'{}.mic'.format(routing_key))) if 'speaker_labels' in data: for result in data['speaker_labels']: routing_key = 'asr.speaker_labels' if result['final'] else 'asr.incremental_speaker_labels' asr_pub.send((json.dumps(data), '{}.mic'.format(routing_key)))
def get_best_transition(self,last_turn,dialogue_manager): if 'user' in last_turn and 'nlu' in last_turn['user']: #look for frame semantics if 'user_turn' in self.transitions: if DEBUG: utils.print_dict(last_turn['user']['nlu']) return self._state_high_prob(self.transitions,dialogue_manager,nlu=last_turn['user']['nlu']) elif 'system_turn' in self.transitions: return self._state_high_prob(self.transitions,dialogue_manager) elif 'system_turn' in self.transitions: return self._state_high_prob(self.transitions,dialogue_manager) else: return self._get_highest_prob_all_states()
def get_nlu_transitions(self,all_transitions, nlu): ''' Combines d_acts and frame semantics to find the transitions for a given nlu :param all_transitions: :param nlu: :return: ''' tr_nlu = [] candidate_nlu = [] transitions = {} if 'd_act' in nlu: for d_act in nlu['d_act']: if d_act not in tr_nlu: tr_nlu.append(d_act) if d_act in all_transitions: candidate_nlu.append(d_act) transitions.update(all_transitions[d_act]) if 'sem' in nlu: for frame in nlu['sem']: if frame not in tr_nlu: tr_nlu.append(frame) if frame in all_transitions: candidate_nlu.append(frame) transitions.update(all_transitions[frame]) if '|'.join(tr_nlu) in all_transitions: return all_transitions['|'.join(tr_nlu)] elif 'other' in all_transitions: logging.warning(f"combined transition {'|'.join(tr_nlu)} could not be found") utils.print_dict(all_transitions) transitions.update(all_transitions['other']) elif len(candidate_nlu) == 0: logging.debug('No NLU match') transitions.update({'request_repeat':1.0}) return transitions
def get_all_utterances(): all_utt = [] for d in all_dialogues: if all_dialogues[d]['dialogue_id'] in dialogue_subsets['test']: #don't use the utterance in the test set to build the bow model logger.debug( f"skipping {all_dialogues[d]['dialogue_id']} since it belongs to the test set" ) continue try: all_utt += [ d['user'].translate(str_trans).lower() if 'user' in d else '<SIL>' for d in all_dialogues[d]['dialogue'] ] except: ### DBG ## utils.print_dict(all_dialogues[d]['dialogue']) input() return all_utt
def _get_highest_prob_all_states(self): ''' In case the nlu was not observable sample from all possible transitions :return: ''' sys_trans = {k: v/2 for k,v in self.transitions['system_turn'].items()} if DEBUG: utils.print_dict(sys_trans) usr_nlu_trans = {} #normalising system for nlu in self.transitions['user_turn']: usr_nlu_trans[nlu] = {k: v/len(self.transitions['user_turn']) for k,v in self.transitions['user_turn'][nlu].items()} #normalising user usr_trans = {} for nlu in usr_nlu_trans: for tr in usr_nlu_trans[nlu]: if tr in usr_trans: usr_trans[tr] += usr_nlu_trans[nlu][tr] else: usr_trans[tr] = usr_nlu_trans[nlu][tr] usr_trans = {k: v / 2 for k, v in usr_trans.items()} #merging trans = sys_trans for tr in usr_trans: if tr in trans: trans[tr] += usr_trans[tr] else: trans[tr] = usr_trans[tr] return self._state_high_prob(trans)
def _state_high_prob(self, dict_transitions, dialogue_manager, nlu = None): ''' given a dictionary with transition probabilities returns the state with the highest probability :param dict_transitions: :return: ''' if nlu is not None: transitions = self.get_nlu_transitions(copy(dict_transitions['user_turn']), nlu) else: transitions = copy(dict_transitions['system_turn']) if DEBUG: utils.print_dict(transitions) for st in list(set().union(dialogue_manager.used_no_transitions,dialogue_manager.subtask_travelled_states)): if st in transitions: try: del transitions[st] logging.debug(f'{st} removed from list of possible transitions') except KeyError: logging.error(f'{st} transition not avaialble in the current state, in subtask {dialogue_manager.dm_manager.subtask}') input() # masking transitions allowed_transitions = mask_transitions(list(transitions.keys()), dialogue_manager, dialogue_manager.nlg_states) for st in list(set(list(transitions.keys()))-set(allowed_transitions)): del transitions[st] if len(transitions) > 0: return max(transitions.items(), key=operator.itemgetter(1))[0] elif len(allowed_transitions) > 0: return random.choice(allowed_transitions) else: logging.error(f'No transitions for state {self.name}, in subtask {dialogue_manager.subtask}')
def __init__(self, message): audio_config = message[0] self.routing_ley = message self._rate = audio_config['rate'] self.chunk_size = audio_config['chunk'] self._num_channels = 1 self._buff = queue.Queue() self.closed = True self.start_time = get_current_time() self.restart_counter = 0 self.audio_input = [] self.last_audio_input = [] self.result_end_time = 0 self.is_final_end_time = 0 self.final_request_end_time = 0 self.bridging_offset = 0 self.last_transcript_was_final = False self.new_stream = True self.address = audio_config['address'] utils.print_dict(audio_config)
def write_transitions(self, output_dir): for st in self.states_nlg: state_dict = collections.OrderedDict({ 'name': self.states_nlg[st].name, 'common': { 'formulations': self.states_nlg[st].utterances, 'transition_states': { 'system_turn': self.states_nlg[st].transitions_system_turn } } }) if len(self.states_nlg[st].transitions_user_turn) > 0: state_dict['common']['transition_states'][ 'user_turn'] = self.states_nlg[st].transitions_user_turn utils.print_dict(state_dict) utils.setup_yaml() with open(os.path.join(output_dir, '{}.yaml'.format(st)), 'w') as of: yaml.dump(state_dict, of)
def extract_features_subset(subset): for d, dial in enumerate(subset): # creating entities dict for current dialogue feature_file = f"{os.path.join(output_feat_dir,str(dial))}.hdf5" if os.path.isfile(feature_file): logger.warning(f"feature file {feature_file} already exists") continue target_data = np.zeros((max_dialogue_len, 1)) df = {} if 'context_concat' in features_sizes: context_vector = rig_utils.create_orca_context_vector( rig_db.mission_db, rig_db.environment, skip_common=True) else: context_vector = rig_utils.create_orca_context_vector( rig_db.mission_db, rig_db.environment, gazebo_state=True) user_turns = [] previous_states = [] if 'convert_dialogue' in args.features: dialogue_history = [] for f in features_sizes: df[f] = np.ones((max_dialogue_len, features_sizes[f])) for t, turn in enumerate(all_dialogues[dial]['dialogue']): if 'action_mask' in turn: action_mask_available = True try: target_data[t] = valid_states.index(turn['current_state']) except: print(f"{turn['current_state']} not found") for da in target_data: print(valid_states[int(da[0])]) sys.exit() feat_values = {} if 'user' not in turn: turn['user'] = '******' user_turns.append(turn['user']) if 'wrd_emb' in args.features: # where the user turn has some content if args.embeddings_file: if emb_type == 'glove': feat_values[ 'wrd_emb_glove'] = emb_utils.get_utterance_embedding( glove_model, turn['user'], glove_dim, wrd_idx) else: feat_values[ 'wrd_emb_gn'] = emb_utils.get_utterance_embedding_w2v( emb, turn['user'], emb.vector_size) if 'bow' in args.features: feat_values['bow'] = bow.transform( [turn['user']]).toarray().reshape( (features_sizes['bow'], )) if 'context' in args.features: feat_values['context'] = rig_utils.get_orca_context_features( context_vector, turn['time'], turn['situation_db'], turn['current_state'], turn, gazebo_state=True, init_mission_db=rig_db.mission_db) context_vector = feat_values['context'] if 'previous_action' in args.features: feat_values['previous_action'] = get_previous_action( all_dialogues[dial]['dialogue'], t - 1) if 'api' in args.features: logging.info('No api features available for orca') continue if 'action_mask' in args.features: feat_values['action_mask'] = np.zeros((len(valid_states), )) mask_indexes = [ valid_states.index(a) for a in turn['action_mask'] ] for m in mask_indexes: feat_values['action_mask'][m] = 1 if 'nlu' in args.features: if 'nlu' in turn: if isinstance(turn['nlu'], str): nlu_tracker.get_nlu_vector(json.loads(turn['nlu'])) else: nlu_tracker.get_nlu_vector(turn['nlu']) else: #returns a vector with zeros nlu_tracker.get_default_vector() feat_values['nlu'] = nlu_tracker.nlu_vect for f in feat_values: #if f == 'context': # print(feat_values[f]) if isinstance(feat_values[f], list): if len(feat_values[f]) != features_sizes[f]: logging.error( 'Feature size does not match for {}'.format(f)) df[f][t, :] = np.array(feat_values[f], dtype=float) elif isinstance(feat_values[f], dict): if len(feat_values[f]) != features_sizes[f]: logging.error( 'Feature size does not match for {}'.format(f)) utils.print_dict(feat_values[f]) if f == 'context': utils.print_dict(turn['situation_db']) if f == 'nlu': nlu_tracker.get_default_vector() utils.print_dict(nlu_tracker.nlu_vect) input() try: df[f][t, :] = np.array(list(feat_values[f].values()), dtype=float) except: utils.print_dict(feat_values[f]) input(f) else: df[f][t, :] = feat_values[f] previous_states.append(turn['current_state']) logger.info(f'Creating file {feature_file}') if DEBUG: input() hcn_utils.data_to_h5(feature_file, df, target_data)
features_sizes['action_mask'] = len(valid_states) if 'previous_action' in args.features: features_sizes['previous_action'] = len(valid_states) if 'nlu' in args.features: nlu_tracker = nlu_utils.NLU_feat(rig_db.robots, rig_db.environment) features_sizes['nlu'] = len(nlu_tracker.nlu_vect) nlu_debug_file = os.path.join(aux_path, 'nlu_feat.txt') if not os.path.isfile(nlu_debug_file): with open(nlu_debug_file, 'w') as nlu_fp: nlu_fp.write('{}'.format('\n'.join( list(nlu_tracker.nlu_vect.keys())))) if DEBUG: utils.print_dict(features_sizes) input() total_feature_size = sum(features_sizes.values()) logger.info('Complete feature size {}'.format(total_feature_size)) for subset in dialogue_subsets: if len(dialogue_subsets[subset]) > 0: features_list = list(features_sizes.keys()) features_list.sort() output_feat_dir = os.path.join(aux_path, f"{'.'.join(features_list)}", subset) if not os.path.isdir(output_feat_dir):
def print_transitions(self): for st in self.states_nlg: utils.print_dict(self.states_nlg[st].transitions_user_turn) utils.print_dict(self.states_nlg[st].transitions_system_turn)
dact_lm = None if model_info.max_dial_len == 0: logger.error('No dialogue found') sys.exit() logger.info('Loading actions set') try: with open(os.path.join(task_dir, 'orca_action_set.txt'), 'r') as lfp: available_actions = pickle.load(lfp) except: with open(os.path.join(task_dir, 'orca_action_set.txt'), 'r') as lfp: available_actions = lfp.read().splitlines() if DEBUG: utils.print_dict(available_actions) input() if args.features is not None: model_info.feature_set = args.features else: # jchiyah fix for fragment in args.datadir.split(os.sep): if '.' in fragment: model_info.feature_set = fragment.split('.') args.features = model_info.feature_set break models_dir = os.path.join(args.datadir, model_info.config) if args.test_dir is None: args.test_dir = task_dir
def extract_semantic_frames(nlu, dialogue_manager): nlu_dict = {'d_act': [], 'sem': []} for d_act in nlu['dialogue_acts']: nlu_dict['d_act'].append(d_act['dialogue_act'].lower()) for frame in nlu['frame_semantics']: nlu_dict['sem'].append(frame['frame'].lower()) if frame['frame'] in ['Sending', 'Motion', 'Being_located']: if 'frame_elements' in frame: theme_tokens = [] goal_tokens = [] cotheme_tokens = [] for elements in frame['frame_elements']: if elements['frame_element'] == 'Theme': theme_tokens += elements['tokens'] if elements['frame_element'] == 'Goal': goal_tokens += elements['tokens'] if elements['frame_element'] == 'Cotheme': cotheme_tokens += elements['tokens'] if theme_tokens != []: theme_tokens = utils.intersect_lists( theme_tokens, d_act['tokens']) nlu_dict['robot'] = rig_utils.parse_sentence_for_robots( get_surface_tokens(theme_tokens, nlu['tokens']), dialogue_manager.rig_db.dynamic_mission_db) if goal_tokens != []: goal_tokens = utils.intersect_lists( goal_tokens, d_act['tokens']) nlu_dict['location'] = get_location_name( nlu, goal_tokens, dialogue_manager) if cotheme_tokens != []: cotheme_tokens = utils.intersect_lists( cotheme_tokens, d_act['tokens']) nlu_dict['robot'] += rig_utils.parse_sentence_for_robots( get_surface_tokens(cotheme_tokens, nlu['tokens']), dialogue_manager.rig_db.dynamic_mission_db) if frame['frame'] == 'Putting_out_fire': if 'frame_elements' in frame: place_tokens = [] agent_tokens = [] for elements in frame['frame_elements']: if elements['frame_element'] == 'Place': place_tokens += elements['tokens'] if elements['frame_element'] == 'Agent': agent_tokens += elements['tokens'] if place_tokens != []: place_tokens = utils.intersect_lists( place_tokens, d_act['tokens']) nlu_dict['location'] = get_location_name( nlu, place_tokens, dialogue_manager) if agent_tokens != []: agent_tokens = utils.intersect_lists( agent_tokens, d_act['tokens']) nlu_dict['robot'] = rig_utils.parse_sentence_for_robots( get_surface_tokens(agent_tokens, nlu['tokens']), dialogue_manager.rig_db.dynamic_mission_db) if frame['frame'] == 'Inspecting': if 'frame_elements' in frame: ground_tokens = [] inspector_tokens = [] for elements in frame['frame_elements']: if elements['frame_element'] == 'Ground': ground_tokens += elements['tokens'] if elements['frame_element'] == 'Inspector': inspector_tokens += elements['tokens'] if ground_tokens != []: ground_tokens = utils.intersect_lists( ground_tokens, d_act['tokens']) nlu_dict['location'] = get_location_name( nlu, ground_tokens, dialogue_manager) if inspector_tokens != []: inspector_tokens = utils.intersect_lists( inspector_tokens, d_act['tokens']) nlu_dict['robot'] = rig_utils.parse_sentence_for_robots( get_surface_tokens(inspector_tokens, nlu['tokens']), dialogue_manager.rig_db.dynamic_mission_db) if frame['frame'] == 'Perception_active': lexical_unit_lemmas = get_lemma_tokens(frame['lexical_unit'], nlu['tokens']) if 'hear' in lexical_unit_lemmas and 'see' not in lexical_unit_lemmas: #hear is always related to somehting that the user did not understand continue if 'frame_elements' in frame: agent_tokens = [] for elements in frame['frame_elements']: if elements['frame_element'] == 'Perceiver_agentive': agent_tokens += elements['tokens'] if agent_tokens != []: agent_tokens = utils.intersect_lists( agent_tokens, d_act['tokens']) nlu_dict['robot'] = rig_utils.parse_sentence_for_robots( get_surface_tokens(agent_tokens, nlu['tokens']), dialogue_manager.rig_db.dynamic_mission_db) if frame['frame'] == 'Using': if 'frame_elements' in frame: for elements in frame['frame_elements']: if elements['frame_element'] == 'Instrument': instrument_tokens = utils.intersect_lists( elements['tokens'], d_act['tokens']) nlu_dict[ 'robot'] = rig_utils.parse_sentence_for_robots( get_surface_tokens(instrument_tokens, nlu['tokens']), dialogue_manager.rig_db.dynamic_mission_db) if frame['frame'] == 'Emptying': if 'frame_elements' in frame: place_tokens = [] for elements in frame['frame_elements']: if elements['frame_element'] == 'Place': place_tokens += elements['tokens'] if place_tokens != []: place_tokens = utils.intersect_lists( place_tokens, d_act['tokens']) nlu_dict['location'] = get_location_name( nlu, place_tokens, dialogue_manager) if frame['frame'] == 'Bringing': if 'frame_elements' in frame: theme_tokens = [] for elements in frame['frame_elements']: if elements['frame_element'] == 'Theme': theme_tokens += elements['tokens'] if theme_tokens != []: theme_tokens = utils.intersect_lists( theme_tokens, d_act['tokens']) nlu_dict[ 'robot'] = rig_utils.parse_sentence_for_robots( get_surface_tokens(theme_tokens, nlu['tokens']), dialogue_manager.rig_db.dynamic_mission_db) if frame['frame'] == 'Being_in_category': if 'frame_elements' in frame: item_tokens = [] for elements in frame['frame_elements']: item_tokens += elements['tokens'] if item_tokens != []: item_tokens = utils.intersect_lists( item_tokens, d_act['tokens']) nlu_dict['robot'] = rig_utils.parse_sentence_for_robots( get_surface_tokens(item_tokens, nlu['tokens']), dialogue_manager.rig_db.dynamic_mission_db) surface_tokens = get_surface_tokens( item_tokens, nlu['tokens'], remove_pron_lemmas=True) if nlu_dict['robot'] != []: # when the robot is mentioned robot_names = rig_utils.parse_sentence_for_robot_name( surface_tokens, dialogue_manager.rig_db.dynamic_mission_db) item = surface_tokens for r in robot_names: item = re.sub(r, '', item) nlu_dict['item'] = item.strip() else: nlu_dict['item'] = surface_tokens if frame['frame'] == 'Being_located': if 'frame_elements' in frame: theme_tokens = [] for elements in frame['frame_elements']: theme_tokens += elements['tokens'] if theme_tokens != []: theme_tokens = utils.intersect_lists( theme_tokens, d_act['tokens']) robots = rig_utils.parse_sentence_for_robots( get_surface_tokens(theme_tokens, nlu['tokens']), dialogue_manager.rig_db.dynamic_mission_db) if len(robots) > 0: nlu_dict['robot'] = robots else: nlu_dict['location'] = get_location_name( nlu, theme_tokens, dialogue_manager) if frame['frame'] == 'Telling': # "[can you] tell me the <robot_info>?" parser if 'frame_elements' in frame: message_tokens = [] for elements in frame['frame_elements']: message_tokens += elements['tokens'] if message_tokens != []: message_tokens = utils.intersect_lists( message_tokens, d_act['tokens']) nlu_dict['robot'] = rig_utils.parse_sentence_for_robots( get_surface_tokens(message_tokens, nlu['tokens']), dialogue_manager.situation_db) surface_tokens = get_surface_tokens( message_tokens, nlu['tokens'], remove_pron_lemmas=True) if nlu_dict['robot'] != []: # when robot is mentioned print(surface_tokens) robot_names = rig_utils.parse_sentence_for_robot_name( surface_tokens, dialogue_manager.situation_db) item = surface_tokens for r in robot_names: item = re.sub(r, '', item) nlu_dict['item'] = item.strip() else: nlu_dict['item'] = surface_tokens for d_act in nlu['dialogue_acts']: if d_act['dialogue_act'] in ['INFORM', 'INSTRUCTION' ] and nlu['frame_semantics'] == []: content = get_surface_tokens(d_act['tokens'], nlu['tokens']) nlu_dict['robot'] = rig_utils.parse_sentence_for_robots( content, dialogue_manager.rig_db.dynamic_mission_db) nlu_dict['item'] = parse_for_items(content) else: logging.debug( f"dialogue act {nlu_dict['d_act']} not in above listed or empty frame semantic output" ) # avoid printing the whole thing if DEBUG: utils.print_dict(nlu['dialogue_acts']) utils.print_dict(nlu['frame_semantics']) print(nlu['sentence']) if DEBUG: utils.print_dict(nlu_dict) return nlu_dict
def compute_scores(y_test, x_test, y_pred, dialogue_setting, action_set, tst_files, model_info, mode='test', log_dir=None, action_mask_test=None, lm_dacts=None): ''' Computes scores for the orca data :param y_test: :param x_test: :param y_pred: :param rig_db: :param action_set: :param tst_files: :param log_dir: :param action_mask_test: :return: ''' test = np.argmax(y_test, axis=-1) pred = np.argmax(y_pred, axis=-1) correct_prediction = [] turn_acc_updates_merged = [] mission_success = [] end_state_success = [] turns_mission_success = [] turns_mission_success_succ_d = [] dialogue_success = [] correct_outcome = [] situated_da_success = [] d_perplex = [] preplex_diff = [] collaborative_task_success = [] actions_test = [] actions_pred = [] if model_info.generate_output and mode != 'train': generate_output = '{}_{}'.format('.'.join(model_info.feature_set), model_info.config) else: generate_output = None # loop over dialogues for d in range(test.shape[0]): # we are to produce entity replacement results dialogue_id = tst_files[d].split(os.sep)[-1].split('.')[0] dialogue_log = json.load( open( utils.get_turn_file(dialogue_id, log_dir=log_dir, extension=f'shcn.data.json'), 'r')) gt_dialogue_success = utils.get_dialogue_success(dialogue_id, log_dir=log_dir) all_turns = dialogue_log['dialogue'] if DEBUG: input(tst_files[d]) if 'condition' in dialogue_log and dialogue_log['condition'] != 'mixed': condition = dialogue_log['condition'] else: condition = 'user_plan' entity_replacement_success = [] generated_subtask_sequence = set(['inspect']) if generate_output: generated_dialogue = { 'condition': condition, 'dialogue_id': dialogue_log['dialogue_id'], 'model_info': model_info.__dict__, 'turns': [] } if not os.path.isdir( os.path.join(log_dir, 'generated', generate_output)): os.makedirs(os.path.join(log_dir, 'generated', generate_output)) output_file_generated_dialogue = os.path.join( log_dir, 'generated', generate_output, f"{dialogue_log['dialogue_id']}.json") if DEBUG: print(output_file_generated_dialogue) # getting the templates for each condition if condition == 'mixed': rig_db = dialogue_setting['user_plan'].rig_db interaction_manager = dialogue_setting['user_plan'] else: rig_db = dialogue_setting[condition].rig_db interaction_manager = dialogue_setting[condition] #reinitialising dialogue interaction_manager.reset_dialogue() end_generated_dialogue = False # controlling if dialogue has reached final state turns_to_complete_mission_pred = test.shape[ 1] #max numbers of turns allowed turns_to_complete_mission_test = len( all_turns) #end of the actual dialogue d_actions_pred = [] d_actions_test = [] # loop over turns for t in range(test.shape[1]): if isinstance(x_test, list): mask_vector_test = x_test[1][d][t] elif isinstance(x_test, dict): for f in x_test: if f.find('action_mask') > -1: break mask_vector_test = x_test[f][d][t] else: mask_vector_test = x_test[d, t, :] if not np.all(mask_vector_test == 1): if DEBUG: if 'prompt' in all_turns[t]: print(f"S: {all_turns[t]['prompt']}") if 'user' in all_turns[t]: print(f"U: {all_turns[t]['user']}") if 'user' in all_turns[t] and generate_output is not None: # gets the input for the user and updates the rig database interaction_manager.turn['user'] = {} interaction_manager.turn['user']['utt'] = all_turns[t][ 'user'] if 'nlu' in all_turns[t]['user']: interaction_manager.get_nlu( all_turns[t]['user']['nlu']) try: # sanity checks if action_set[test[d, t]] != all_turns[t]['current_state']: logging.error( 'Dialogue in the test vector is {}, but in the turns file is {}' .format(action_set[test[d, t]], all_turns[t]['current_state'])) except: print(t) if all_turns[t]['current_state'] == 'inform_emergency_solved': turns_to_complete_mission_test = t actions_test.append(test[d, t]) d_actions_test.append(test[d, t]) actions_pred.append(pred[d, t]) d_actions_pred.append(pred[d, t]) predicted_action = action_set[pred[d, t]] true_action = action_set[test[d, t]] rig_db.dynamic_mission_db['time_left'] = ( rig_db.dynamic_mission_db['mission_time'] - all_turns[t]['time']) if DEBUG: print(f"P: {predicted_action} [{true_action}]") if 'prompt' in all_turns[t]: plain_prompt = utils.remove_punctuation( all_turns[t]['prompt']).lower( ) # cleaning original prompt for comparison else: plain_prompt = None gen_turn_dict = { 'd_act_pred': action_set[pred[d, t]], 'd_act_gt': action_set[test[d, t]], 'utt_gt': plain_prompt } if 'user' in all_turns[t]: gen_turn_dict['user'] = all_turns[t]['user'] # if predicted actions are different then the entity replacement will also fail if predicted_action != true_action: entity_replacement_success.append(0) situated_da_success.append(0) gen_turn_dict['entity'] = False gen_turn_dict['situated_da'] = False if predicted_action in dialogue_setting[condition].nlg.status_updates and \ true_action in dialogue_setting[condition].nlg.status_updates: turn_acc_updates_merged.append(1) else: turn_acc_updates_merged.append(0) if action_set[pred[d, t]] in dialogue_setting[ 'user_plan'].nlg.fixed_dialogue_acts: if generate_output: gen_turn_dict['utt_pred'] = action_set[pred[d, t]] else: candidate_utterances = nlg_utils.generate_utt_entities( action_set[pred[d, t]], interaction_manager) if generate_output: if len(candidate_utterances) == 0: if DEBUG: print(action_set[pred[d, t]]) input(condition) gen_turn_dict['utt_pred'] = None else: gen_turn_dict['utt_pred'] = random.choice( candidate_utterances) # if action predicted is one of those which are not part of the template else: turn_acc_updates_merged.append(1) if predicted_action in [ 'user_turn', 'okay', 'activate.robot', 'deactivate.robot', 'holdon2seconds', 'gesture_emotion_neutral', 'actionperformed', 'repeat', 'BC', 'UNK', 'sorrycanyourepeatthat', 'yes', 'no' ]: if generate_output: gen_turn_dict['utt_pred'] = predicted_action gen_turn_dict['entity'] = True entity_replacement_success.append(1) # if action is part of the template checks if the exact utterance can be reconstructed from the templates else: utt_in_template = False candidate_utterances = nlg_utils.generate_utt_entities( predicted_action, interaction_manager) for ca in candidate_utterances: try: ca = utils.remove_punctuation(ca).lower() if ca == plain_prompt: utt_in_template = True break except KeyError: logging.warning(f'No prompt found in turn {t}') break if generate_output and len(candidate_utterances) > 0: gen_turn_dict['utt_pred'] = ca if utt_in_template: entity_replacement_success.append(1) gen_turn_dict['entity'] = True else: entity_replacement_success.append(0) gen_turn_dict['entity'] = False if DEBUG: logging.debug( 'error in entity replacement for dialogue act {}' .format(action_set[pred[d, t]])) utils.print_dict(candidate_utterances) utils.print_dict(all_turns[t]) input() if t > 0: if all_turns[t]['current_state'] in [ 'inform_moving', 'inform_robot_eta', 'inform_arrival', 'inform_inspection', 'inform_damage_inspection_robot', 'inform_returning_to_base', 'inform_robot_battery', 'inform_robot_progress', 'inform_robot_velocity', 'inform_robot_status' ]: if pred[d, t] == test[d, t]: situated_da_success.append(1) gen_turn_dict['situated_da'] = True else: situated_da_success.append(0) gen_turn_dict['situated_da'] = False if generate_output: generated_dialogue['turns'].append(gen_turn_dict) if action_set[pred[d, t]] == 'inform_emergency_status': generated_subtask_sequence.add('extinguish') interaction_manager.subtask = 'extinguish' if action_set[pred[d, t]] == 'inform_emergency_solved': generated_subtask_sequence.add('assess_damage') interaction_manager.subtask = 'assess_damage' turns_to_complete_mission_pred = t if generate_output: generated_dialogue['turns_to_complete'] = t #print(generated_subtask_sequence) # if one of the two ending states is reached if action_set[pred[d,t]] in ['mission_timeout', 'inform_mission_completed'] and \ not end_generated_dialogue: #check if sequence of sub-dialogues fullfils the task needs end_generated_dialogue = True #end_state_success.append(1) if DEBUG: utils.print_dict(list(generated_subtask_sequence)) if set(generated_subtask_sequence) == set( ['inspect', 'extinguish', 'assess_damage']): mission_success.append(1) if generate_output: generated_dialogue['mission_success'] = True else: mission_success.append(0) if generate_output: generated_dialogue['mission_success'] = False if action_set[pred[d,t]] == 'inform_mission_completed' and mission_success[-1] == 1 or\ action_set[pred[d,t]] == 'mission_timeout' and mission_success[-1] == 0: end_state_success.append(1) if generate_output: generated_dialogue['end_state'] = True else: end_state_success.append(0) if generate_output: generated_dialogue['end_state'] = False #end_state_success.append(1) if not end_generated_dialogue: if set(generated_subtask_sequence) == set( ['inspect', 'extinguish', 'assess_damage']): mission_success.append(1) if generate_output: generated_dialogue['mission_success'] = True else: mission_success.append(0) if generate_output: generated_dialogue['mission_success'] = False # if we reach the end of the dialogue without reaching one of the end states # dialogues are considered non-successful if generate_output: generated_dialogue['end_state'] = False end_state_success.append(0) if generate_output: generated_dialogue[ 'relative_turns_on_task'] = turns_to_complete_mission_pred / turns_to_complete_mission_test if gt_dialogue_success is not None: if gt_dialogue_success == mission_success[-1]: correct_outcome.append(1) if generate_output: generated_dialogue['correct_outcome'] = True generated_dialogue[ 'expected_outcome'] = gt_dialogue_success generated_dialogue['achieved_outcome'] = mission_success[ -1] else: correct_outcome.append(0) if generate_output: generated_dialogue['correct_outcome'] = False generated_dialogue[ 'expected_outcome'] = gt_dialogue_success generated_dialogue['achieved_outcome'] = mission_success[ -1] if gt_dialogue_success: # if the gt dialogue is successful then check if the output of # the model is also success. Else give a 0 if gt_dialogue_success == mission_success[-1]: collaborative_task_success.append(1) else: collaborative_task_success.append(0) # do not add anything to collaborative_task_success otherwise else: correct_outcome.append(0) #no meaning at all in this case turns_mission_success.append(turns_to_complete_mission_pred / turns_to_complete_mission_test) if mission_success[-1]: turns_mission_success_succ_d.append( turns_to_complete_mission_pred / turns_to_complete_mission_test) if sum(entity_replacement_success) == len(entity_replacement_success): if mission_success[-1] == 0: logging.error('Mission was successful') dialogue_success.append(1) if generate_output: generated_dialogue['dialogue_entity_success'] = True else: dialogue_success.append(0) if generate_output: generated_dialogue['dialogue_entity_success'] = False correct_prediction += entity_replacement_success #input() if generate_output: generated_dialogue['turn_accuracy'] = accuracy_score( d_actions_test, d_actions_pred) generated_dialogue['turn_accuracy_merged_updates'] = sum( turn_acc_updates_merged) / len(turn_acc_updates_merged) if lm_dacts: d_prob_pred = lm_dacts.p(' '.join( [t['d_act_pred'] for t in generated_dialogue['turns']])) d_perplex.append(utils.perplexity(d_prob_pred)) if generate_output: generated_dialogue['da_preplexity'] = utils.perplexity( d_prob_pred) d_prob_test = lm_dacts.p(' '.join( [t['current_state'] for t in all_turns])) preplex_diff.append( utils.perplexity(d_prob_test) - utils.perplexity(d_prob_pred)) if generate_output: generated_dialogue['da_preplexity_diff'] = utils.perplexity( utils.perplexity(d_prob_test) - utils.perplexity(d_prob_pred)) else: d_perplex.append(0.0) preplex_diff.append(0.0) if generate_output: with open(output_file_generated_dialogue, 'w') as dg_fp: json.dump(generated_dialogue, dg_fp, indent=2) if len(mission_success) != len(tst_files): utils.print_dict(all_turns) print(len(mission_success), len(end_state_success), len(correct_prediction)) input() return accuracy_score(actions_test,actions_pred),\ sum(correct_prediction)/len(correct_prediction),\ sum(mission_success)/len(mission_success),\ sum(dialogue_success)/len(dialogue_success),\ sum(end_state_success)/len(end_state_success),\ sum(correct_outcome)/len(correct_outcome),\ sum(turns_mission_success)/len(turns_mission_success),\ sum(situated_da_success)/len(situated_da_success),\ sum(d_perplex)/len(d_perplex),\ sum(preplex_diff)/len(preplex_diff),\ sum(turn_acc_updates_merged)/len(turn_acc_updates_merged),\ turns_mission_success_succ_d,\ actions_pred,actions_test,\ sum(collaborative_task_success) / len(collaborative_task_success) if len(collaborative_task_success) > 0 else np.nan