def dialog_train(self, dialog): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() loss = 0. # iterate through dialog for (u,r) in dialog: # print("Here in the dialog loop") u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = torch.autograd.Variable(torch.from_numpy(np.concatenate((u_ent_features, u_emb, u_bow), axis=0))).float() # print(features) # get action mask action_mask = torch.autograd.Variable(torch.from_numpy(at.action_mask())) r = torch.autograd.Variable(torch.LongTensor([r])) # print(r) # forward propagation # train step loss += self.net.train_step(features, r, action_mask) return loss/len(dialog)
def dialog_train(self, dialog): ################################################################### if self.lang_type == 'eng': from modules.entities import EntityTracker from modules.data_utils import Data from modules.actions import ActionTracker from modules.bow import BoW_encoder elif self.lang_type == 'kor': from modules.entities_kor import EntityTracker from modules.data_utils_kor import Data from modules.actions_kor import ActionTracker from modules.bow_kor import BoW_encoder ################################################################### et = EntityTracker() at = ActionTracker(et) # reset state in network_type self.net.reset_state() loss = 0. for (u, r) in dialog: u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_bow = self.bow_enc.encode(u) if self.is_emb: u_emb = self.emb.encode(u) if self.is_action_mask: action_mask = at.action_mask() # print(u, r) # print(u_ent_features) # print('================================') # print(u_emb) # print('================================') # print(u_bow) # print('================================') # print(action_mask) # concatenated features if self.is_action_mask and self.is_emb: features = np.concatenate( (u_ent_features, u_emb, u_bow, action_mask), axis=0) elif self.is_action_mask and not (self.is_emb): features = np.concatenate((u_ent_features, u_bow, action_mask), axis=0) elif not (self.is_action_mask) and self.is_emb: features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) elif not (self.is_action_mask) and not (self.is_emb): features = np.concatenate((u_ent_features, u_bow), axis=0) # forward propagation with cumulative loss if self.is_action_mask: loss += self.net.train_step(features, r, action_mask) else: loss += self.net.train_step(features, r) return loss / len(dialog)
def main(in_dataset_folder, in_model_folder, in_no_ood_evaluation): rev_vocab, kb, action_templates, config = load_model(in_model_folder) test_dialogs, test_indices = read_dialogs(os.path.join( in_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) et = EntityTracker(kb) at = ActionTracker(None, et) at.set_action_templates(action_templates) vocab = {word: idx for idx, word in enumerate(rev_vocab)} X, context_features, action_masks, y = make_dataset_for_hierarchical_lstm( test_dialogs, test_indices, vocab, et, at, **config) net = HierarchicalLSTM(config, context_features.shape[-1], action_masks.shape[-1]) net.restore(in_model_folder) eval_stats_full_dataset = evaluate_advanced( net, (X, context_features, action_masks, y), test_dialogs, at.action_templates) print( 'Full dataset: {} turns overall, {} turns after the first OOD'.format( eval_stats_full_dataset['total_turns'], eval_stats_full_dataset['total_turns_after_ood'])) print('Accuracy:') accuracy = eval_stats_full_dataset[ 'correct_turns'] / eval_stats_full_dataset['total_turns'] accuracy_after_ood = eval_stats_full_dataset['correct_turns_after_ood'] / eval_stats_full_dataset['total_turns_after_ood'] \ if eval_stats_full_dataset['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_full_dataset['correct_post_ood_turns'] / eval_stats_full_dataset['total_post_ood_turns'] \ if eval_stats_full_dataset['total_post_ood_turns'] != 0 \ else 0 print( 'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'. format(accuracy, accuracy_after_ood, accuracy_post_ood)) print('Loss : {:.3f}'.format(eval_stats_full_dataset['avg_loss'])) if in_no_ood_evaluation: eval_stats_no_ood = evaluate_advanced( net, (X, context_features, action_masks, y), test_indices, at.action_templates, ignore_ood_accuracy=True) print('Accuracy (OOD turns ignored):') accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood[ 'total_turns'] accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \ if eval_stats_no_ood['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_no_ood['correct_post_ood_turns'] / eval_stats_no_ood['total_post_ood_turns'] \ if eval_stats_no_ood['total_post_ood_turns'] != 0 \ else 0 print( 'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}' .format(accuracy, accuracy_after_ood, accuracy_post_ood)) print('Loss : {:.3f}'.format(eval_stats_no_ood['avg_loss']))
def main(in_dataset_folder, in_noisy_dataset_folder, in_custom_vocab_file, in_model_folder, in_config): with open(in_config, encoding='utf-8') as config_in: config = json.load(config_in) train_dialogs, train_indices = read_dialogs(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-trn.txt'), with_indices=True) dev_dialogs, dev_indices = read_dialogs(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-dev.txt'), with_indices=True) test_dialogs, test_indices = read_dialogs(os.path.join(in_noisy_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) kb = make_augmented_knowledge_base(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-kb.txt'), os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-candidates.txt')) max_noisy_dialog_length = max([item['end'] - item['start'] + 1 for item in test_indices]) config['max_input_length'] = max_noisy_dialog_length post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns(test_dialogs, BABI_CONFIG['backoff_utterance'].lower()) et = EntityTracker(kb) at = ActionTracker(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-candidates.txt'), et) if in_custom_vocab_file is not None: with open(in_custom_vocab_file) as vocab_in: rev_vocab = [line.rstrip() for line in vocab_in] vocab = {word: idx for idx, word in enumerate(rev_vocab)} else: utterances_tokenized = [] for dialog in train_dialogs: utterances_tokenized += list(map(lambda x: x.split(), dialog)) vocab, rev_vocab = make_vocabulary(utterances_tokenized, config['max_vocabulary_size'], special_tokens=[PAD, START, UNK, EOS] + list(kb.keys())) config['vocabulary_size'] = len(vocab) data_train = make_dataset_for_hierarchical_hcn(train_dialogs, train_indices, vocab, et, at, **config) data_dev = make_dataset_for_hierarchical_hcn(dev_dialogs, dev_indices, vocab, et, at, **config) data_test = make_dataset_for_hierarchical_hcn(test_dialogs, test_indices, vocab, et, at, **config) random_input = generate_dropout_turns_for_hierarchical_hcn(10000, config['max_sequence_length'], [utterance[0] for utterance in train_dialogs], vocab, config['turn_word_dropout_prob']) save_model(rev_vocab, config, kb, at.action_templates, in_model_folder) trainer = Trainer(data_train, data_dev, data_test, at.action_templates, random_input, post_ood_turns_noisy, config, vocab, in_model_folder) trainer.train()
def __init__(self): self.et = EntityTracker() self.at = ActionTracker(self.et) self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed() obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features self.action_templates = self.at.get_action_templates() action_size = self.at.action_size nb_hidden = 128 self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden) # restore checkpoint self.net.restore() self.net.reset_state()
def main(in_dataset_folder, in_custom_vocab_file, in_model_folder, in_config): with open(in_config, encoding='utf-8') as config_in: config = json.load(config_in) train_dialogs, train_indices = read_dialogs(os.path.join( in_dataset_folder, 'dialog-babi-task6-dstc2-trn.txt'), with_indices=True) dev_dialogs, dev_indices = read_dialogs(os.path.join( in_dataset_folder, 'dialog-babi-task6-dstc2-dev.txt'), with_indices=True) test_dialogs, test_indices = read_dialogs(os.path.join( in_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) kb = make_augmented_knowledge_base( os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-kb.txt'), os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-candidates.txt')) et = EntityTracker(kb) at = ActionTracker( os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-candidates.txt'), et) if in_custom_vocab_file is not None: with open(in_custom_vocab_file) as vocab_in: rev_vocab = [line.rstrip() for line in vocab_in] vocab = {word: idx for idx, word in enumerate(rev_vocab)} else: utterances_tokenized = [] for dialog in train_dialogs: utterances_tokenized += list(map(lambda x: x.split(), dialog)) vocab, rev_vocab = make_vocabulary( utterances_tokenized, config['max_vocabulary_size'], special_tokens=[PAD, START, UNK, EOS] + list(kb.keys())) config['vocabulary_size'] = len(vocab) data_train = make_dataset_for_variational_hcn(train_dialogs, train_indices, vocab, et, at, **config) data_dev = make_dataset_for_variational_hcn(dev_dialogs, dev_indices, vocab, et, at, **config) data_test = make_dataset_for_variational_hcn(test_dialogs, test_indices, vocab, et, at, **config) random_input = generate_random_input_for_variational_hcn( 10000, config['max_sequence_length'], vocab, rev_vocab) save_model(rev_vocab, config, kb, at.action_templates, in_model_folder) trainer = Trainer(data_train, data_dev, data_test, at.action_templates, random_input, rev_vocab, config, in_model_folder) trainer.train()
def dialog_train(self, dialog): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() loss = 0. # iterate through dialog for (u, r) in dialog: u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask action_mask = at.action_mask() # forward propagation # train step loss += self.net.train_step(features, r, action_mask) return loss/len(dialog)
def evaluate(self): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() dialog_accuracy = 0. r_count = 0 #Count of task 15 count = 0 # Total count of rewards for dialog_idx in self.dialog_indices_dev: start, end = dialog_idx['start'], dialog_idx['end'] dialog = self.dataset[start:end] num_dev_examples = len(self.dialog_indices_dev) # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # iterate through dialog correct_examples = 0 for (u,r) in dialog: # encode utterance u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features # get action mask features = torch.autograd.Variable(torch.from_numpy(np.concatenate((u_ent_features, u_emb, u_bow), axis=0))).float() # print(features) # get action mask action_mask = torch.autograd.Variable(torch.from_numpy(at.action_mask())) # r = torch.autograd.Variable(torch.LongTensor([r])) # forward propagation # train step logits,probs,prediction = self.net.forward(features, action_mask) # print("logits", logits) # print("probs", probs) # print("prediction", logits,probs,prediction) # print(prediction,r) correct_examples += int(prediction == r) if r==15: r_count += 1 count += 1 # get dialog accuracy dialog_accuracy += correct_examples/len(dialog) print("task 15 was the answer with freq",r_count/count) return dialog_accuracy/num_dev_examples
def interact(self): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # begin interaction loop while True: # get input from user u = input(':: ') # check if user wants to begin new session if u == 'clear' or u == 'reset' or u == 'restart': self.net.reset_state() et = EntityTracker() at = ActionTracker(et) print('') # check for exit command elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q': break else: # ENTER press : silence if not u: u = '<SILENCE>' # encode u_ent, u_entities = et.extract_entities(u, is_test=True) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask action_mask = at.action_mask() # forward prediction = self.net.forward(features, action_mask) if self.post_process(prediction, u_ent_features): print( '>>', 'api_call ' + u_entities['<cuisine>'] + ' ' + u_entities['<location>'] + ' ' + u_entities['<party_size>'] + ' ' + u_entities['<rest_type>']) else: prediction = self.action_post_process( prediction, u_entities) print('>>', self.action_templates[prediction])
def __init__(self): et = EntityTracker() self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed() at = ActionTracker(et) self.dataset, dialog_indices = Data(et, at).trainset self.dialog_indices_tr = dialog_indices[:200] self.dialog_indices_dev = dialog_indices[200:250] obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features self.action_templates = at.get_action_templates() action_size = at.action_size nb_hidden = 128 self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden)
def evaluate(self): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() dialog_accuracy = 0. correct_dialogue_count = 0 for dialog_idx in self.dialog_indices_dev: start, end = dialog_idx['start'], dialog_idx['end'] dialog = self.dataset[start:end] num_dev_examples = len(self.dialog_indices_dev) # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # iterate through dialog correct_examples = 0 for (u, r) in dialog: # encode utterance u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask action_mask = at.action_mask() # forward propagation # train step prediction = self.net.forward(features, action_mask) correct_examples += int(prediction == r) if correct_examples == len(dialog): correct_dialogue_count += 1 # get dialog accuracy dialog_accuracy += correct_examples / len(dialog) per_response_accuracy = dialog_accuracy / num_dev_examples per_dialogue_accuracy = correct_dialogue_count / num_dev_examples return per_response_accuracy, per_dialogue_accuracy
def __init__(self): et = EntityTracker() self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed() at = ActionTracker(et) ''' ['any preference on a type of cuisine', 'api_call <cuisine> <location> <party_size> <rest_type>', 'great let me do the reservation', 'hello what can i help you with today', 'here it is <info_address>', 'here it is <info_phone>', 'how many people would be in your party', "i'm on it", 'is there anything i can help you with', 'ok let me look into some options for you', 'sure is there anything else to update', 'sure let me find an other option for you', 'what do you think of this option: <restaurant>', 'where should it be', 'which price range are looking for', "you're welcome"] ''' self.dataset, dialog_indices = Data(et, at).trainset self.dialog_indices_tr = dialog_indices[:200] self.dialog_indices_dev = dialog_indices[200:250] obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features self.action_templates = at.get_action_templates() action_size = at.action_size nb_hidden = 128 self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden)
def __init__(self): et = EntityTracker() self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed() at = ActionTracker(et) self.dataset, dialog_indices = Data(et, at).trainset train_indices = joblib.load('data/train_test_list/train_indices_759') test_indices = joblib.load('data/train_test_list/test_indices_759_949') self.dialog_indices_tr = train_indices self.dialog_indices_dev = test_indices obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features self.action_templates = at.get_action_templates() action_size = at.action_size nb_hidden = 128 self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden)
def interact(self, input): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # begin interaction loop #while True: # get input from user #u = input(':: ') u = input # check if user wants to begin new session if u == 'clear' or u == 'reset' or u == 'restart': self.net.reset_state() et = EntityTracker() at = ActionTracker(et) print('') # check for exit command #elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q': #break else: # ENTER press : silence if not u: u = '<SILENCE>' # encode u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask action_mask = at.action_mask() # forward prediction = self.net.forward(features, action_mask) #print('>>', self.action_templates[prediction]) return self.action_templates[prediction]
def main(in_clean_dataset_folder, in_noisy_dataset_folder, in_model_folder, in_mode, in_runs_number): rev_vocab, kb, action_templates, config = load_model(in_model_folder) train_json = load_hcn_json( os.path.join(in_clean_dataset_folder, 'train.json')) test_json = load_hcn_json( os.path.join(in_clean_dataset_folder, 'test.json')) test_ood_json = load_hcn_json( os.path.join(in_noisy_dataset_folder, 'test_ood.json')) max_noisy_dialog_length = max( [len(dialog['turns']) for dialog in test_ood_json['dialogs']]) config['max_input_length'] = max_noisy_dialog_length post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns( test_ood_json) et = EntityTracker(kb) action_weights = defaultdict(lambda: 1.0) action_weights[0] = 0.0 action_weighting = np.vectorize(action_weights.__getitem__) vocab = {word: idx for idx, word in enumerate(rev_vocab)} ctx_features = [] for dialog in train_json['dialogs']: for utterance in dialog['turns']: if 'context_features' in utterance: ctx_features.append(utterance['context_features']) ctx_features_vocab, ctx_features_rev_vocab = make_vocabulary( ctx_features, config['max_vocabulary_size'], special_tokens=[]) data_preparation_function = getattr(utils.preprocessing, config['data_preparation_function']) data_clean = data_preparation_function(test_json, vocab, ctx_features_vocab, et, **config) data_noisy = data_preparation_function(test_ood_json, vocab, ctx_features_vocab, et, **config) data_clean_with_weights = *data_clean, action_weighting(data_clean[-1]) data_noisy_with_weights = *data_noisy, action_weighting(data_noisy[-1]) net = getattr(modules, config['model_name'])(vocab, config, len(ctx_features_vocab), len(action_templates)) net.restore(in_model_folder) if in_mode == 'clean': eval_stats_clean = evaluate_advanced( net, data_clean_with_weights, action_templates, BABI_CONFIG['backoff_utterance'], post_ood_turns=post_ood_turns_clean, runs_number=in_runs_number) print('Clean dataset: {} turns overall'.format( eval_stats_clean['total_turns'])) print('Accuracy:') accuracy = eval_stats_clean['correct_turns'] / eval_stats_clean[ 'total_turns'] accuracy_continuous = eval_stats_clean[ 'correct_continuous_turns'] / eval_stats_clean['total_turns'] accuracy_post_ood = eval_stats_clean['correct_post_ood_turns'] / eval_stats_clean['total_post_ood_turns'] \ if eval_stats_clean['total_post_ood_turns'] != 0 \ else 0 ood_f1 = eval_stats_clean['ood_f1'] print('overall acc: {:.3f}; continuous acc: {:.3f}; ' 'directly post-OOD acc: {:.3f}; OOD F1: {:.3f}'.format( accuracy, accuracy_continuous, accuracy_post_ood, ood_f1)) print('Loss : {:.3f}'.format(eval_stats_clean['avg_loss'])) elif in_mode == 'noisy': eval_stats_noisy = evaluate_advanced( net, data_noisy_with_weights, action_templates, BABI_CONFIG['backoff_utterance'], post_ood_turns=post_ood_turns_noisy, runs_number=in_runs_number) print('\n\n') print('Noisy dataset: {} turns overall'.format( eval_stats_noisy['total_turns'])) print('Accuracy:') accuracy = eval_stats_noisy['correct_turns'] / eval_stats_noisy[ 'total_turns'] accuracy_continuous = eval_stats_noisy[ 'correct_continuous_turns'] / eval_stats_noisy['total_turns'] accuracy_post_ood = eval_stats_noisy['correct_post_ood_turns'] / eval_stats_noisy['total_post_ood_turns'] \ if eval_stats_noisy['total_post_ood_turns'] != 0 \ else 0 accuracy_ood = eval_stats_noisy['correct_ood_turns'] / eval_stats_noisy['total_ood_turns'] \ if eval_stats_noisy['total_ood_turns'] != 0 \ else 0 ood_f1 = eval_stats_noisy['ood_f1'] print('overall acc: {:.3f}; continuous acc: {:.3f}; ' 'directly post-OOD acc: {:.3f}; OOD acc: {:.3f}; OOD F1: {:.3f}'. format(accuracy, accuracy_continuous, accuracy_post_ood, accuracy_ood, ood_f1)) print('Loss : {:.3f}'.format(eval_stats_noisy['avg_loss'])) elif in_mode == 'noisy_ignore_ood': eval_stats_no_ood = evaluate_advanced( net, data_noisy_with_weights, action_templates, BABI_CONFIG['backoff_utterance'], post_ood_turns=post_ood_turns_noisy, ignore_ood_accuracy=True, runs_number=in_runs_number) print('Accuracy (OOD turns ignored):') accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood[ 'total_turns'] accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \ if eval_stats_no_ood['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_no_ood['correct_post_ood_turns'] / eval_stats_no_ood['total_post_ood_turns'] \ if eval_stats_no_ood['total_post_ood_turns'] != 0 \ else 0 print('overall: {:.3f}; ' 'after first OOD: {:.3f}; ' 'directly post-OOD: {:.3f}'.format(accuracy, accuracy_after_ood, accuracy_post_ood))
def interact(self): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # begin interaction loop while True: # get input from user u = input('User: '******'clear' or u == 'reset' or u == 'restart': self.net.reset_state() et = EntityTracker() at = ActionTracker(et) print('Bot: Reset successfully') # check for entrance and exit command elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q': print("Bot: Thank you for using") break elif u == 'hello' or u == 'hi': print("Bot: Hello, what can i do for you") elif u == 'thank you' or u == 'thanks' or u == 'thank you very much': print('Bot: You are welcome') break else: if not u: continue u = u.lower() # encode u_ent = et.extract_entities(u) u_ent_features = et.context_features() # 5 # print(et.entities) # print(et.ctxt_features) u_emb = self.emb.encode(u) # 300 u_bow = self.bow_enc.encode(u) # 60 # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # print(features.shape) # get action mask action_mask = at.action_mask() # action_mask = np.ones(self.net.action_size) # print("action_mask: ", action_mask) # forward prediction = self.net.forward(features, action_mask) response = self.action_templates[prediction] if prediction == 0: slot_values = copy.deepcopy(et.entities) slot_values.pop('<location>') memory = [] count = 0 for k, v in slot_values.items(): memory.append('='.join([k, v])) count += 1 if count == 2: memory.append('\n') response = response.replace("memory", ', '.join(memory)) # memory = ', '.join(slot_values.values()) # response = response.replace("memory", memory) self.net.reset_state() et = EntityTracker() at = ActionTracker(et) # print('Execute successfully and begin new session') if prediction == 1: response = response.replace("location", '<location>=' + et.entities['<location>']) print('Bot: ', response)
def main(in_clean_dataset_folder, in_noisy_dataset_folder, in_model_folder, in_mode, in_runs_number): rev_vocab, kb, action_templates, config = load_model(in_model_folder) clean_dialogs, clean_indices = read_dialogs(os.path.join(in_clean_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) noisy_dialogs, noisy_indices = read_dialogs(os.path.join(in_noisy_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) max_noisy_dialog_length = max([item['end'] - item['start'] + 1 for item in noisy_indices]) config['max_input_length'] = max_noisy_dialog_length post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns(noisy_dialogs, BABI_CONFIG['backoff_utterance'].lower()) et = EntityTracker(kb) at = ActionTracker(None, et) at.set_action_templates(action_templates) vocab = {word: idx for idx, word in enumerate(rev_vocab)} data_clean = make_dataset_for_vhcn_v2(clean_dialogs, clean_indices, vocab, et, at, **config) data_noisy = make_dataset_for_vhcn_v2(noisy_dialogs, noisy_indices, vocab, et, at, **config) context_features_clean, action_masks_clean = data_clean[2:4] net = VariationalHierarchicalLSTMv3(rev_vocab, config, context_features_clean.shape[-1], action_masks_clean.shape[-1]) net.restore(in_model_folder) if in_mode == 'clean': eval_stats_clean = evaluate_advanced(net, data_clean, at.action_templates, BABI_CONFIG['backoff_utterance'].lower(), post_ood_turns=post_ood_turns_clean, runs_number=in_runs_number) print('Clean dataset: {} turns overall'.format(eval_stats_clean['total_turns'])) print('Accuracy:') accuracy = eval_stats_clean['correct_turns'] / eval_stats_clean['total_turns'] accuracy_continuous = eval_stats_clean['correct_continuous_turns'] / eval_stats_clean['total_turns'] accuracy_post_ood = eval_stats_clean['correct_post_ood_turns'] / eval_stats_clean['total_post_ood_turns'] \ if eval_stats_clean['total_post_ood_turns'] != 0 \ else 0 print('overall: {:.3f}; continuous: {:.3f}; directly post-OOD: {:.3f}'.format(accuracy, accuracy_continuous, accuracy_post_ood)) print('Loss : {:.3f}'.format(eval_stats_clean['avg_loss'])) elif in_mode == 'noisy': eval_stats_noisy = evaluate_advanced(net, data_noisy, at.action_templates, BABI_CONFIG['backoff_utterance'].lower(), post_ood_turns=post_ood_turns_noisy, runs_number=in_runs_number) print('\n\n') print('Noisy dataset: {} turns overall'.format(eval_stats_noisy['total_turns'])) print('Accuracy:') accuracy = eval_stats_noisy['correct_turns'] / eval_stats_noisy['total_turns'] accuracy_continuous = eval_stats_noisy['correct_continuous_turns'] / eval_stats_noisy['total_turns'] accuracy_post_ood = eval_stats_noisy['correct_post_ood_turns'] / eval_stats_noisy['total_post_ood_turns'] \ if eval_stats_noisy['total_post_ood_turns'] != 0 \ else 0 accuracy_ood = eval_stats_noisy['correct_ood_turns'] / eval_stats_noisy['total_ood_turns'] \ if eval_stats_noisy['total_ood_turns'] != 0 \ else 0 print('overall: {:.3f}; continuous: {:.3f}; directly post-OOD: {:.3f}; OOD: {:.3f}'.format(accuracy, accuracy_continuous, accuracy_post_ood, accuracy_ood)) print('Loss : {:.3f}'.format(eval_stats_noisy['avg_loss'])) elif in_mode == 'noisy_ignore_ood': eval_stats_no_ood = evaluate_advanced(net, data_noisy, at.action_templates, BABI_CONFIG['backoff_utterance'].lower(), post_ood_turns=post_ood_turns_noisy, ignore_ood_accuracy=True, runs_number=in_runs_number) print('Accuracy (OOD turns ignored):') accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood['total_turns'] accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \ if eval_stats_no_ood['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_no_ood['correct_post_ood_turns'] / eval_stats_no_ood['total_post_ood_turns'] \ if eval_stats_no_ood['total_post_ood_turns'] != 0 \ else 0 print('overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'.format(accuracy, accuracy_after_ood, accuracy_post_ood))
def evaluate(self, eval=False): ################################################################### if self.lang_type == 'eng': from modules.entities import EntityTracker from modules.data_utils import Data from modules.actions import ActionTracker from modules.bow import BoW_encoder elif self.lang_type == 'kor': from modules.entities_kor import EntityTracker from modules.data_utils_kor import Data from modules.actions_kor import ActionTracker from modules.bow_kor import BoW_encoder ################################################################### et = EntityTracker() at = ActionTracker(et) # only for evaluation purpose if eval: self.net.restore() # reset entities extractor turn_accuracy = 0. dialog_accuracy = 0. for dialog_idx in self.dialog_indices_dev: start, end = dialog_idx['start'], dialog_idx['end'] dialog = self.dataset[start:end] num_dev_examples = len(self.dialog_indices_dev) et = EntityTracker() at = ActionTracker(et) # reset network_type before evaluate. self.net.reset_state() correct_examples = 0 for (u, r) in dialog: u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_bow = self.bow_enc.encode(u) if self.is_emb: u_emb = self.emb.encode(u) if self.is_action_mask: action_mask = at.action_mask() # concatenated features if self.is_action_mask and self.is_emb: features = np.concatenate( (u_ent_features, u_emb, u_bow, action_mask), axis=0) elif self.is_action_mask and not (self.is_emb): features = np.concatenate( (u_ent_features, u_bow, action_mask), axis=0) elif not (self.is_action_mask) and self.is_emb: features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) elif not (self.is_action_mask) and not (self.is_emb): features = np.concatenate((u_ent_features, u_bow), axis=0) if self.is_action_mask: probs, prediction = self.net.forward(features, action_mask) else: probs, prediction = self.net.forward(features) correct_examples += int(prediction == r) turn_accuracy += correct_examples / len(dialog) accuracy = correct_examples / len(dialog) if (accuracy == 1.0): dialog_accuracy += 1 turn_accuracy = turn_accuracy / num_dev_examples dialog_accuracy = dialog_accuracy / num_dev_examples return turn_accuracy, dialog_accuracy
def main(in_dataset_folder, in_noisy_dataset_folder, in_custom_vocab_file, in_model_folder, in_config): with open(in_config, encoding='utf-8') as config_in: config = json.load(config_in) train_json = load_hcn_json(os.path.join(in_dataset_folder, 'train.json')) dev_json = load_hcn_json(os.path.join(in_dataset_folder, 'dev.json')) # test_json = load_hcn_json(os.path.join(in_dataset_folder, 'test.json')) test_ood_json = load_hcn_json( os.path.join(in_noisy_dataset_folder, 'test_ood.json')) kb = make_augmented_knowledge_base( os.path.join(BABI_FOLDER, 'dialog-babi-task6-dstc2-kb.txt'), os.path.join(BABI_FOLDER, 'dialog-babi-task6-dstc2-candidates.txt')) action_templates = train_json['actions'] max_noisy_dialog_length = max( [len(dialog['turns']) for dialog in test_ood_json['dialogs']]) config['max_input_length'] = max_noisy_dialog_length et = EntityTracker(kb) post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns( test_ood_json) if in_custom_vocab_file is not None: with open(in_custom_vocab_file) as vocab_in: rev_vocab = [line.rstrip() for line in vocab_in] vocab = {word: idx for idx, word in enumerate(rev_vocab)} else: utterances_tokenized = [] for dialog in train_json['dialogs']: for utterance in dialog['turns']: utterances_tokenized.append(utterance['input'].split()) vocab, rev_vocab = make_vocabulary( utterances_tokenized, config['max_vocabulary_size'], special_tokens=[PAD, START, UNK, EOS] + list(kb.keys())) ctx_features = [] for dialog in train_json['dialogs']: for utterance in dialog['turns']: if 'context_features' in utterance: ctx_features.append(utterance['context_features']) ctx_features_vocab, ctx_features_rev_vocab = make_vocabulary( ctx_features, config['max_vocabulary_size'], special_tokens=[]) config['vocabulary_size'] = len(vocab) print('Training with config: {}'.format(json.dumps(config))) data_preparation_function = getattr(utils.preprocessing, config['data_preparation_function']) data_train = data_preparation_function(train_json, vocab, ctx_features_vocab, et, **config) data_dev = data_preparation_function(dev_json, vocab, ctx_features_vocab, et, **config) # data_test = data_preparation_function(test_json, vocab, ctx_features_vocab, et, **config) data_test_ood = data_preparation_function(test_ood_json, vocab, ctx_features_vocab, et, **config) dropout_turn_generation_function = getattr( utils.preprocessing, config['dropout_turn_generation_function']) random_input = dropout_turn_generation_function( 10000, 3, config['max_sequence_length'], train_json, vocab, config['turn_word_dropout_prob']) save_model(rev_vocab, config, kb, action_templates, in_model_folder) net = getattr(modules, config['model_name'])(vocab, config, len(ctx_features_vocab), len(action_templates)) trainer = Trainer(data_train, data_dev, data_test_ood, action_templates, random_input, post_ood_turns_noisy, config, net, in_model_folder) trainer.train()
def __init__(self, args): self.response_accuracy = [] self.dialog_accuracy = [] try: ###################### selective import ############################# if args[0] == 'am': self.is_action_mask = True else: self.is_action_mask = False if args[1] == 'emb': self.is_emb = True else: self.is_emb = False self.network_type = args[2] self.lang_type = args[3] if self.lang_type == 'eng': from modules.entities import EntityTracker from modules.data_utils import Data from modules.actions import ActionTracker from modules.bow import BoW_encoder elif self.lang_type == 'kor': from modules.entities_kor import EntityTracker from modules.data_utils_kor import Data from modules.actions_kor import ActionTracker from modules.bow_kor import BoW_encoder ################################################################### except: rospy.logwarn( "please try again. i.e. ... train.py <am> <emb> <bidirectional_lstm> <eng>" ) if self.is_emb: if self.lang_type == 'eng': self.emb = UtteranceEmbed(lang=self.lang_type) elif self.lang_type == 'kor': self.emb = UtteranceEmbed(lang=self.lang_type) et = EntityTracker() self.bow_enc = BoW_encoder() at = ActionTracker(et) at.do_display_template() self.dataset, dialog_indices = Data(et, at).trainset # self.dialog_indices_tr = dialog_indices[:200] self.dialog_indices_tr = random.sample(dialog_indices, 200) # self.dialog_indices_dev = dialog_indices[200:250] self.dialog_indices_dev = random.sample(dialog_indices, 50) self.action_templates = at.get_action_templates() action_size = at.action_size nb_hidden = 128 # set feature input if self.is_action_mask and self.is_emb: obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features + at.action_size elif self.is_action_mask and not (self.is_emb): obs_size = self.bow_enc.vocab_size + et.num_features + at.action_size elif not (self.is_action_mask) and self.is_emb: obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features elif not (self.is_action_mask) and not (self.is_emb): obs_size = self.bow_enc.vocab_size + et.num_features # set network_type type if self.network_type == 'gru': self.net = GRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'reversed_lstm': self.net = ReversingLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'reversed_gru': self.net = ReversingGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'stacked_gru': self.net = StackedGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'stacked_lstm': self.net = StackedLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'lstm': self.net = LSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'bidirectional_lstm': self.net = BidirectionalLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'bidirectional_gru': self.net = BidirectionalGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) file_path = os.path.join(rospkg.RosPack().get_path('dialogue_system'), 'log', self.network_type) # init logging self.logger = self.get_logger(file_path) msg = "'\033[94m[%s trainer]\033[0m initialized - %s (action_mask: %s, embedding: %s, lang: %s, obs_size: %s, bow: %s, context_feature: %s, action_size: %s)" % ( rospy.get_name(), self.net.__class__.__name__, self.is_action_mask, self.is_emb, self.lang_type, obs_size, self.bow_enc.vocab_size, et.num_features, action_size) rospy.loginfo(msg)
class InteractiveSession: def __init__(self): self.et = EntityTracker() self.at = ActionTracker(self.et) self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed() obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features self.action_templates = self.at.get_action_templates() action_size = self.at.action_size nb_hidden = 128 self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden) # restore checkpoint self.net.restore() self.net.reset_state() def reset(self): self.net.reset_state() self.et = EntityTracker() self.at = ActionTracker(self.et) def interact(self, utterance): # get input from user u = utterance.lower() # check if user wants to begin new session if u == 'clear' or u == 'reset' or u == 'restart': self.reset() return "reset successfully" # check for entrance and exit command elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q': self.reset() return "Thank you for using" elif u == 'hello' or u == 'hi': self.reset() return "what can i do for you" elif u == 'thank you' or u == 'thanks' or u == 'thank you very much': self.reset() return 'you are welcome' else: # encode u_ent = self.et.extract_entities(u) u_ent_features = self.et.context_features() # 5 u_emb = self.emb.encode(u) # 300 u_bow = self.bow_enc.encode(u) # 60 # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask action_mask = self.at.action_mask() # action_mask = np.ones(self.net.action_size) # forward prediction = self.net.forward(features, action_mask) response = self.action_templates[prediction] if prediction == 0: slot_values = copy.deepcopy(self.et.entities) slot_values.pop('<location>') memory = ', '.join(slot_values.values()) response = response.replace("memory", memory) self.reset() print('API CALL execute successfully and begin new session') if prediction == 1: response = response.replace("location", self.et.entities['<location>']) return response
def main(in_clean_dataset_folder, in_noisy_dataset_folder, in_model_folder, in_no_ood_evaluation): rev_vocab, kb, action_templates, config = load_model(in_model_folder) clean_dialogs, clean_indices = read_dialogs(os.path.join( in_clean_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) noisy_dialogs, noisy_indices = read_dialogs(os.path.join( in_noisy_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns( noisy_dialogs) assert len(post_ood_turns_clean) == len(post_ood_turns_noisy) for post_ood_turn_clean, post_ood_turn_noisy in zip( sorted(post_ood_turns_clean), sorted(post_ood_turns_noisy)): noisy_dialogs[post_ood_turn_noisy][0] = clean_dialogs[ post_ood_turn_clean][0] et = EntityTracker(kb) at = ActionTracker(None, et) at.set_action_templates(action_templates) vocab = {word: idx for idx, word in enumerate(rev_vocab)} X_clean, context_features_clean, action_masks_clean, y_clean = make_dataset_for_hierarchical_lstm( clean_dialogs, clean_indices, vocab, et, at, **config) X_noisy, context_features_noisy, action_masks_noisy, y_noisy = make_dataset_for_hierarchical_lstm( noisy_dialogs, noisy_indices, vocab, et, at, **config) net = HierarchicalLSTM(config, context_features_clean.shape[-1], action_masks_clean.shape[-1]) net.restore(in_model_folder) eval_stats_clean = evaluate_advanced( net, (X_clean, context_features_clean, action_masks_clean, y_clean), at.action_templates, post_ood_turns=post_ood_turns_clean) print('Clean dataset: {} turns overall'.format( eval_stats_clean['total_turns'])) print('Accuracy:') accuracy = eval_stats_clean['correct_turns'] / eval_stats_clean[ 'total_turns'] accuracy_post_ood = eval_stats_clean['correct_post_ood_turns'] / eval_stats_clean['total_post_ood_turns'] \ if eval_stats_clean['total_post_ood_turns'] != 0 \ else 0 print('overall: {:.3f}; directly post-OOD: {:.3f}'.format( accuracy, accuracy_post_ood)) print('Loss : {:.3f}'.format(eval_stats_clean['avg_loss'])) eval_stats_noisy = evaluate_advanced( net, (X_noisy, context_features_noisy, action_masks_noisy, y_noisy), at.action_templates, post_ood_turns=post_ood_turns_noisy) print('\n\n') print( 'Noisy dataset: {} turns overall, {} turns after the first OOD'.format( eval_stats_noisy['total_turns'], eval_stats_noisy['total_turns_after_ood'])) print('Accuracy:') accuracy = eval_stats_noisy['correct_turns'] / eval_stats_noisy[ 'total_turns'] accuracy_after_ood = eval_stats_noisy['correct_turns_after_ood'] / eval_stats_noisy['total_turns_after_ood'] \ if eval_stats_noisy['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_noisy['correct_post_ood_turns'] / eval_stats_noisy['total_post_ood_turns'] \ if eval_stats_noisy['total_post_ood_turns'] != 0 \ else 0 print( 'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'. format(accuracy, accuracy_after_ood, accuracy_post_ood)) print('Loss : {:.3f}'.format(eval_stats_noisy['avg_loss'])) if in_no_ood_evaluation: eval_stats_no_ood = evaluate_advanced( net, (X_noisy, context_features_noisy, action_masks_noisy, y_noisy), at.action_templates, post_ood_turns=post_ood_turns_noisy, ignore_ood_accuracy=True) print('Accuracy (OOD turns ignored):') accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood[ 'total_turns'] accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \ if eval_stats_no_ood['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_no_ood['correct_post_ood_turns'] / eval_stats_no_ood['total_post_ood_turns'] \ if eval_stats_no_ood['total_post_ood_turns'] != 0 \ else 0 print( 'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}' .format(accuracy, accuracy_after_ood, accuracy_post_ood))
def reset(self): self.net.reset_state() self.et = EntityTracker() self.at = ActionTracker(self.et)
def __init__(self): # stor whole dialogues self.story = [] self.sp_confidecne = [] self.file_path = os.path.join( rospkg.RosPack().get_path('dialogue_system'), 'log', 'dialogue.txt') # count turn taking self.usr_count = 0 self.sys_count = 0 # paramaters self.network_type = rospy.get_param('~network_model', 'stacked_lstm') self.lang_type = rospy.get_param('~lang', 'eng') self.is_emb = rospy.get_param('~embedding', 'false') self.is_am = rospy.get_param('~action_mask', "true") self.user_num = rospy.get_param('~user_number', '0') # call rest of modules self.et = EntityTracker() self.at = ActionTracker(self.et) self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed(lang=self.lang_type) # select observation size for RNN if self.is_am and self.is_emb: obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features + self.at.action_size elif self.is_am and not (self.is_emb): obs_size = self.bow_enc.vocab_size + self.et.num_features + self.at.action_size elif not (self.is_am) and self.is_emb: obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features elif not (self.is_am) and not (self.is_emb): obs_size = self.bow_enc.vocab_size + self.et.num_features self.action_template = self.at.get_action_templates() self.at.do_display_template() # must clear entities space self.et.do_clear_entities() action_size = self.at.action_size nb_hidden = 128 if self.network_type == 'gru': self.net = GRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'reversed_lstm': self.net = ReversingLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'reversed_gru': self.net = ReversingGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'stacked_gru': self.net = StackedGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'stacked_lstm': self.net = StackedLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'lstm': self.net = LSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'bidirectional_lstm': self.net = BidirectionalLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'bidirectional_gru': self.net = BidirectionalGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) # restore trained model self.net.restore() # rostopics self.pub_reply = rospy.Publisher('reply', Reply, queue_size=10) self.pub_complete = rospy.Publisher('complete_execute_scenario', Empty, queue_size=10) rospy.Subscriber('raising_events', RaisingEvents, self.handle_raise_events) try: rospy.wait_for_service('reception_db/query_data') self.get_response_db = rospy.ServiceProxy( 'reception_db/query_data', DBQuery) rospy.logwarn("waiting for reception DB module...") except rospy.exceptions.ROSInterruptException as e: rospy.logerr(e) quit() rospy.logwarn( "network: {}, lang: {}, action_mask: {}, embedding: {}, user_number: {}" .format(self.network_type, self.lang_type, self.is_am, self.is_emb, self.user_num)) self.story.append('user number: %s' % self.user_num) rospy.loginfo('\033[94m[%s]\033[0m initialized.' % rospy.get_name())
class Dialogue(): def __init__(self): # stor whole dialogues self.story = [] self.sp_confidecne = [] self.file_path = os.path.join( rospkg.RosPack().get_path('dialogue_system'), 'log', 'dialogue.txt') # count turn taking self.usr_count = 0 self.sys_count = 0 # paramaters self.network_type = rospy.get_param('~network_model', 'stacked_lstm') self.lang_type = rospy.get_param('~lang', 'eng') self.is_emb = rospy.get_param('~embedding', 'false') self.is_am = rospy.get_param('~action_mask', "true") self.user_num = rospy.get_param('~user_number', '0') # call rest of modules self.et = EntityTracker() self.at = ActionTracker(self.et) self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed(lang=self.lang_type) # select observation size for RNN if self.is_am and self.is_emb: obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features + self.at.action_size elif self.is_am and not (self.is_emb): obs_size = self.bow_enc.vocab_size + self.et.num_features + self.at.action_size elif not (self.is_am) and self.is_emb: obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features elif not (self.is_am) and not (self.is_emb): obs_size = self.bow_enc.vocab_size + self.et.num_features self.action_template = self.at.get_action_templates() self.at.do_display_template() # must clear entities space self.et.do_clear_entities() action_size = self.at.action_size nb_hidden = 128 if self.network_type == 'gru': self.net = GRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'reversed_lstm': self.net = ReversingLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'reversed_gru': self.net = ReversingGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'stacked_gru': self.net = StackedGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'stacked_lstm': self.net = StackedLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'lstm': self.net = LSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'bidirectional_lstm': self.net = BidirectionalLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'bidirectional_gru': self.net = BidirectionalGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) # restore trained model self.net.restore() # rostopics self.pub_reply = rospy.Publisher('reply', Reply, queue_size=10) self.pub_complete = rospy.Publisher('complete_execute_scenario', Empty, queue_size=10) rospy.Subscriber('raising_events', RaisingEvents, self.handle_raise_events) try: rospy.wait_for_service('reception_db/query_data') self.get_response_db = rospy.ServiceProxy( 'reception_db/query_data', DBQuery) rospy.logwarn("waiting for reception DB module...") except rospy.exceptions.ROSInterruptException as e: rospy.logerr(e) quit() rospy.logwarn( "network: {}, lang: {}, action_mask: {}, embedding: {}, user_number: {}" .format(self.network_type, self.lang_type, self.is_am, self.is_emb, self.user_num)) self.story.append('user number: %s' % self.user_num) rospy.loginfo('\033[94m[%s]\033[0m initialized.' % rospy.get_name()) # if utterance == 'clear': # self.net.reset_state() # self.et.do_clear_entities() # response = 'context has been cleared.' def get_response(self, utterance): rospy.loginfo("actual input: %s" % utterance) # check actual user input # clean utterance # utterance = re.sub(r'[^ a-z A-Z 0-9]', " ", utterance) # utterance preprocessing u_ent, u_entities = self.et.extract_entities(utterance, is_test=True) u_ent_features = self.et.context_features() u_bow = self.bow_enc.encode(utterance) if self.is_emb: u_emb = self.emb.encode(utterance) try: if self.is_am: action_mask = self.at.action_mask() # concatenated features if self.is_am and self.is_emb: features = np.concatenate( (u_ent_features, u_emb, u_bow, action_mask), axis=0) elif self.is_am and not (self.is_emb): features = np.concatenate((u_ent_features, u_bow, action_mask), axis=0) elif not (self.is_am) and self.is_emb: features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) elif not (self.is_am) and not (self.is_emb): features = np.concatenate((u_ent_features, u_bow), axis=0) # try: # predict template number if self.is_am: probs, prediction = self.net.forward(features, action_mask) else: probs, prediction = self.net.forward(features) # check response confidence if max(probs) > BOUNDARY_CONFIDENCE: response = self.action_template[prediction] prediction = self.pre_action_process(prediction, u_entities) # handle api call if self.post_process(prediction, u_entities): if prediction == 1: response = 'api_call appointment {} {} {} {} {} {} {}'.format( u_entities['<first_name>'], u_entities['<last_name>'], u_entities['<address_number>'], u_entities['<address_name>'], u_entities['<address_type>'], u_entities['<time>'], u_entities['<pm_am>']) elif prediction == 2: response = 'api_call location {}'.format( u_entities['<location>']) elif prediction == 3: response = 'api_call prescription {} {} {} {} {}'.format( u_entities['<first_name>'], u_entities['<last_name>'], u_entities['<address_number>'], u_entities['<address_name>'], u_entities['<address_type>']) elif prediction == 4: response = 'api_call waiting_time {} {} {} {} {} {} {}'.format( u_entities['<first_name>'], u_entities['<last_name>'], u_entities['<address_number>'], u_entities['<address_name>'], u_entities['<address_type>'], u_entities['<time>'], u_entities['<pm_am>']) response = self.get_response_db( response ) # query knowledge base; here we use dynamo db response = response.response elif prediction in [6, 9, 11]: response = self.action_template[prediction] response = response.split(' ') response = [ word.replace('<first_name>', u_entities['<first_name>']) for word in response ] response = ' '.join(response) else: response = self.action_template[prediction] else: response = random.choice( REPROMPT ) # if prediction confidence less than 40%, reprompt except: response = random.choice(REPROMPT) return prediction, probs, response def handle_raise_events(self, msg): utterance = msg.recognized_word try: # get confidence data = json.loads(msg.data[0]) confidence = data['confidence'] except: confidence = None if confidence > BOUNDARY_CONFIDENCE or confidence == None: if 'silency_detected' in msg.events: utterance = '<SILENCE>' else: try: self.story.append( "U%i: %s (sp_conf:%f)" % (self.usr_count + 1, utterance, confidence)) self.sp_confidecne.append(confidence) except: self.story.append("U%i: %s" % (self.usr_count + 1, utterance)) self.usr_count += 1 utterance = utterance.lower() # generate system response prediction, probs, response = self.get_response(utterance) else: prediction = -1 probs = -1 response = random.choice(REPROMPT) # add system turn self.story.append("A%i: %s" % (self.sys_count + 1, response)) self.sys_count += 1 # finish interaction if (prediction == 6): self.pub_complete.publish() # logging user and system turn self.story.append("user: %i, system: %i" % (self.usr_count, self.sys_count)) self.story.append("mean_sp_conf: %f" % (reduce(lambda x, y: x + y, self.sp_confidecne) / len(self.sp_confidecne))) self.story.append( '===================================================================' ) self.write_file(self.file_path, self.story) # display system response rospy.loginfo(json.dumps(self.et.entities, indent=2)) # recognized entity values try: rospy.logwarn("System: [conf: %f, predict: %d] / %s\n" % (max(probs), prediction, response)) except: rospy.logwarn("System: [] / %s\n" % (response)) reply_msg = Reply() reply_msg.header.stamp = rospy.Time.now() reply_msg.reply = response self.pub_reply.publish(reply_msg) def post_process(self, prediction, u_ent_features): api_call_list = [1, 2, 3, 4] if prediction in api_call_list: return True attr_list = [0, 9, 10, 11, 12] if all(u_ent_featur == 1 for u_ent_featur in u_ent_features) and prediction in attr_list: return True else: return False def action_post_process(self, prediction, u_entities): attr_mapping_dict = { 11: '<first_name>', 11: '<last_name>', 12: '<address_number>', 12: '<address_name>', 12: '<address_type>', 10: '<time>', 10: '<pm_am>', } # find exist and non-exist entity exist_ent_index = [ key for key, value in u_entities.items() if value != None ] non_exist_ent_index = [ key for key, value in u_entities.items() if value == None ] # if predicted key is already in exist entity index then find non exist entity index # and leads the user to input non exist entity. if prediction in attr_mapping_dict: pred_key = attr_mapping_dict[prediction] if pred_key in exist_ent_index: for key, value in attr_mapping_dict.items(): if value == non_exist_ent_index[0]: return key else: return prediction else: return prediction def pre_action_process(self, prediction, u_entities): api_call_list = [1, 3, 4] attr_mapping_dict = { '<first_name>': 11, '<last_name>': 11, '<address_number>': 12, '<address_name>': 12, '<address_type>': 12, '<time>': 10, '<pm_am>': 10, } # find exist and non-exist entity non_exist_ent_index = [ key for key, value in u_entities.items() if value == None ] if prediction in api_call_list: if '<first_name>' in non_exist_ent_index: prediction = attr_mapping_dict['<first_name>'] return prediction ''' writing story log file ''' def write_file(self, path, story_list): with open(path, 'a') as f: for item in story_list: f.write("%s\n" % item) rospy.logwarn('save dialogue histories.')