def evaluate(self): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() dialog_accuracy = 0. r_count = 0 #Count of task 15 count = 0 # Total count of rewards for dialog_idx in self.dialog_indices_dev: start, end = dialog_idx['start'], dialog_idx['end'] dialog = self.dataset[start:end] num_dev_examples = len(self.dialog_indices_dev) # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # iterate through dialog correct_examples = 0 for (u,r) in dialog: # encode utterance u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features # get action mask features = torch.autograd.Variable(torch.from_numpy(np.concatenate((u_ent_features, u_emb, u_bow), axis=0))).float() # print(features) # get action mask action_mask = torch.autograd.Variable(torch.from_numpy(at.action_mask())) # r = torch.autograd.Variable(torch.LongTensor([r])) # forward propagation # train step logits,probs,prediction = self.net.forward(features, action_mask) # print("logits", logits) # print("probs", probs) # print("prediction", logits,probs,prediction) # print(prediction,r) correct_examples += int(prediction == r) if r==15: r_count += 1 count += 1 # get dialog accuracy dialog_accuracy += correct_examples/len(dialog) print("task 15 was the answer with freq",r_count/count) return dialog_accuracy/num_dev_examples
def interact(self): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # begin interaction loop while True: # get input from user u = input(':: ') # check if user wants to begin new session if u == 'clear' or u == 'reset' or u == 'restart': self.net.reset_state() et = EntityTracker() at = ActionTracker(et) print('') # check for exit command elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q': break else: # ENTER press : silence if not u: u = '<SILENCE>' # encode u_ent, u_entities = et.extract_entities(u, is_test=True) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask action_mask = at.action_mask() # forward prediction = self.net.forward(features, action_mask) if self.post_process(prediction, u_ent_features): print( '>>', 'api_call ' + u_entities['<cuisine>'] + ' ' + u_entities['<location>'] + ' ' + u_entities['<party_size>'] + ' ' + u_entities['<rest_type>']) else: prediction = self.action_post_process( prediction, u_entities) print('>>', self.action_templates[prediction])
def evaluate(self): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() dialog_accuracy = 0. correct_dialogue_count = 0 for dialog_idx in self.dialog_indices_dev: start, end = dialog_idx['start'], dialog_idx['end'] dialog = self.dataset[start:end] num_dev_examples = len(self.dialog_indices_dev) # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # iterate through dialog correct_examples = 0 for (u, r) in dialog: # encode utterance u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask action_mask = at.action_mask() # forward propagation # train step prediction = self.net.forward(features, action_mask) correct_examples += int(prediction == r) if correct_examples == len(dialog): correct_dialogue_count += 1 # get dialog accuracy dialog_accuracy += correct_examples / len(dialog) per_response_accuracy = dialog_accuracy / num_dev_examples per_dialogue_accuracy = correct_dialogue_count / num_dev_examples return per_response_accuracy, per_dialogue_accuracy
def dialog_train(self, dialog): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() loss = 0. # iterate through dialog for (u,r) in dialog: # print("Here in the dialog loop") u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = torch.autograd.Variable(torch.from_numpy(np.concatenate((u_ent_features, u_emb, u_bow), axis=0))).float() # print(features) # get action mask action_mask = torch.autograd.Variable(torch.from_numpy(at.action_mask())) r = torch.autograd.Variable(torch.LongTensor([r])) # print(r) # forward propagation # train step loss += self.net.train_step(features, r, action_mask) return loss/len(dialog)
def __init__(self): import os #实体追踪 et = EntityTracker() #词袋 word2vec self.bow_enc = BoW_encoder() #加载word2vec embedding self.emb = UtteranceEmbed() #将实体追踪器添加到动作追踪器中 at = ActionTracker(et) #得到数据集和对话开始 结束行数 self.dataset, dialog_indices = Data(et, at).trainset #划分数据集:200对做训练 50对做测试 self.dialog_indices_tr = dialog_indices self.dialog_indices_dev = dialog_indices #obs_size 300维的词向量 + 85个袋中的词 + 4个槽位 obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features #话术模板 self.action_templates = at.get_action_templates() #动作个数 action_size = at.action_size #隐藏层神经元个数 nb_hidden = 128 self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden)
def dialog_train(self, dialog): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() loss = 0. # iterate through dialog #u 用户输入 r 对应的动作索引 for (u, r) in dialog: #u_ent 分词后的字符串 u_ent = et.extract_entities(u) #槽位填充情况 【0 0 0 0】 u_ent_features = et.context_features() #word2vec u_emb = self.emb.encode(u) #multi-hot u_bow = self.bow_enc.encode(u) # concat features #300 + 85 + 4 = 389 features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask action_mask = at.action_mask() # forward propagation # train step loss += self.net.train_step(features, r, action_mask) return loss / len(dialog)
def interact(self): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # begin interaction loop while True: # get input from user u = input(':: ') # check if user wants to begin new session if u == 'clear' or u == 'reset' or u == 'restart': self.net.reset_state() et = EntityTracker() at = ActionTracker(et) print('') # check for exit command elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q': break else: # ENTER press : silence if not u: u = '<SILENCE>' # encode u_ent = et.extract_entities(u) print("get u ent " + u_ent) u_ent_features = et.context_features() print("槽位情况" + str(u_ent_features)) u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask action_mask = at.action_mask() # forward prediction = self.net.forward(features, action_mask) print("prediction " + str(prediction)) print('>>', self.action_templates[prediction])
def evaluate(self): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() dialog_accuracy = 0. #加载测试集 for dialog_idx in self.dialog_indices_dev: start, end = dialog_idx['start'], dialog_idx['end'] dialog = self.dataset[start:end] num_dev_examples = len(self.dialog_indices_dev) # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # iterate through dialog correct_examples = 0 #对于每个dialog 提取出utterance 和 response for (u, r) in dialog: # encode utterance #提取出user中带有的实体 u_ent = et.extract_entities(u) #提取当前槽位填充情况 u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask 16维的multi-hot 向量 action_mask = at.action_mask() # forward propagation # train step prediction = self.net.forward(features, action_mask) correct_examples += int(prediction == r) # get dialog accuracy dialog_accuracy += correct_examples / len(dialog) return dialog_accuracy / num_dev_examples
def dialog_train(self, dialog): ################################################################### if self.lang_type == 'eng': from modules.entities import EntityTracker from modules.data_utils import Data from modules.actions import ActionTracker from modules.bow import BoW_encoder elif self.lang_type == 'kor': from modules.entities_kor import EntityTracker from modules.data_utils_kor import Data from modules.actions_kor import ActionTracker from modules.bow_kor import BoW_encoder ################################################################### et = EntityTracker() at = ActionTracker(et) # reset state in network_type self.net.reset_state() loss = 0. for (u, r) in dialog: u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_bow = self.bow_enc.encode(u) if self.is_emb: u_emb = self.emb.encode(u) if self.is_action_mask: action_mask = at.action_mask() # print(u, r) # print(u_ent_features) # print('================================') # print(u_emb) # print('================================') # print(u_bow) # print('================================') # print(action_mask) # concatenated features if self.is_action_mask and self.is_emb: features = np.concatenate( (u_ent_features, u_emb, u_bow, action_mask), axis=0) elif self.is_action_mask and not (self.is_emb): features = np.concatenate((u_ent_features, u_bow, action_mask), axis=0) elif not (self.is_action_mask) and self.is_emb: features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) elif not (self.is_action_mask) and not (self.is_emb): features = np.concatenate((u_ent_features, u_bow), axis=0) # forward propagation with cumulative loss if self.is_action_mask: loss += self.net.train_step(features, r, action_mask) else: loss += self.net.train_step(features, r) return loss / len(dialog)
def main(in_dataset_folder, in_model_folder, in_no_ood_evaluation): rev_vocab, kb, action_templates, config = load_model(in_model_folder) test_dialogs, test_indices = read_dialogs(os.path.join( in_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) et = EntityTracker(kb) at = ActionTracker(None, et) at.set_action_templates(action_templates) vocab = {word: idx for idx, word in enumerate(rev_vocab)} X, context_features, action_masks, y = make_dataset_for_hierarchical_lstm( test_dialogs, test_indices, vocab, et, at, **config) net = HierarchicalLSTM(config, context_features.shape[-1], action_masks.shape[-1]) net.restore(in_model_folder) eval_stats_full_dataset = evaluate_advanced( net, (X, context_features, action_masks, y), test_dialogs, at.action_templates) print( 'Full dataset: {} turns overall, {} turns after the first OOD'.format( eval_stats_full_dataset['total_turns'], eval_stats_full_dataset['total_turns_after_ood'])) print('Accuracy:') accuracy = eval_stats_full_dataset[ 'correct_turns'] / eval_stats_full_dataset['total_turns'] accuracy_after_ood = eval_stats_full_dataset['correct_turns_after_ood'] / eval_stats_full_dataset['total_turns_after_ood'] \ if eval_stats_full_dataset['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_full_dataset['correct_post_ood_turns'] / eval_stats_full_dataset['total_post_ood_turns'] \ if eval_stats_full_dataset['total_post_ood_turns'] != 0 \ else 0 print( 'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'. format(accuracy, accuracy_after_ood, accuracy_post_ood)) print('Loss : {:.3f}'.format(eval_stats_full_dataset['avg_loss'])) if in_no_ood_evaluation: eval_stats_no_ood = evaluate_advanced( net, (X, context_features, action_masks, y), test_indices, at.action_templates, ignore_ood_accuracy=True) print('Accuracy (OOD turns ignored):') accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood[ 'total_turns'] accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \ if eval_stats_no_ood['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_no_ood['correct_post_ood_turns'] / eval_stats_no_ood['total_post_ood_turns'] \ if eval_stats_no_ood['total_post_ood_turns'] != 0 \ else 0 print( 'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}' .format(accuracy, accuracy_after_ood, accuracy_post_ood)) print('Loss : {:.3f}'.format(eval_stats_no_ood['avg_loss']))
def main(in_dataset_folder, in_noisy_dataset_folder, in_custom_vocab_file, in_model_folder, in_config): with open(in_config, encoding='utf-8') as config_in: config = json.load(config_in) train_dialogs, train_indices = read_dialogs(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-trn.txt'), with_indices=True) dev_dialogs, dev_indices = read_dialogs(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-dev.txt'), with_indices=True) test_dialogs, test_indices = read_dialogs(os.path.join(in_noisy_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) kb = make_augmented_knowledge_base(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-kb.txt'), os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-candidates.txt')) max_noisy_dialog_length = max([item['end'] - item['start'] + 1 for item in test_indices]) config['max_input_length'] = max_noisy_dialog_length post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns(test_dialogs, BABI_CONFIG['backoff_utterance'].lower()) et = EntityTracker(kb) at = ActionTracker(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-candidates.txt'), et) if in_custom_vocab_file is not None: with open(in_custom_vocab_file) as vocab_in: rev_vocab = [line.rstrip() for line in vocab_in] vocab = {word: idx for idx, word in enumerate(rev_vocab)} else: utterances_tokenized = [] for dialog in train_dialogs: utterances_tokenized += list(map(lambda x: x.split(), dialog)) vocab, rev_vocab = make_vocabulary(utterances_tokenized, config['max_vocabulary_size'], special_tokens=[PAD, START, UNK, EOS] + list(kb.keys())) config['vocabulary_size'] = len(vocab) data_train = make_dataset_for_hierarchical_hcn(train_dialogs, train_indices, vocab, et, at, **config) data_dev = make_dataset_for_hierarchical_hcn(dev_dialogs, dev_indices, vocab, et, at, **config) data_test = make_dataset_for_hierarchical_hcn(test_dialogs, test_indices, vocab, et, at, **config) random_input = generate_dropout_turns_for_hierarchical_hcn(10000, config['max_sequence_length'], [utterance[0] for utterance in train_dialogs], vocab, config['turn_word_dropout_prob']) save_model(rev_vocab, config, kb, at.action_templates, in_model_folder) trainer = Trainer(data_train, data_dev, data_test, at.action_templates, random_input, post_ood_turns_noisy, config, vocab, in_model_folder) trainer.train()
def main(in_dataset_folder, in_custom_vocab_file, in_model_folder, in_config): with open(in_config, encoding='utf-8') as config_in: config = json.load(config_in) train_dialogs, train_indices = read_dialogs(os.path.join( in_dataset_folder, 'dialog-babi-task6-dstc2-trn.txt'), with_indices=True) dev_dialogs, dev_indices = read_dialogs(os.path.join( in_dataset_folder, 'dialog-babi-task6-dstc2-dev.txt'), with_indices=True) test_dialogs, test_indices = read_dialogs(os.path.join( in_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) kb = make_augmented_knowledge_base( os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-kb.txt'), os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-candidates.txt')) et = EntityTracker(kb) at = ActionTracker( os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-candidates.txt'), et) if in_custom_vocab_file is not None: with open(in_custom_vocab_file) as vocab_in: rev_vocab = [line.rstrip() for line in vocab_in] vocab = {word: idx for idx, word in enumerate(rev_vocab)} else: utterances_tokenized = [] for dialog in train_dialogs: utterances_tokenized += list(map(lambda x: x.split(), dialog)) vocab, rev_vocab = make_vocabulary( utterances_tokenized, config['max_vocabulary_size'], special_tokens=[PAD, START, UNK, EOS] + list(kb.keys())) config['vocabulary_size'] = len(vocab) data_train = make_dataset_for_variational_hcn(train_dialogs, train_indices, vocab, et, at, **config) data_dev = make_dataset_for_variational_hcn(dev_dialogs, dev_indices, vocab, et, at, **config) data_test = make_dataset_for_variational_hcn(test_dialogs, test_indices, vocab, et, at, **config) random_input = generate_random_input_for_variational_hcn( 10000, config['max_sequence_length'], vocab, rev_vocab) save_model(rev_vocab, config, kb, at.action_templates, in_model_folder) trainer = Trainer(data_train, data_dev, data_test, at.action_templates, random_input, rev_vocab, config, in_model_folder) trainer.train()
def __init__(self): et = EntityTracker() self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed() at = ActionTracker(et) self.train_dataset, train_dialog_indices = Data(et, at).train_set self.test_dataset, test_dialog_indices = Data(et, at).test_set print('=========================\n') print('length of Train dialog indices : ', len(train_dialog_indices)) print('=========================\n') print('=========================\n') print('length of Test dialog indices : ', len(test_dialog_indices)) print('=========================\n') # Shuffle Training Dataset random.shuffle(train_dialog_indices) self.dialog_indices_tr = train_dialog_indices self.dialog_indices_dev = test_dialog_indices obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features self.action_templates = at.get_action_templates() action_size = at.action_size # nb_hidden = 128 nb_hidden = 150 print('=========================\n') print('Action_templates: ', action_size) print('=========================\n') self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden) self.et = et self.at = at action_projection = [] for action in self.action_templates: action_projection.append(self.emb.encode(action)) self.action_projection = np.transpose(action_projection) self.action_size = action_size
def __init__(self): et = EntityTracker() self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed() at = ActionTracker(et) obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features self.action_templates = at.get_action_templates() action_size = at.action_size nb_hidden = 128 self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden) # restore checkpoint self.net.restore()
def __init__(self): et = EntityTracker() self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed() at = ActionTracker(et) self.dataset, dialog_indices = Data(et, at).trainset self.dialog_indices_tr = dialog_indices[:200] self.dialog_indices_dev = dialog_indices[200:250] obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features self.action_templates = at.get_action_templates() action_size = at.action_size nb_hidden = 128 self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden)
def __init__(self): et = EntityTracker() self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed() at = ActionTracker(et) ''' ['any preference on a type of cuisine', 'api_call <cuisine> <location> <party_size> <rest_type>', 'great let me do the reservation', 'hello what can i help you with today', 'here it is <info_address>', 'here it is <info_phone>', 'how many people would be in your party', "i'm on it", 'is there anything i can help you with', 'ok let me look into some options for you', 'sure is there anything else to update', 'sure let me find an other option for you', 'what do you think of this option: <restaurant>', 'where should it be', 'which price range are looking for', "you're welcome"] ''' self.dataset, dialog_indices = Data(et, at).trainset self.dialog_indices_tr = dialog_indices[:200] self.dialog_indices_dev = dialog_indices[200:250] obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features self.action_templates = at.get_action_templates() action_size = at.action_size nb_hidden = 128 self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden)
def dialog_train(self, dialog): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() loss = 0. # iterate through dialog for (u, r) in dialog: u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_emb = self.emb.encode(u) u_bow = self.bow_enc.encode(u) # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # get action mask action_mask = at.action_mask() # forward propagation # train step loss += self.net.train_step(features, r, action_mask) return loss/len(dialog)
def __init__(self): et = EntityTracker() self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed() at = ActionTracker(et) self.dataset, dialog_indices = Data(et, at).trainset train_indices = joblib.load('data/train_test_list/train_indices_759') test_indices = joblib.load('data/train_test_list/test_indices_759_949') self.dialog_indices_tr = train_indices self.dialog_indices_dev = test_indices obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features self.action_templates = at.get_action_templates() action_size = at.action_size nb_hidden = 128 self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden)
def interact(self): # create entity tracker et = EntityTracker() # create action tracker at = ActionTracker(et) # reset network self.net.reset_state() # begin interaction loop while True: # get input from user u = input('User: '******'clear' or u == 'reset' or u == 'restart': self.net.reset_state() et = EntityTracker() at = ActionTracker(et) print('Bot: Reset successfully') # check for entrance and exit command elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q': print("Bot: Thank you for using") break elif u == 'hello' or u == 'hi': print("Bot: Hello, what can i do for you") elif u == 'thank you' or u == 'thanks' or u == 'thank you very much': print('Bot: You are welcome') break else: if not u: continue u = u.lower() # encode u_ent = et.extract_entities(u) u_ent_features = et.context_features() # 5 # print(et.entities) # print(et.ctxt_features) u_emb = self.emb.encode(u) # 300 u_bow = self.bow_enc.encode(u) # 60 # concat features features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) # print(features.shape) # get action mask action_mask = at.action_mask() # action_mask = np.ones(self.net.action_size) # print("action_mask: ", action_mask) # forward prediction = self.net.forward(features, action_mask) response = self.action_templates[prediction] if prediction == 0: slot_values = copy.deepcopy(et.entities) slot_values.pop('<location>') memory = [] count = 0 for k, v in slot_values.items(): memory.append('='.join([k, v])) count += 1 if count == 2: memory.append('\n') response = response.replace("memory", ', '.join(memory)) # memory = ', '.join(slot_values.values()) # response = response.replace("memory", memory) self.net.reset_state() et = EntityTracker() at = ActionTracker(et) # print('Execute successfully and begin new session') if prediction == 1: response = response.replace("location", '<location>=' + et.entities['<location>']) print('Bot: ', response)
def evaluate(self, eval=False): ################################################################### if self.lang_type == 'eng': from modules.entities import EntityTracker from modules.data_utils import Data from modules.actions import ActionTracker from modules.bow import BoW_encoder elif self.lang_type == 'kor': from modules.entities_kor import EntityTracker from modules.data_utils_kor import Data from modules.actions_kor import ActionTracker from modules.bow_kor import BoW_encoder ################################################################### et = EntityTracker() at = ActionTracker(et) # only for evaluation purpose if eval: self.net.restore() # reset entities extractor turn_accuracy = 0. dialog_accuracy = 0. for dialog_idx in self.dialog_indices_dev: start, end = dialog_idx['start'], dialog_idx['end'] dialog = self.dataset[start:end] num_dev_examples = len(self.dialog_indices_dev) et = EntityTracker() at = ActionTracker(et) # reset network_type before evaluate. self.net.reset_state() correct_examples = 0 for (u, r) in dialog: u_ent = et.extract_entities(u) u_ent_features = et.context_features() u_bow = self.bow_enc.encode(u) if self.is_emb: u_emb = self.emb.encode(u) if self.is_action_mask: action_mask = at.action_mask() # concatenated features if self.is_action_mask and self.is_emb: features = np.concatenate( (u_ent_features, u_emb, u_bow, action_mask), axis=0) elif self.is_action_mask and not (self.is_emb): features = np.concatenate( (u_ent_features, u_bow, action_mask), axis=0) elif not (self.is_action_mask) and self.is_emb: features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0) elif not (self.is_action_mask) and not (self.is_emb): features = np.concatenate((u_ent_features, u_bow), axis=0) if self.is_action_mask: probs, prediction = self.net.forward(features, action_mask) else: probs, prediction = self.net.forward(features) correct_examples += int(prediction == r) turn_accuracy += correct_examples / len(dialog) accuracy = correct_examples / len(dialog) if (accuracy == 1.0): dialog_accuracy += 1 turn_accuracy = turn_accuracy / num_dev_examples dialog_accuracy = dialog_accuracy / num_dev_examples return turn_accuracy, dialog_accuracy
def __init__(self, args): self.response_accuracy = [] self.dialog_accuracy = [] try: ###################### selective import ############################# if args[0] == 'am': self.is_action_mask = True else: self.is_action_mask = False if args[1] == 'emb': self.is_emb = True else: self.is_emb = False self.network_type = args[2] self.lang_type = args[3] if self.lang_type == 'eng': from modules.entities import EntityTracker from modules.data_utils import Data from modules.actions import ActionTracker from modules.bow import BoW_encoder elif self.lang_type == 'kor': from modules.entities_kor import EntityTracker from modules.data_utils_kor import Data from modules.actions_kor import ActionTracker from modules.bow_kor import BoW_encoder ################################################################### except: rospy.logwarn( "please try again. i.e. ... train.py <am> <emb> <bidirectional_lstm> <eng>" ) if self.is_emb: if self.lang_type == 'eng': self.emb = UtteranceEmbed(lang=self.lang_type) elif self.lang_type == 'kor': self.emb = UtteranceEmbed(lang=self.lang_type) et = EntityTracker() self.bow_enc = BoW_encoder() at = ActionTracker(et) at.do_display_template() self.dataset, dialog_indices = Data(et, at).trainset # self.dialog_indices_tr = dialog_indices[:200] self.dialog_indices_tr = random.sample(dialog_indices, 200) # self.dialog_indices_dev = dialog_indices[200:250] self.dialog_indices_dev = random.sample(dialog_indices, 50) self.action_templates = at.get_action_templates() action_size = at.action_size nb_hidden = 128 # set feature input if self.is_action_mask and self.is_emb: obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features + at.action_size elif self.is_action_mask and not (self.is_emb): obs_size = self.bow_enc.vocab_size + et.num_features + at.action_size elif not (self.is_action_mask) and self.is_emb: obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features elif not (self.is_action_mask) and not (self.is_emb): obs_size = self.bow_enc.vocab_size + et.num_features # set network_type type if self.network_type == 'gru': self.net = GRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'reversed_lstm': self.net = ReversingLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'reversed_gru': self.net = ReversingGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'stacked_gru': self.net = StackedGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'stacked_lstm': self.net = StackedLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'lstm': self.net = LSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'bidirectional_lstm': self.net = BidirectionalLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) elif self.network_type == 'bidirectional_gru': self.net = BidirectionalGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_action_mask) file_path = os.path.join(rospkg.RosPack().get_path('dialogue_system'), 'log', self.network_type) # init logging self.logger = self.get_logger(file_path) msg = "'\033[94m[%s trainer]\033[0m initialized - %s (action_mask: %s, embedding: %s, lang: %s, obs_size: %s, bow: %s, context_feature: %s, action_size: %s)" % ( rospy.get_name(), self.net.__class__.__name__, self.is_action_mask, self.is_emb, self.lang_type, obs_size, self.bow_enc.vocab_size, et.num_features, action_size) rospy.loginfo(msg)
def main(in_clean_dataset_folder, in_noisy_dataset_folder, in_model_folder, in_mode, in_runs_number): rev_vocab, kb, action_templates, config = load_model(in_model_folder) clean_dialogs, clean_indices = read_dialogs(os.path.join(in_clean_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) noisy_dialogs, noisy_indices = read_dialogs(os.path.join(in_noisy_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) max_noisy_dialog_length = max([item['end'] - item['start'] + 1 for item in noisy_indices]) config['max_input_length'] = max_noisy_dialog_length post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns(noisy_dialogs, BABI_CONFIG['backoff_utterance'].lower()) et = EntityTracker(kb) at = ActionTracker(None, et) at.set_action_templates(action_templates) vocab = {word: idx for idx, word in enumerate(rev_vocab)} data_clean = make_dataset_for_vhcn_v2(clean_dialogs, clean_indices, vocab, et, at, **config) data_noisy = make_dataset_for_vhcn_v2(noisy_dialogs, noisy_indices, vocab, et, at, **config) context_features_clean, action_masks_clean = data_clean[2:4] net = VariationalHierarchicalLSTMv3(rev_vocab, config, context_features_clean.shape[-1], action_masks_clean.shape[-1]) net.restore(in_model_folder) if in_mode == 'clean': eval_stats_clean = evaluate_advanced(net, data_clean, at.action_templates, BABI_CONFIG['backoff_utterance'].lower(), post_ood_turns=post_ood_turns_clean, runs_number=in_runs_number) print('Clean dataset: {} turns overall'.format(eval_stats_clean['total_turns'])) print('Accuracy:') accuracy = eval_stats_clean['correct_turns'] / eval_stats_clean['total_turns'] accuracy_continuous = eval_stats_clean['correct_continuous_turns'] / eval_stats_clean['total_turns'] accuracy_post_ood = eval_stats_clean['correct_post_ood_turns'] / eval_stats_clean['total_post_ood_turns'] \ if eval_stats_clean['total_post_ood_turns'] != 0 \ else 0 print('overall: {:.3f}; continuous: {:.3f}; directly post-OOD: {:.3f}'.format(accuracy, accuracy_continuous, accuracy_post_ood)) print('Loss : {:.3f}'.format(eval_stats_clean['avg_loss'])) elif in_mode == 'noisy': eval_stats_noisy = evaluate_advanced(net, data_noisy, at.action_templates, BABI_CONFIG['backoff_utterance'].lower(), post_ood_turns=post_ood_turns_noisy, runs_number=in_runs_number) print('\n\n') print('Noisy dataset: {} turns overall'.format(eval_stats_noisy['total_turns'])) print('Accuracy:') accuracy = eval_stats_noisy['correct_turns'] / eval_stats_noisy['total_turns'] accuracy_continuous = eval_stats_noisy['correct_continuous_turns'] / eval_stats_noisy['total_turns'] accuracy_post_ood = eval_stats_noisy['correct_post_ood_turns'] / eval_stats_noisy['total_post_ood_turns'] \ if eval_stats_noisy['total_post_ood_turns'] != 0 \ else 0 accuracy_ood = eval_stats_noisy['correct_ood_turns'] / eval_stats_noisy['total_ood_turns'] \ if eval_stats_noisy['total_ood_turns'] != 0 \ else 0 print('overall: {:.3f}; continuous: {:.3f}; directly post-OOD: {:.3f}; OOD: {:.3f}'.format(accuracy, accuracy_continuous, accuracy_post_ood, accuracy_ood)) print('Loss : {:.3f}'.format(eval_stats_noisy['avg_loss'])) elif in_mode == 'noisy_ignore_ood': eval_stats_no_ood = evaluate_advanced(net, data_noisy, at.action_templates, BABI_CONFIG['backoff_utterance'].lower(), post_ood_turns=post_ood_turns_noisy, ignore_ood_accuracy=True, runs_number=in_runs_number) print('Accuracy (OOD turns ignored):') accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood['total_turns'] accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \ if eval_stats_no_ood['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_no_ood['correct_post_ood_turns'] / eval_stats_no_ood['total_post_ood_turns'] \ if eval_stats_no_ood['total_post_ood_turns'] != 0 \ else 0 print('overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'.format(accuracy, accuracy_after_ood, accuracy_post_ood))
def __init__(self): # stor whole dialogues self.story = [] self.sp_confidecne = [] self.file_path = os.path.join( rospkg.RosPack().get_path('dialogue_system'), 'log', 'dialogue.txt') # count turn taking self.usr_count = 0 self.sys_count = 0 # paramaters self.network_type = rospy.get_param('~network_model', 'stacked_lstm') self.lang_type = rospy.get_param('~lang', 'eng') self.is_emb = rospy.get_param('~embedding', 'false') self.is_am = rospy.get_param('~action_mask', "true") self.user_num = rospy.get_param('~user_number', '0') # call rest of modules self.et = EntityTracker() self.at = ActionTracker(self.et) self.bow_enc = BoW_encoder() self.emb = UtteranceEmbed(lang=self.lang_type) # select observation size for RNN if self.is_am and self.is_emb: obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features + self.at.action_size elif self.is_am and not (self.is_emb): obs_size = self.bow_enc.vocab_size + self.et.num_features + self.at.action_size elif not (self.is_am) and self.is_emb: obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features elif not (self.is_am) and not (self.is_emb): obs_size = self.bow_enc.vocab_size + self.et.num_features self.action_template = self.at.get_action_templates() self.at.do_display_template() # must clear entities space self.et.do_clear_entities() action_size = self.at.action_size nb_hidden = 128 if self.network_type == 'gru': self.net = GRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'reversed_lstm': self.net = ReversingLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'reversed_gru': self.net = ReversingGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'stacked_gru': self.net = StackedGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'stacked_lstm': self.net = StackedLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'lstm': self.net = LSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'bidirectional_lstm': self.net = BidirectionalLSTM(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) elif self.network_type == 'bidirectional_gru': self.net = BidirectionalGRU(obs_size=obs_size, nb_hidden=nb_hidden, action_size=action_size, lang=self.lang_type, is_action_mask=self.is_am) # restore trained model self.net.restore() # rostopics self.pub_reply = rospy.Publisher('reply', Reply, queue_size=10) self.pub_complete = rospy.Publisher('complete_execute_scenario', Empty, queue_size=10) rospy.Subscriber('raising_events', RaisingEvents, self.handle_raise_events) try: rospy.wait_for_service('reception_db/query_data') self.get_response_db = rospy.ServiceProxy( 'reception_db/query_data', DBQuery) rospy.logwarn("waiting for reception DB module...") except rospy.exceptions.ROSInterruptException as e: rospy.logerr(e) quit() rospy.logwarn( "network: {}, lang: {}, action_mask: {}, embedding: {}, user_number: {}" .format(self.network_type, self.lang_type, self.is_am, self.is_emb, self.user_num)) self.story.append('user number: %s' % self.user_num) rospy.loginfo('\033[94m[%s]\033[0m initialized.' % rospy.get_name())
def main(in_clean_dataset_folder, in_noisy_dataset_folder, in_model_folder, in_no_ood_evaluation): rev_vocab, kb, action_templates, config = load_model(in_model_folder) clean_dialogs, clean_indices = read_dialogs(os.path.join( in_clean_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) noisy_dialogs, noisy_indices = read_dialogs(os.path.join( in_noisy_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'), with_indices=True) post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns( noisy_dialogs) assert len(post_ood_turns_clean) == len(post_ood_turns_noisy) for post_ood_turn_clean, post_ood_turn_noisy in zip( sorted(post_ood_turns_clean), sorted(post_ood_turns_noisy)): noisy_dialogs[post_ood_turn_noisy][0] = clean_dialogs[ post_ood_turn_clean][0] et = EntityTracker(kb) at = ActionTracker(None, et) at.set_action_templates(action_templates) vocab = {word: idx for idx, word in enumerate(rev_vocab)} X_clean, context_features_clean, action_masks_clean, y_clean = make_dataset_for_hierarchical_lstm( clean_dialogs, clean_indices, vocab, et, at, **config) X_noisy, context_features_noisy, action_masks_noisy, y_noisy = make_dataset_for_hierarchical_lstm( noisy_dialogs, noisy_indices, vocab, et, at, **config) net = HierarchicalLSTM(config, context_features_clean.shape[-1], action_masks_clean.shape[-1]) net.restore(in_model_folder) eval_stats_clean = evaluate_advanced( net, (X_clean, context_features_clean, action_masks_clean, y_clean), at.action_templates, post_ood_turns=post_ood_turns_clean) print('Clean dataset: {} turns overall'.format( eval_stats_clean['total_turns'])) print('Accuracy:') accuracy = eval_stats_clean['correct_turns'] / eval_stats_clean[ 'total_turns'] accuracy_post_ood = eval_stats_clean['correct_post_ood_turns'] / eval_stats_clean['total_post_ood_turns'] \ if eval_stats_clean['total_post_ood_turns'] != 0 \ else 0 print('overall: {:.3f}; directly post-OOD: {:.3f}'.format( accuracy, accuracy_post_ood)) print('Loss : {:.3f}'.format(eval_stats_clean['avg_loss'])) eval_stats_noisy = evaluate_advanced( net, (X_noisy, context_features_noisy, action_masks_noisy, y_noisy), at.action_templates, post_ood_turns=post_ood_turns_noisy) print('\n\n') print( 'Noisy dataset: {} turns overall, {} turns after the first OOD'.format( eval_stats_noisy['total_turns'], eval_stats_noisy['total_turns_after_ood'])) print('Accuracy:') accuracy = eval_stats_noisy['correct_turns'] / eval_stats_noisy[ 'total_turns'] accuracy_after_ood = eval_stats_noisy['correct_turns_after_ood'] / eval_stats_noisy['total_turns_after_ood'] \ if eval_stats_noisy['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_noisy['correct_post_ood_turns'] / eval_stats_noisy['total_post_ood_turns'] \ if eval_stats_noisy['total_post_ood_turns'] != 0 \ else 0 print( 'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'. format(accuracy, accuracy_after_ood, accuracy_post_ood)) print('Loss : {:.3f}'.format(eval_stats_noisy['avg_loss'])) if in_no_ood_evaluation: eval_stats_no_ood = evaluate_advanced( net, (X_noisy, context_features_noisy, action_masks_noisy, y_noisy), at.action_templates, post_ood_turns=post_ood_turns_noisy, ignore_ood_accuracy=True) print('Accuracy (OOD turns ignored):') accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood[ 'total_turns'] accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \ if eval_stats_no_ood['total_turns_after_ood'] != 0 \ else 0 accuracy_post_ood = eval_stats_no_ood['correct_post_ood_turns'] / eval_stats_no_ood['total_post_ood_turns'] \ if eval_stats_no_ood['total_post_ood_turns'] != 0 \ else 0 print( 'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}' .format(accuracy, accuracy_after_ood, accuracy_post_ood))
def reset(self): self.net.reset_state() self.et = EntityTracker() self.at = ActionTracker(self.et)