def __init__(self):
import os
#实体追踪
et = EntityTracker()
#词袋 word2vec
self.bow_enc = BoW_encoder()
#加载word2vec embedding
self.emb = UtteranceEmbed()
#将实体追踪器添加到动作追踪器中
at = ActionTracker(et)
#得到数据集和对话开始 结束行数
self.dataset, dialog_indices = Data(et, at).trainset
#划分数据集：200对做训练 50对做测试
self.dialog_indices_tr = dialog_indices
self.dialog_indices_dev = dialog_indices
#obs_size 300维的词向量 + 85个袋中的词 + 4个槽位
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features
#话术模板
self.action_templates = at.get_action_templates()
#动作个数
action_size = at.action_size
#隐藏层神经元个数
nb_hidden = 128
self.net = LSTM_net(obs_size=obs_size,
action_size=action_size,
nb_hidden=nb_hidden)
def dialog_train(self, dialog):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
loss = 0.
# iterate through dialog
for (u,r) in dialog:
# print("Here in the dialog loop")
u_ent = et.extract_entities(u)
u_ent_features = et.context_features()
u_emb = self.emb.encode(u)
u_bow = self.bow_enc.encode(u)
# concat features
features = torch.autograd.Variable(torch.from_numpy(np.concatenate((u_ent_features, u_emb, u_bow), axis=0))).float()
# print(features)
# get action mask
action_mask = torch.autograd.Variable(torch.from_numpy(at.action_mask()))
r = torch.autograd.Variable(torch.LongTensor([r]))
# print(r)
# forward propagation
# train step
loss += self.net.train_step(features, r, action_mask)
return loss/len(dialog)
def dialog_train(self, dialog):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
loss = 0.
# iterate through dialog
#u 用户输入 r 对应的动作索引
for (u, r) in dialog:
#u_ent 分词后的字符串
u_ent = et.extract_entities(u)
#槽位填充情况 【0 0 0 0】
u_ent_features = et.context_features()
#word2vec
u_emb = self.emb.encode(u)
#multi-hot
u_bow = self.bow_enc.encode(u)
# concat features
#300 + 85 + 4 = 389
features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0)
# get action mask
action_mask = at.action_mask()
# forward propagation
# train step
loss += self.net.train_step(features, r, action_mask)
return loss / len(dialog)
def dialog_train(self, dialog):
###################################################################
if self.lang_type == 'eng':
from modules.entities import EntityTracker
from modules.data_utils import Data
from modules.actions import ActionTracker
from modules.bow import BoW_encoder
elif self.lang_type == 'kor':
from modules.entities_kor import EntityTracker
from modules.data_utils_kor import Data
from modules.actions_kor import ActionTracker
from modules.bow_kor import BoW_encoder
###################################################################
et = EntityTracker()
at = ActionTracker(et)
# reset state in network_type
self.net.reset_state()
loss = 0.
for (u, r) in dialog:
u_ent = et.extract_entities(u)
u_ent_features = et.context_features()
u_bow = self.bow_enc.encode(u)
if self.is_emb:
u_emb = self.emb.encode(u)
if self.is_action_mask:
action_mask = at.action_mask()
# print(u, r)
# print(u_ent_features)
# print('================================')
# print(u_emb)
# print('================================')
# print(u_bow)
# print('================================')
# print(action_mask)
# concatenated features
if self.is_action_mask and self.is_emb:
features = np.concatenate(
(u_ent_features, u_emb, u_bow, action_mask), axis=0)
elif self.is_action_mask and not (self.is_emb):
features = np.concatenate((u_ent_features, u_bow, action_mask),
axis=0)
elif not (self.is_action_mask) and self.is_emb:
features = np.concatenate((u_ent_features, u_emb, u_bow),
axis=0)
elif not (self.is_action_mask) and not (self.is_emb):
features = np.concatenate((u_ent_features, u_bow), axis=0)
# forward propagation with cumulative loss
if self.is_action_mask:
loss += self.net.train_step(features, r, action_mask)
else:
loss += self.net.train_step(features, r)
return loss / len(dialog)
def main(in_dataset_folder, in_model_folder, in_no_ood_evaluation):
rev_vocab, kb, action_templates, config = load_model(in_model_folder)
test_dialogs, test_indices = read_dialogs(os.path.join(
in_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'),
with_indices=True)
et = EntityTracker(kb)
at = ActionTracker(None, et)
at.set_action_templates(action_templates)
vocab = {word: idx for idx, word in enumerate(rev_vocab)}
X, context_features, action_masks, y = make_dataset_for_hierarchical_lstm(
test_dialogs, test_indices, vocab, et, at, **config)
net = HierarchicalLSTM(config, context_features.shape[-1],
action_masks.shape[-1])
net.restore(in_model_folder)
eval_stats_full_dataset = evaluate_advanced(
net, (X, context_features, action_masks, y), test_dialogs,
at.action_templates)
print(
'Full dataset: {} turns overall, {} turns after the first OOD'.format(
eval_stats_full_dataset['total_turns'],
eval_stats_full_dataset['total_turns_after_ood']))
print('Accuracy:')
accuracy = eval_stats_full_dataset[
'correct_turns'] / eval_stats_full_dataset['total_turns']
accuracy_after_ood = eval_stats_full_dataset['correct_turns_after_ood'] / eval_stats_full_dataset['total_turns_after_ood'] \
if eval_stats_full_dataset['total_turns_after_ood'] != 0 \
else 0
accuracy_post_ood = eval_stats_full_dataset['correct_post_ood_turns'] / eval_stats_full_dataset['total_post_ood_turns'] \
if eval_stats_full_dataset['total_post_ood_turns'] != 0 \
else 0
print(
'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'.
format(accuracy, accuracy_after_ood, accuracy_post_ood))
print('Loss : {:.3f}'.format(eval_stats_full_dataset['avg_loss']))
if in_no_ood_evaluation:
eval_stats_no_ood = evaluate_advanced(
net, (X, context_features, action_masks, y),
test_indices,
at.action_templates,
ignore_ood_accuracy=True)
print('Accuracy (OOD turns ignored):')
accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood[
'total_turns']
accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \
if eval_stats_no_ood['total_turns_after_ood'] != 0 \
else 0
accuracy_post_ood = eval_stats_no_ood['correct_post_ood_turns'] / eval_stats_no_ood['total_post_ood_turns'] \
if eval_stats_no_ood['total_post_ood_turns'] != 0 \
else 0
print(
'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'
.format(accuracy, accuracy_after_ood, accuracy_post_ood))
print('Loss : {:.3f}'.format(eval_stats_no_ood['avg_loss']))
def evaluate(self):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
dialog_accuracy = 0.
r_count = 0 #Count of task 15
count = 0 # Total count of rewards
for dialog_idx in self.dialog_indices_dev:
start, end = dialog_idx['start'], dialog_idx['end']
dialog = self.dataset[start:end]
num_dev_examples = len(self.dialog_indices_dev)
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
# iterate through dialog
correct_examples = 0
for (u,r) in dialog:
# encode utterance
u_ent = et.extract_entities(u)
u_ent_features = et.context_features()
u_emb = self.emb.encode(u)
u_bow = self.bow_enc.encode(u)
# concat features
# get action mask
features = torch.autograd.Variable(torch.from_numpy(np.concatenate((u_ent_features, u_emb, u_bow), axis=0))).float()
# print(features)
# get action mask
action_mask = torch.autograd.Variable(torch.from_numpy(at.action_mask()))
# r = torch.autograd.Variable(torch.LongTensor([r]))
# forward propagation
# train step
logits,probs,prediction = self.net.forward(features, action_mask)
# print("logits", logits)
# print("probs", probs)
# print("prediction", logits,probs,prediction)
# print(prediction,r)
correct_examples += int(prediction == r)
if r==15:
r_count += 1
count += 1
# get dialog accuracy
dialog_accuracy += correct_examples/len(dialog)
print("task 15 was the answer with freq",r_count/count)
return dialog_accuracy/num_dev_examples
def interact(self):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
# begin interaction loop
while True:
# get input from user
u = input(':: ')
# check if user wants to begin new session
if u == 'clear' or u == 'reset' or u == 'restart':
self.net.reset_state()
et = EntityTracker()
at = ActionTracker(et)
print('')
# check for exit command
elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q':
break
else:
# ENTER press : silence
if not u:
u = '<SILENCE>'
# encode
u_ent, u_entities = et.extract_entities(u, is_test=True)
u_ent_features = et.context_features()
u_emb = self.emb.encode(u)
u_bow = self.bow_enc.encode(u)
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow),
axis=0)
# get action mask
action_mask = at.action_mask()
# forward
prediction = self.net.forward(features, action_mask)
if self.post_process(prediction, u_ent_features):
print(
'>>', 'api_call ' + u_entities['<cuisine>'] + ' ' +
u_entities['<location>'] + ' ' +
u_entities['<party_size>'] + ' ' +
u_entities['<rest_type>'])
else:
prediction = self.action_post_process(
prediction, u_entities)
print('>>', self.action_templates[prediction])
def __init__(self):
et = EntityTracker()
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed()
at = ActionTracker(et)
self.train_dataset, train_dialog_indices = Data(et, at).train_set
self.test_dataset, test_dialog_indices = Data(et, at).test_set
print('=========================\n')
print('length of Train dialog indices : ', len(train_dialog_indices))
print('=========================\n')
print('=========================\n')
print('length of Test dialog indices : ', len(test_dialog_indices))
print('=========================\n')
# Shuffle Training Dataset
random.shuffle(train_dialog_indices)
self.dialog_indices_tr = train_dialog_indices
self.dialog_indices_dev = test_dialog_indices
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features
self.action_templates = at.get_action_templates()
action_size = at.action_size
# nb_hidden = 128
nb_hidden = 150
print('=========================\n')
print('Action_templates: ', action_size)
print('=========================\n')
self.net = LSTM_net(obs_size=obs_size,
action_size=action_size,
nb_hidden=nb_hidden)
self.et = et
self.at = at
action_projection = []
for action in self.action_templates:
action_projection.append(self.emb.encode(action))
self.action_projection = np.transpose(action_projection)
self.action_size = action_size
def evaluate(self):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
dialog_accuracy = 0.
correct_dialogue_count = 0
for dialog_idx in self.dialog_indices_dev:
start, end = dialog_idx['start'], dialog_idx['end']
dialog = self.dataset[start:end]
num_dev_examples = len(self.dialog_indices_dev)
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
# iterate through dialog
correct_examples = 0
for (u, r) in dialog:
# encode utterance
u_ent = et.extract_entities(u)
u_ent_features = et.context_features()
u_emb = self.emb.encode(u)
u_bow = self.bow_enc.encode(u)
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow),
axis=0)
# get action mask
action_mask = at.action_mask()
# forward propagation
# train step
prediction = self.net.forward(features, action_mask)
correct_examples += int(prediction == r)
if correct_examples == len(dialog):
correct_dialogue_count += 1
# get dialog accuracy
dialog_accuracy += correct_examples / len(dialog)
per_response_accuracy = dialog_accuracy / num_dev_examples
per_dialogue_accuracy = correct_dialogue_count / num_dev_examples
return per_response_accuracy, per_dialogue_accuracy
def __init__(self):
et = EntityTracker()
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed()
at = ActionTracker(et)
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features
self.action_templates = at.get_action_templates()
action_size = at.action_size
nb_hidden = 128
self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden)
# restore checkpoint
self.net.restore()
def evaluate(self):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
dialog_accuracy = 0.
#加载测试集
for dialog_idx in self.dialog_indices_dev:
start, end = dialog_idx['start'], dialog_idx['end']
dialog = self.dataset[start:end]
num_dev_examples = len(self.dialog_indices_dev)
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
# iterate through dialog
correct_examples = 0
#对于每个dialog 提取出utterance 和 response
for (u, r) in dialog:
# encode utterance
#提取出user中带有的实体
u_ent = et.extract_entities(u)
#提取当前槽位填充情况
u_ent_features = et.context_features()
u_emb = self.emb.encode(u)
u_bow = self.bow_enc.encode(u)
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow),
axis=0)
# get action mask 16维的multi-hot 向量
action_mask = at.action_mask()
# forward propagation
# train step
prediction = self.net.forward(features, action_mask)
correct_examples += int(prediction == r)
# get dialog accuracy
dialog_accuracy += correct_examples / len(dialog)
return dialog_accuracy / num_dev_examples
def interact(self):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
# begin interaction loop
while True:
# get input from user
u = input(':: ')
# check if user wants to begin new session
if u == 'clear' or u == 'reset' or u == 'restart':
self.net.reset_state()
et = EntityTracker()
at = ActionTracker(et)
print('')
# check for exit command
elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q':
break
else:
# ENTER press : silence
if not u:
u = '<SILENCE>'
# encode
u_ent = et.extract_entities(u)
print("get u ent " + u_ent)
u_ent_features = et.context_features()
print("槽位情况" + str(u_ent_features))
u_emb = self.emb.encode(u)
u_bow = self.bow_enc.encode(u)
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow),
axis=0)
# get action mask
action_mask = at.action_mask()
# forward
prediction = self.net.forward(features, action_mask)
print("prediction " + str(prediction))
print('>>', self.action_templates[prediction])
def __init__(self):
et = EntityTracker()
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed()
at = ActionTracker(et)
self.dataset, dialog_indices = Data(et, at).trainset
self.dialog_indices_tr = dialog_indices[:200]
self.dialog_indices_dev = dialog_indices[200:250]
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features
self.action_templates = at.get_action_templates()
action_size = at.action_size
nb_hidden = 128
self.net = LSTM_net(obs_size=obs_size,
action_size=action_size,
nb_hidden=nb_hidden)
def main(in_dataset_folder, in_noisy_dataset_folder, in_custom_vocab_file, in_model_folder, in_config):
with open(in_config, encoding='utf-8') as config_in:
config = json.load(config_in)
train_dialogs, train_indices = read_dialogs(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-trn.txt'),
with_indices=True)
dev_dialogs, dev_indices = read_dialogs(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-dev.txt'),
with_indices=True)
test_dialogs, test_indices = read_dialogs(os.path.join(in_noisy_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'),
with_indices=True)
kb = make_augmented_knowledge_base(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-kb.txt'),
os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-candidates.txt'))
max_noisy_dialog_length = max([item['end'] - item['start'] + 1 for item in test_indices])
config['max_input_length'] = max_noisy_dialog_length
post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns(test_dialogs, BABI_CONFIG['backoff_utterance'].lower())
et = EntityTracker(kb)
at = ActionTracker(os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-candidates.txt'), et)
if in_custom_vocab_file is not None:
with open(in_custom_vocab_file) as vocab_in:
rev_vocab = [line.rstrip() for line in vocab_in]
vocab = {word: idx for idx, word in enumerate(rev_vocab)}
else:
utterances_tokenized = []
for dialog in train_dialogs:
utterances_tokenized += list(map(lambda x: x.split(), dialog))
vocab, rev_vocab = make_vocabulary(utterances_tokenized,
config['max_vocabulary_size'],
special_tokens=[PAD, START, UNK, EOS] + list(kb.keys()))
config['vocabulary_size'] = len(vocab)
data_train = make_dataset_for_hierarchical_hcn(train_dialogs, train_indices, vocab, et, at, **config)
data_dev = make_dataset_for_hierarchical_hcn(dev_dialogs, dev_indices, vocab, et, at, **config)
data_test = make_dataset_for_hierarchical_hcn(test_dialogs, test_indices, vocab, et, at, **config)
random_input = generate_dropout_turns_for_hierarchical_hcn(10000,
config['max_sequence_length'],
[utterance[0] for utterance in train_dialogs],
vocab,
config['turn_word_dropout_prob'])
save_model(rev_vocab, config, kb, at.action_templates, in_model_folder)
trainer = Trainer(data_train,
data_dev,
data_test,
at.action_templates,
random_input,
post_ood_turns_noisy,
config,
vocab,
in_model_folder)
trainer.train()
def main(in_dataset_folder, in_custom_vocab_file, in_model_folder, in_config):
with open(in_config, encoding='utf-8') as config_in:
config = json.load(config_in)
train_dialogs, train_indices = read_dialogs(os.path.join(
in_dataset_folder, 'dialog-babi-task6-dstc2-trn.txt'),
with_indices=True)
dev_dialogs, dev_indices = read_dialogs(os.path.join(
in_dataset_folder, 'dialog-babi-task6-dstc2-dev.txt'),
with_indices=True)
test_dialogs, test_indices = read_dialogs(os.path.join(
in_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'),
with_indices=True)
kb = make_augmented_knowledge_base(
os.path.join(in_dataset_folder, 'dialog-babi-task6-dstc2-kb.txt'),
os.path.join(in_dataset_folder,
'dialog-babi-task6-dstc2-candidates.txt'))
et = EntityTracker(kb)
at = ActionTracker(
os.path.join(in_dataset_folder,
'dialog-babi-task6-dstc2-candidates.txt'), et)
if in_custom_vocab_file is not None:
with open(in_custom_vocab_file) as vocab_in:
rev_vocab = [line.rstrip() for line in vocab_in]
vocab = {word: idx for idx, word in enumerate(rev_vocab)}
else:
utterances_tokenized = []
for dialog in train_dialogs:
utterances_tokenized += list(map(lambda x: x.split(), dialog))
vocab, rev_vocab = make_vocabulary(
utterances_tokenized,
config['max_vocabulary_size'],
special_tokens=[PAD, START, UNK, EOS] + list(kb.keys()))
config['vocabulary_size'] = len(vocab)
data_train = make_dataset_for_variational_hcn(train_dialogs, train_indices,
vocab, et, at, **config)
data_dev = make_dataset_for_variational_hcn(dev_dialogs, dev_indices,
vocab, et, at, **config)
data_test = make_dataset_for_variational_hcn(test_dialogs, test_indices,
vocab, et, at, **config)
random_input = generate_random_input_for_variational_hcn(
10000, config['max_sequence_length'], vocab, rev_vocab)
save_model(rev_vocab, config, kb, at.action_templates, in_model_folder)
trainer = Trainer(data_train, data_dev, data_test, at.action_templates,
random_input, rev_vocab, config, in_model_folder)
trainer.train()
def __init__(self):
et = EntityTracker()
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed()
at = ActionTracker(et)
'''
['any preference on a type of cuisine', 'api_call <cuisine> <location> <party_size> <rest_type>', 'great let me do the reservation', 'hello what can i help you with today', 'here it is <info_address>', 'here it is <info_phone>', 'how many people would be in your party', "i'm on it", 'is there anything i can help you with', 'ok let me look into some options for you', 'sure is there anything else to update', 'sure let me find an other option for you', 'what do you think of this option: <restaurant>', 'where should it be', 'which price range are looking for', "you're welcome"]
'''
self.dataset, dialog_indices = Data(et, at).trainset
self.dialog_indices_tr = dialog_indices[:200]
self.dialog_indices_dev = dialog_indices[200:250]
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features
self.action_templates = at.get_action_templates()
action_size = at.action_size
nb_hidden = 128
self.net = LSTM_net(obs_size=obs_size,
action_size=action_size,
nb_hidden=nb_hidden)
def __init__(self):
et = EntityTracker()
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed()
at = ActionTracker(et)
self.dataset, dialog_indices = Data(et, at).trainset
train_indices = joblib.load('data/train_test_list/train_indices_759')
test_indices = joblib.load('data/train_test_list/test_indices_759_949')
self.dialog_indices_tr = train_indices
self.dialog_indices_dev = test_indices
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features
self.action_templates = at.get_action_templates()
action_size = at.action_size
nb_hidden = 128
self.net = LSTM_net(obs_size=obs_size,
action_size=action_size,
nb_hidden=nb_hidden)
def dialog_train(self, dialog):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
loss = 0.
# iterate through dialog
for (u, r) in dialog:
u_ent = et.extract_entities(u)
u_ent_features = et.context_features()
u_emb = self.emb.encode(u)
u_bow = self.bow_enc.encode(u)
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0)
# get action mask
action_mask = at.action_mask()
# forward propagation
# train step
loss += self.net.train_step(features, r, action_mask)
return loss/len(dialog)
def evaluate(self, eval=False):
###################################################################
if self.lang_type == 'eng':
from modules.entities import EntityTracker
from modules.data_utils import Data
from modules.actions import ActionTracker
from modules.bow import BoW_encoder
elif self.lang_type == 'kor':
from modules.entities_kor import EntityTracker
from modules.data_utils_kor import Data
from modules.actions_kor import ActionTracker
from modules.bow_kor import BoW_encoder
###################################################################
et = EntityTracker()
at = ActionTracker(et)
# only for evaluation purpose
if eval:
self.net.restore()
# reset entities extractor
turn_accuracy = 0.
dialog_accuracy = 0.
for dialog_idx in self.dialog_indices_dev:
start, end = dialog_idx['start'], dialog_idx['end']
dialog = self.dataset[start:end]
num_dev_examples = len(self.dialog_indices_dev)
et = EntityTracker()
at = ActionTracker(et)
# reset network_type before evaluate.
self.net.reset_state()
correct_examples = 0
for (u, r) in dialog:
u_ent = et.extract_entities(u)
u_ent_features = et.context_features()
u_bow = self.bow_enc.encode(u)
if self.is_emb:
u_emb = self.emb.encode(u)
if self.is_action_mask:
action_mask = at.action_mask()
# concatenated features
if self.is_action_mask and self.is_emb:
features = np.concatenate(
(u_ent_features, u_emb, u_bow, action_mask), axis=0)
elif self.is_action_mask and not (self.is_emb):
features = np.concatenate(
(u_ent_features, u_bow, action_mask), axis=0)
elif not (self.is_action_mask) and self.is_emb:
features = np.concatenate((u_ent_features, u_emb, u_bow),
axis=0)
elif not (self.is_action_mask) and not (self.is_emb):
features = np.concatenate((u_ent_features, u_bow), axis=0)
if self.is_action_mask:
probs, prediction = self.net.forward(features, action_mask)
else:
probs, prediction = self.net.forward(features)
correct_examples += int(prediction == r)
turn_accuracy += correct_examples / len(dialog)
accuracy = correct_examples / len(dialog)
if (accuracy == 1.0):
dialog_accuracy += 1
turn_accuracy = turn_accuracy / num_dev_examples
dialog_accuracy = dialog_accuracy / num_dev_examples
return turn_accuracy, dialog_accuracy
def reset(self):
self.net.reset_state()
self.et = EntityTracker()
self.at = ActionTracker(self.et)
def __init__(self, args):
self.response_accuracy = []
self.dialog_accuracy = []
try:
###################### selective import #############################
if args[0] == 'am':
self.is_action_mask = True
else:
self.is_action_mask = False
if args[1] == 'emb':
self.is_emb = True
else:
self.is_emb = False
self.network_type = args[2]
self.lang_type = args[3]
if self.lang_type == 'eng':
from modules.entities import EntityTracker
from modules.data_utils import Data
from modules.actions import ActionTracker
from modules.bow import BoW_encoder
elif self.lang_type == 'kor':
from modules.entities_kor import EntityTracker
from modules.data_utils_kor import Data
from modules.actions_kor import ActionTracker
from modules.bow_kor import BoW_encoder
###################################################################
except:
rospy.logwarn(
"please try again. i.e. ... train.py <am> <emb> <bidirectional_lstm> <eng>"
)
if self.is_emb:
if self.lang_type == 'eng':
self.emb = UtteranceEmbed(lang=self.lang_type)
elif self.lang_type == 'kor':
self.emb = UtteranceEmbed(lang=self.lang_type)
et = EntityTracker()
self.bow_enc = BoW_encoder()
at = ActionTracker(et)
at.do_display_template()
self.dataset, dialog_indices = Data(et, at).trainset
# self.dialog_indices_tr = dialog_indices[:200]
self.dialog_indices_tr = random.sample(dialog_indices, 200)
# self.dialog_indices_dev = dialog_indices[200:250]
self.dialog_indices_dev = random.sample(dialog_indices, 50)
self.action_templates = at.get_action_templates()
action_size = at.action_size
nb_hidden = 128
# set feature input
if self.is_action_mask and self.is_emb:
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features + at.action_size
elif self.is_action_mask and not (self.is_emb):
obs_size = self.bow_enc.vocab_size + et.num_features + at.action_size
elif not (self.is_action_mask) and self.is_emb:
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features
elif not (self.is_action_mask) and not (self.is_emb):
obs_size = self.bow_enc.vocab_size + et.num_features
# set network_type type
if self.network_type == 'gru':
self.net = GRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_action_mask)
elif self.network_type == 'reversed_lstm':
self.net = ReversingLSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_action_mask)
elif self.network_type == 'reversed_gru':
self.net = ReversingGRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_action_mask)
elif self.network_type == 'stacked_gru':
self.net = StackedGRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_action_mask)
elif self.network_type == 'stacked_lstm':
self.net = StackedLSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_action_mask)
elif self.network_type == 'lstm':
self.net = LSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_action_mask)
elif self.network_type == 'bidirectional_lstm':
self.net = BidirectionalLSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_action_mask)
elif self.network_type == 'bidirectional_gru':
self.net = BidirectionalGRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_action_mask)
file_path = os.path.join(rospkg.RosPack().get_path('dialogue_system'),
'log', self.network_type)
# init logging
self.logger = self.get_logger(file_path)
msg = "'\033[94m[%s trainer]\033[0m initialized - %s (action_mask: %s, embedding: %s, lang: %s, obs_size: %s, bow: %s, context_feature: %s, action_size: %s)" % (
rospy.get_name(), self.net.__class__.__name__, self.is_action_mask,
self.is_emb, self.lang_type, obs_size, self.bow_enc.vocab_size,
et.num_features, action_size)
rospy.loginfo(msg)
def interact(self):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
# begin interaction loop
while True:
# get input from user
u = input('User: '******'clear' or u == 'reset' or u == 'restart':
self.net.reset_state()
et = EntityTracker()
at = ActionTracker(et)
print('Bot: Reset successfully')
# check for entrance and exit command
elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q':
print("Bot: Thank you for using")
break
elif u == 'hello' or u == 'hi':
print("Bot: Hello, what can i do for you")
elif u == 'thank you' or u == 'thanks' or u == 'thank you very much':
print('Bot: You are welcome')
break
else:
if not u:
continue
u = u.lower()
# encode
u_ent = et.extract_entities(u)
u_ent_features = et.context_features() # 5
# print(et.entities)
# print(et.ctxt_features)
u_emb = self.emb.encode(u) # 300
u_bow = self.bow_enc.encode(u) # 60
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0)
# print(features.shape)
# get action mask
action_mask = at.action_mask()
# action_mask = np.ones(self.net.action_size)
# print("action_mask: ", action_mask)
# forward
prediction = self.net.forward(features, action_mask)
response = self.action_templates[prediction]
if prediction == 0:
slot_values = copy.deepcopy(et.entities)
slot_values.pop('<location>')
memory = []
count = 0
for k, v in slot_values.items():
memory.append('='.join([k, v]))
count += 1
if count == 2:
memory.append('\n')
response = response.replace("memory", ', '.join(memory))
# memory = ', '.join(slot_values.values())
# response = response.replace("memory", memory)
self.net.reset_state()
et = EntityTracker()
at = ActionTracker(et)
# print('Execute successfully and begin new session')
if prediction == 1:
response = response.replace("location", '<location>=' + et.entities['<location>'])
print('Bot: ', response)
class Dialogue():
def __init__(self):
# stor whole dialogues
self.story = []
self.sp_confidecne = []
self.file_path = os.path.join(
rospkg.RosPack().get_path('dialogue_system'), 'log',
'dialogue.txt')
# count turn taking
self.usr_count = 0
self.sys_count = 0
# paramaters
self.network_type = rospy.get_param('~network_model', 'stacked_lstm')
self.lang_type = rospy.get_param('~lang', 'eng')
self.is_emb = rospy.get_param('~embedding', 'false')
self.is_am = rospy.get_param('~action_mask', "true")
self.user_num = rospy.get_param('~user_number', '0')
# call rest of modules
self.et = EntityTracker()
self.at = ActionTracker(self.et)
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed(lang=self.lang_type)
# select observation size for RNN
if self.is_am and self.is_emb:
obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features + self.at.action_size
elif self.is_am and not (self.is_emb):
obs_size = self.bow_enc.vocab_size + self.et.num_features + self.at.action_size
elif not (self.is_am) and self.is_emb:
obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features
elif not (self.is_am) and not (self.is_emb):
obs_size = self.bow_enc.vocab_size + self.et.num_features
self.action_template = self.at.get_action_templates()
self.at.do_display_template()
# must clear entities space
self.et.do_clear_entities()
action_size = self.at.action_size
nb_hidden = 128
if self.network_type == 'gru':
self.net = GRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'reversed_lstm':
self.net = ReversingLSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'reversed_gru':
self.net = ReversingGRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'stacked_gru':
self.net = StackedGRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'stacked_lstm':
self.net = StackedLSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'lstm':
self.net = LSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'bidirectional_lstm':
self.net = BidirectionalLSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'bidirectional_gru':
self.net = BidirectionalGRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
# restore trained model
self.net.restore()
# rostopics
self.pub_reply = rospy.Publisher('reply', Reply, queue_size=10)
self.pub_complete = rospy.Publisher('complete_execute_scenario',
Empty,
queue_size=10)
rospy.Subscriber('raising_events', RaisingEvents,
self.handle_raise_events)
try:
rospy.wait_for_service('reception_db/query_data')
self.get_response_db = rospy.ServiceProxy(
'reception_db/query_data', DBQuery)
rospy.logwarn("waiting for reception DB module...")
except rospy.exceptions.ROSInterruptException as e:
rospy.logerr(e)
quit()
rospy.logwarn(
"network: {}, lang: {}, action_mask: {}, embedding: {}, user_number: {}"
.format(self.network_type, self.lang_type, self.is_am, self.is_emb,
self.user_num))
self.story.append('user number: %s' % self.user_num)
rospy.loginfo('\033[94m[%s]\033[0m initialized.' % rospy.get_name())
# if utterance == 'clear':
# self.net.reset_state()
# self.et.do_clear_entities()
# response = 'context has been cleared.'
def get_response(self, utterance):
rospy.loginfo("actual input: %s" %
utterance) # check actual user input
# clean utterance
# utterance = re.sub(r'[^ a-z A-Z 0-9]', " ", utterance)
# utterance preprocessing
u_ent, u_entities = self.et.extract_entities(utterance, is_test=True)
u_ent_features = self.et.context_features()
u_bow = self.bow_enc.encode(utterance)
if self.is_emb:
u_emb = self.emb.encode(utterance)
try:
if self.is_am:
action_mask = self.at.action_mask()
# concatenated features
if self.is_am and self.is_emb:
features = np.concatenate(
(u_ent_features, u_emb, u_bow, action_mask), axis=0)
elif self.is_am and not (self.is_emb):
features = np.concatenate((u_ent_features, u_bow, action_mask),
axis=0)
elif not (self.is_am) and self.is_emb:
features = np.concatenate((u_ent_features, u_emb, u_bow),
axis=0)
elif not (self.is_am) and not (self.is_emb):
features = np.concatenate((u_ent_features, u_bow), axis=0)
# try:
# predict template number
if self.is_am:
probs, prediction = self.net.forward(features, action_mask)
else:
probs, prediction = self.net.forward(features)
# check response confidence
if max(probs) > BOUNDARY_CONFIDENCE:
response = self.action_template[prediction]
prediction = self.pre_action_process(prediction, u_entities)
# handle api call
if self.post_process(prediction, u_entities):
if prediction == 1:
response = 'api_call appointment {} {} {} {} {} {} {}'.format(
u_entities['<first_name>'],
u_entities['<last_name>'],
u_entities['<address_number>'],
u_entities['<address_name>'],
u_entities['<address_type>'], u_entities['<time>'],
u_entities['<pm_am>'])
elif prediction == 2:
response = 'api_call location {}'.format(
u_entities['<location>'])
elif prediction == 3:
response = 'api_call prescription {} {} {} {} {}'.format(
u_entities['<first_name>'],
u_entities['<last_name>'],
u_entities['<address_number>'],
u_entities['<address_name>'],
u_entities['<address_type>'])
elif prediction == 4:
response = 'api_call waiting_time {} {} {} {} {} {} {}'.format(
u_entities['<first_name>'],
u_entities['<last_name>'],
u_entities['<address_number>'],
u_entities['<address_name>'],
u_entities['<address_type>'], u_entities['<time>'],
u_entities['<pm_am>'])
response = self.get_response_db(
response
) # query knowledge base; here we use dynamo db
response = response.response
elif prediction in [6, 9, 11]:
response = self.action_template[prediction]
response = response.split(' ')
response = [
word.replace('<first_name>',
u_entities['<first_name>'])
for word in response
]
response = ' '.join(response)
else:
response = self.action_template[prediction]
else:
response = random.choice(
REPROMPT
) # if prediction confidence less than 40%, reprompt
except:
response = random.choice(REPROMPT)
return prediction, probs, response
def handle_raise_events(self, msg):
utterance = msg.recognized_word
try:
# get confidence
data = json.loads(msg.data[0])
confidence = data['confidence']
except:
confidence = None
if confidence > BOUNDARY_CONFIDENCE or confidence == None:
if 'silency_detected' in msg.events:
utterance = '<SILENCE>'
else:
try:
self.story.append(
"U%i: %s (sp_conf:%f)" %
(self.usr_count + 1, utterance, confidence))
self.sp_confidecne.append(confidence)
except:
self.story.append("U%i: %s" %
(self.usr_count + 1, utterance))
self.usr_count += 1
utterance = utterance.lower()
# generate system response
prediction, probs, response = self.get_response(utterance)
else:
prediction = -1
probs = -1
response = random.choice(REPROMPT)
# add system turn
self.story.append("A%i: %s" % (self.sys_count + 1, response))
self.sys_count += 1
# finish interaction
if (prediction == 6):
self.pub_complete.publish()
# logging user and system turn
self.story.append("user: %i, system: %i" %
(self.usr_count, self.sys_count))
self.story.append("mean_sp_conf: %f" %
(reduce(lambda x, y: x + y, self.sp_confidecne) /
len(self.sp_confidecne)))
self.story.append(
'==================================================================='
)
self.write_file(self.file_path, self.story)
# display system response
rospy.loginfo(json.dumps(self.et.entities,
indent=2)) # recognized entity values
try:
rospy.logwarn("System: [conf: %f, predict: %d] / %s\n" %
(max(probs), prediction, response))
except:
rospy.logwarn("System: [] / %s\n" % (response))
reply_msg = Reply()
reply_msg.header.stamp = rospy.Time.now()
reply_msg.reply = response
self.pub_reply.publish(reply_msg)
def post_process(self, prediction, u_ent_features):
api_call_list = [1, 2, 3, 4]
if prediction in api_call_list:
return True
attr_list = [0, 9, 10, 11, 12]
if all(u_ent_featur == 1
for u_ent_featur in u_ent_features) and prediction in attr_list:
return True
else:
return False
def action_post_process(self, prediction, u_entities):
attr_mapping_dict = {
11: '<first_name>',
11: '<last_name>',
12: '<address_number>',
12: '<address_name>',
12: '<address_type>',
10: '<time>',
10: '<pm_am>',
}
# find exist and non-exist entity
exist_ent_index = [
key for key, value in u_entities.items() if value != None
]
non_exist_ent_index = [
key for key, value in u_entities.items() if value == None
]
# if predicted key is already in exist entity index then find non exist entity index
# and leads the user to input non exist entity.
if prediction in attr_mapping_dict:
pred_key = attr_mapping_dict[prediction]
if pred_key in exist_ent_index:
for key, value in attr_mapping_dict.items():
if value == non_exist_ent_index[0]:
return key
else:
return prediction
else:
return prediction
def pre_action_process(self, prediction, u_entities):
api_call_list = [1, 3, 4]
attr_mapping_dict = {
'<first_name>': 11,
'<last_name>': 11,
'<address_number>': 12,
'<address_name>': 12,
'<address_type>': 12,
'<time>': 10,
'<pm_am>': 10,
}
# find exist and non-exist entity
non_exist_ent_index = [
key for key, value in u_entities.items() if value == None
]
if prediction in api_call_list:
if '<first_name>' in non_exist_ent_index:
prediction = attr_mapping_dict['<first_name>']
return prediction
'''
writing story log file
'''
def write_file(self, path, story_list):
with open(path, 'a') as f:
for item in story_list:
f.write("%s\n" % item)
rospy.logwarn('save dialogue histories.')
def __init__(self):
# stor whole dialogues
self.story = []
self.sp_confidecne = []
self.file_path = os.path.join(
rospkg.RosPack().get_path('dialogue_system'), 'log',
'dialogue.txt')
# count turn taking
self.usr_count = 0
self.sys_count = 0
# paramaters
self.network_type = rospy.get_param('~network_model', 'stacked_lstm')
self.lang_type = rospy.get_param('~lang', 'eng')
self.is_emb = rospy.get_param('~embedding', 'false')
self.is_am = rospy.get_param('~action_mask', "true")
self.user_num = rospy.get_param('~user_number', '0')
# call rest of modules
self.et = EntityTracker()
self.at = ActionTracker(self.et)
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed(lang=self.lang_type)
# select observation size for RNN
if self.is_am and self.is_emb:
obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features + self.at.action_size
elif self.is_am and not (self.is_emb):
obs_size = self.bow_enc.vocab_size + self.et.num_features + self.at.action_size
elif not (self.is_am) and self.is_emb:
obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features
elif not (self.is_am) and not (self.is_emb):
obs_size = self.bow_enc.vocab_size + self.et.num_features
self.action_template = self.at.get_action_templates()
self.at.do_display_template()
# must clear entities space
self.et.do_clear_entities()
action_size = self.at.action_size
nb_hidden = 128
if self.network_type == 'gru':
self.net = GRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'reversed_lstm':
self.net = ReversingLSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'reversed_gru':
self.net = ReversingGRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'stacked_gru':
self.net = StackedGRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'stacked_lstm':
self.net = StackedLSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'lstm':
self.net = LSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'bidirectional_lstm':
self.net = BidirectionalLSTM(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
elif self.network_type == 'bidirectional_gru':
self.net = BidirectionalGRU(obs_size=obs_size,
nb_hidden=nb_hidden,
action_size=action_size,
lang=self.lang_type,
is_action_mask=self.is_am)
# restore trained model
self.net.restore()
# rostopics
self.pub_reply = rospy.Publisher('reply', Reply, queue_size=10)
self.pub_complete = rospy.Publisher('complete_execute_scenario',
Empty,
queue_size=10)
rospy.Subscriber('raising_events', RaisingEvents,
self.handle_raise_events)
try:
rospy.wait_for_service('reception_db/query_data')
self.get_response_db = rospy.ServiceProxy(
'reception_db/query_data', DBQuery)
rospy.logwarn("waiting for reception DB module...")
except rospy.exceptions.ROSInterruptException as e:
rospy.logerr(e)
quit()
rospy.logwarn(
"network: {}, lang: {}, action_mask: {}, embedding: {}, user_number: {}"
.format(self.network_type, self.lang_type, self.is_am, self.is_emb,
self.user_num))
self.story.append('user number: %s' % self.user_num)
rospy.loginfo('\033[94m[%s]\033[0m initialized.' % rospy.get_name())
def main(in_clean_dataset_folder, in_noisy_dataset_folder, in_model_folder,
in_no_ood_evaluation):
rev_vocab, kb, action_templates, config = load_model(in_model_folder)
clean_dialogs, clean_indices = read_dialogs(os.path.join(
in_clean_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'),
with_indices=True)
noisy_dialogs, noisy_indices = read_dialogs(os.path.join(
in_noisy_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'),
with_indices=True)
post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns(
noisy_dialogs)
assert len(post_ood_turns_clean) == len(post_ood_turns_noisy)
for post_ood_turn_clean, post_ood_turn_noisy in zip(
sorted(post_ood_turns_clean), sorted(post_ood_turns_noisy)):
noisy_dialogs[post_ood_turn_noisy][0] = clean_dialogs[
post_ood_turn_clean][0]
et = EntityTracker(kb)
at = ActionTracker(None, et)
at.set_action_templates(action_templates)
vocab = {word: idx for idx, word in enumerate(rev_vocab)}
X_clean, context_features_clean, action_masks_clean, y_clean = make_dataset_for_hierarchical_lstm(
clean_dialogs, clean_indices, vocab, et, at, **config)
X_noisy, context_features_noisy, action_masks_noisy, y_noisy = make_dataset_for_hierarchical_lstm(
noisy_dialogs, noisy_indices, vocab, et, at, **config)
net = HierarchicalLSTM(config, context_features_clean.shape[-1],
action_masks_clean.shape[-1])
net.restore(in_model_folder)
eval_stats_clean = evaluate_advanced(
net, (X_clean, context_features_clean, action_masks_clean, y_clean),
at.action_templates,
post_ood_turns=post_ood_turns_clean)
print('Clean dataset: {} turns overall'.format(
eval_stats_clean['total_turns']))
print('Accuracy:')
accuracy = eval_stats_clean['correct_turns'] / eval_stats_clean[
'total_turns']
accuracy_post_ood = eval_stats_clean['correct_post_ood_turns'] / eval_stats_clean['total_post_ood_turns'] \
if eval_stats_clean['total_post_ood_turns'] != 0 \
else 0
print('overall: {:.3f}; directly post-OOD: {:.3f}'.format(
accuracy, accuracy_post_ood))
print('Loss : {:.3f}'.format(eval_stats_clean['avg_loss']))
eval_stats_noisy = evaluate_advanced(
net, (X_noisy, context_features_noisy, action_masks_noisy, y_noisy),
at.action_templates,
post_ood_turns=post_ood_turns_noisy)
print('\n\n')
print(
'Noisy dataset: {} turns overall, {} turns after the first OOD'.format(
eval_stats_noisy['total_turns'],
eval_stats_noisy['total_turns_after_ood']))
print('Accuracy:')
accuracy = eval_stats_noisy['correct_turns'] / eval_stats_noisy[
'total_turns']
accuracy_after_ood = eval_stats_noisy['correct_turns_after_ood'] / eval_stats_noisy['total_turns_after_ood'] \
if eval_stats_noisy['total_turns_after_ood'] != 0 \
else 0
accuracy_post_ood = eval_stats_noisy['correct_post_ood_turns'] / eval_stats_noisy['total_post_ood_turns'] \
if eval_stats_noisy['total_post_ood_turns'] != 0 \
else 0
print(
'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'.
format(accuracy, accuracy_after_ood, accuracy_post_ood))
print('Loss : {:.3f}'.format(eval_stats_noisy['avg_loss']))
if in_no_ood_evaluation:
eval_stats_no_ood = evaluate_advanced(
net,
(X_noisy, context_features_noisy, action_masks_noisy, y_noisy),
at.action_templates,
post_ood_turns=post_ood_turns_noisy,
ignore_ood_accuracy=True)
print('Accuracy (OOD turns ignored):')
accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood[
'total_turns']
accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \
if eval_stats_no_ood['total_turns_after_ood'] != 0 \
else 0
accuracy_post_ood = eval_stats_no_ood['correct_post_ood_turns'] / eval_stats_no_ood['total_post_ood_turns'] \
if eval_stats_no_ood['total_post_ood_turns'] != 0 \
else 0
print(
'overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'
.format(accuracy, accuracy_after_ood, accuracy_post_ood))
def main(in_clean_dataset_folder, in_noisy_dataset_folder, in_model_folder, in_mode, in_runs_number):
rev_vocab, kb, action_templates, config = load_model(in_model_folder)
clean_dialogs, clean_indices = read_dialogs(os.path.join(in_clean_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'),
with_indices=True)
noisy_dialogs, noisy_indices = read_dialogs(os.path.join(in_noisy_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'),
with_indices=True)
max_noisy_dialog_length = max([item['end'] - item['start'] + 1 for item in noisy_indices])
config['max_input_length'] = max_noisy_dialog_length
post_ood_turns_clean, post_ood_turns_noisy = mark_post_ood_turns(noisy_dialogs, BABI_CONFIG['backoff_utterance'].lower())
et = EntityTracker(kb)
at = ActionTracker(None, et)
at.set_action_templates(action_templates)
vocab = {word: idx for idx, word in enumerate(rev_vocab)}
data_clean = make_dataset_for_vhcn_v2(clean_dialogs, clean_indices, vocab, et, at, **config)
data_noisy = make_dataset_for_vhcn_v2(noisy_dialogs, noisy_indices, vocab, et, at, **config)
context_features_clean, action_masks_clean = data_clean[2:4]
net = VariationalHierarchicalLSTMv3(rev_vocab, config, context_features_clean.shape[-1], action_masks_clean.shape[-1])
net.restore(in_model_folder)
if in_mode == 'clean':
eval_stats_clean = evaluate_advanced(net,
data_clean,
at.action_templates,
BABI_CONFIG['backoff_utterance'].lower(),
post_ood_turns=post_ood_turns_clean,
runs_number=in_runs_number)
print('Clean dataset: {} turns overall'.format(eval_stats_clean['total_turns']))
print('Accuracy:')
accuracy = eval_stats_clean['correct_turns'] / eval_stats_clean['total_turns']
accuracy_continuous = eval_stats_clean['correct_continuous_turns'] / eval_stats_clean['total_turns']
accuracy_post_ood = eval_stats_clean['correct_post_ood_turns'] / eval_stats_clean['total_post_ood_turns'] \
if eval_stats_clean['total_post_ood_turns'] != 0 \
else 0
print('overall: {:.3f}; continuous: {:.3f}; directly post-OOD: {:.3f}'.format(accuracy, accuracy_continuous, accuracy_post_ood))
print('Loss : {:.3f}'.format(eval_stats_clean['avg_loss']))
elif in_mode == 'noisy':
eval_stats_noisy = evaluate_advanced(net,
data_noisy,
at.action_templates,
BABI_CONFIG['backoff_utterance'].lower(),
post_ood_turns=post_ood_turns_noisy,
runs_number=in_runs_number)
print('\n\n')
print('Noisy dataset: {} turns overall'.format(eval_stats_noisy['total_turns']))
print('Accuracy:')
accuracy = eval_stats_noisy['correct_turns'] / eval_stats_noisy['total_turns']
accuracy_continuous = eval_stats_noisy['correct_continuous_turns'] / eval_stats_noisy['total_turns']
accuracy_post_ood = eval_stats_noisy['correct_post_ood_turns'] / eval_stats_noisy['total_post_ood_turns'] \
if eval_stats_noisy['total_post_ood_turns'] != 0 \
else 0
accuracy_ood = eval_stats_noisy['correct_ood_turns'] / eval_stats_noisy['total_ood_turns'] \
if eval_stats_noisy['total_ood_turns'] != 0 \
else 0
print('overall: {:.3f}; continuous: {:.3f}; directly post-OOD: {:.3f}; OOD: {:.3f}'.format(accuracy,
accuracy_continuous,
accuracy_post_ood,
accuracy_ood))
print('Loss : {:.3f}'.format(eval_stats_noisy['avg_loss']))
elif in_mode == 'noisy_ignore_ood':
eval_stats_no_ood = evaluate_advanced(net,
data_noisy,
at.action_templates,
BABI_CONFIG['backoff_utterance'].lower(),
post_ood_turns=post_ood_turns_noisy,
ignore_ood_accuracy=True,
runs_number=in_runs_number)
print('Accuracy (OOD turns ignored):')
accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood['total_turns']
accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \
if eval_stats_no_ood['total_turns_after_ood'] != 0 \
else 0
accuracy_post_ood = eval_stats_no_ood['correct_post_ood_turns'] / eval_stats_no_ood['total_post_ood_turns'] \
if eval_stats_no_ood['total_post_ood_turns'] != 0 \
else 0
print('overall: {:.3f}; after first OOD: {:.3f}, directly post-OOD: {:.3f}'.format(accuracy, accuracy_after_ood, accuracy_post_ood))
class InteractiveSession:
def __init__(self):
self.et = EntityTracker()
self.at = ActionTracker(self.et)
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed()
obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features
self.action_templates = self.at.get_action_templates()
action_size = self.at.action_size
nb_hidden = 128
self.net = LSTM_net(obs_size=obs_size,
action_size=action_size,
nb_hidden=nb_hidden)
# restore checkpoint
self.net.restore()
self.net.reset_state()
def reset(self):
self.net.reset_state()
self.et = EntityTracker()
self.at = ActionTracker(self.et)
def interact(self, utterance, intent, slot_values):
# get input from user
u = utterance.lower()
# check if user wants to begin new session
if u == 'clear' or u == 'reset' or u == 'restart':
self.reset()
return "reset successfully"
# check for entrance and exit command
elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q':
self.reset()
return "Thank you for using"
elif u == 'hello' or u == 'hi':
self.reset()
return "what can i do for you"
elif u == 'thank you' or u == 'thanks' or u == 'thank you very much':
self.reset()
return 'you are welcome'
else:
# encode
u_ent = self.et.extract_entities(u, intent, slot_values)
u_ent_features = self.et.context_features() # 5
u_emb = self.emb.encode(u) # 300
u_bow = self.bow_enc.encode(u) # 60
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0)
# get action mask
action_mask = self.at.action_mask()
# action_mask = np.ones(self.net.action_size)
# forward
prediction = self.net.forward(features, action_mask)
response = self.action_templates[prediction]
if prediction == 0:
slot_values = copy.deepcopy(self.et.entities)
slot_values.pop('location')
memory = ', '.join(slot_values.values())
response = response.replace("memory", memory)
self.reset()
print('API CALL execute successfully and begin new session')
if prediction == 1:
response = response.replace("location",
self.et.entities['location'])
return response
