def main(in_dataset_folder, in_model_folder, in_no_ood_evaluation):
rev_vocab, kb, action_templates, config = load_model(in_model_folder)
test_dialogs, test_indices = read_dialogs(os.path.join(
in_dataset_folder, 'dialog-babi-task6-dstc2-tst.txt'),
with_indices=True)
et = EntityTracker(kb)
at = ActionTracker(None, et)
at.set_action_templates(action_templates)
vocab = {word: idx for idx, word in enumerate(rev_vocab)}
X, action_masks, sequence_masks, y = make_dataset(
test_dialogs, test_indices, vocab, et, at, config['max_input_length'])
net = LSTM_net(config, X.shape[-1], action_masks.shape[-1])
net.restore(in_model_folder)
eval_stats_full_dataset = evaluate_advanced(
net, (X, action_masks, sequence_masks, y), test_indices,
at.action_templates)
print(
'Full dataset: {} turns overall, {} turns after the first OOD'.format(
eval_stats_full_dataset['total_turns'],
eval_stats_full_dataset['total_turns_after_ood']))
print('Accuracy:')
accuracy = eval_stats_full_dataset[
'correct_turns'] / eval_stats_full_dataset['total_turns']
accuracy_after_ood = eval_stats_full_dataset['correct_turns_after_ood'] / eval_stats_full_dataset['total_turns_after_ood'] \
if eval_stats_full_dataset['total_turns_after_ood'] != 0 \
else 0
print('overall: {:.3f}; after first OOD: {:.3f}'.format(
accuracy, accuracy_after_ood))
print('Loss : {:.3f}'.format(eval_stats_full_dataset['avg_loss']))
if in_no_ood_evaluation:
eval_stats_no_ood = evaluate_advanced(
net, (X, action_masks, sequence_masks, y),
test_indices,
at.action_templates,
ignore_ood_accuracy=True)
print('Accuracy (OOD turns ignored):')
accuracy = eval_stats_no_ood['correct_turns'] / eval_stats_no_ood[
'total_turns']
accuracy_after_ood = eval_stats_no_ood['correct_turns_after_ood'] / eval_stats_no_ood['total_turns_after_ood'] \
if eval_stats_no_ood['total_turns_after_ood'] != 0 \
else 0
print('overall: {:.3f}; after first OOD: {:.3f}'.format(
accuracy, accuracy_after_ood))
print('Loss : {:.3f}'.format(eval_stats_no_ood['avg_loss']))
class InteractiveSession:
def __init__(self):
et = EntityTracker()
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed()
at = ActionTracker(et)
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features
self.action_templates = at.get_action_templates()
action_size = at.action_size
nb_hidden = 128
self.net = LSTM_net(obs_size=obs_size, action_size=action_size, nb_hidden=nb_hidden)
# restore checkpoint
self.net.restore()
def interact(self):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
# begin interaction loop
while True:
# get input from user
u = input('User: '******'clear' or u == 'reset' or u == 'restart':
self.net.reset_state()
et = EntityTracker()
at = ActionTracker(et)
print('Bot: Reset successfully')
# check for entrance and exit command
elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q':
print("Bot: Thank you for using")
break
elif u == 'hello' or u == 'hi':
print("Bot: Hello, what can i do for you")
elif u == 'thank you' or u == 'thanks' or u == 'thank you very much':
print('Bot: You are welcome')
break
else:
if not u:
continue
u = u.lower()
# encode
u_ent = et.extract_entities(u)
u_ent_features = et.context_features() # 5
# print(et.entities)
# print(et.ctxt_features)
u_emb = self.emb.encode(u) # 300
u_bow = self.bow_enc.encode(u) # 60
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0)
# print(features.shape)
# get action mask
action_mask = at.action_mask()
# action_mask = np.ones(self.net.action_size)
# print("action_mask: ", action_mask)
# forward
prediction = self.net.forward(features, action_mask)
response = self.action_templates[prediction]
if prediction == 0:
slot_values = copy.deepcopy(et.entities)
slot_values.pop('<location>')
memory = []
count = 0
for k, v in slot_values.items():
memory.append('='.join([k, v]))
count += 1
if count == 2:
memory.append('\n')
response = response.replace("memory", ', '.join(memory))
# memory = ', '.join(slot_values.values())
# response = response.replace("memory", memory)
self.net.reset_state()
et = EntityTracker()
at = ActionTracker(et)
# print('Execute successfully and begin new session')
if prediction == 1:
response = response.replace("location", '<location>=' + et.entities['<location>'])
print('Bot: ', response)
class InteractiveSession():
def __init__(self):
et = EntityTracker()
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed()
at = ActionTracker(et)
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features
self.action_templates = at.get_action_templates()
action_size = at.action_size
nb_hidden = 128
self.net = LSTM_net(obs_size=obs_size,
action_size=action_size,
nb_hidden=nb_hidden)
# restore checkpoint
self.net.restore()
def interact(self):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
# begin interaction loop
while True:
# get input from user
u = input(':: ')
# check if user wants to begin new session
if u == 'clear' or u == 'reset' or u == 'restart':
self.net.reset_state()
et = EntityTracker()
at = ActionTracker(et)
print('')
# check for exit command
elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q':
break
else:
# ENTER press : silence
if not u:
u = '<SILENCE>'
# encode
u_ent, u_entities = et.extract_entities(u, is_test=True)
u_ent_features = et.context_features()
u_emb = self.emb.encode(u)
u_bow = self.bow_enc.encode(u)
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow),
axis=0)
# get action mask
action_mask = at.action_mask()
# forward
prediction = self.net.forward(features, action_mask)
print('prediction : ', prediction)
print(u_entities)
print('\n')
if self.post_process(prediction, u_ent_features):
print(
'>>', 'api_call ' + u_entities['<cuisine>'] + ' ' +
u_entities['<location>'] + ' ' +
u_entities['<party_size>'] + ' ' +
u_entities['<rest_type>'])
else:
prediction = self.action_post_process(
prediction, u_entities)
print('>>', self.action_templates[prediction])
# if all entities is satisfied and the user agree to make a reservation.
if all(u_ent_featur == 1
for u_ent_featur in u_ent_features) and (prediction
== 10):
break
def post_process(self, prediction, u_ent_features):
if prediction == 0:
return True
attr_list = [9, 12, 6, 1]
if all(u_ent_featur == 1
for u_ent_featur in u_ent_features) and prediction in attr_list:
return True
else:
return False
def action_post_process(self, prediction, u_entities):
attr_mapping_dict = {
9: '<cuisine>',
12: '<location>',
6: '<party_size>',
1: '<rest_type>'
}
# find exist and non-exist entity
exist_ent_index = [
key for key, value in u_entities.items() if value != None
]
non_exist_ent_index = [
key for key, value in u_entities.items() if value == None
]
# if predicted key is already in exist entity index then find non exist entity index
# and leads the user to input non exist entity.
if prediction in attr_mapping_dict:
pred_key = attr_mapping_dict[prediction]
if pred_key in exist_ent_index:
for key, value in attr_mapping_dict.items():
if value == non_exist_ent_index[0]:
return key
else:
return prediction
else:
return prediction
class InteractiveSession():
def __init__(self):
et = EntityTracker()
self.bow_enc = BoW_encoder(et)
self.emb = UtteranceEmbed()
at = ActionTracker(et)
obs_size = self.emb.dim + self.bow_enc.vocab_size + et.num_features
self.action_templates = at.get_action_templates()
action_size = at.action_size
nb_hidden = 128
self.net = LSTM_net(obs_size=obs_size,
action_size=action_size,
nb_hidden=nb_hidden)
# restore checkpoint
self.net.restore()
def interact(self, input):
# create entity tracker
et = EntityTracker()
# create action tracker
at = ActionTracker(et)
# reset network
self.net.reset_state()
# begin interaction loop
#while True:
# get input from user
#u = input(':: ')
u = input
# check if user wants to begin new session
if u == 'clear' or u == 'reset' or u == 'restart':
self.net.reset_state()
et = EntityTracker()
at = ActionTracker(et)
print('')
# check for exit command
#elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q':
#break
else:
# ENTER press : silence
if not u:
u = '<SILENCE>'
# encode
u_ent = et.extract_entities(u)
u_ent_features = et.context_features()
u_emb = self.emb.encode(u)
u_bow = self.bow_enc.encode(u)
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0)
# get action mask
action_mask = at.action_mask()
# forward
prediction = self.net.forward(features, action_mask)
#print('>>', self.action_templates[prediction])
return self.action_templates[prediction]
class InteractiveSession:
def __init__(self):
self.et = EntityTracker()
self.at = ActionTracker(self.et)
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed()
obs_size = self.emb.dim + self.bow_enc.vocab_size + self.et.num_features
self.action_templates = self.at.get_action_templates()
action_size = self.at.action_size
nb_hidden = 128
self.net = LSTM_net(obs_size=obs_size,
action_size=action_size,
nb_hidden=nb_hidden)
# restore checkpoint
self.net.restore()
self.net.reset_state()
def reset(self):
self.net.reset_state()
self.et = EntityTracker()
self.at = ActionTracker(self.et)
def interact(self, utterance, intent, slot_values):
# get input from user
u = utterance.lower()
# check if user wants to begin new session
if u == 'clear' or u == 'reset' or u == 'restart':
self.reset()
return "reset successfully"
# check for entrance and exit command
elif u == 'exit' or u == 'stop' or u == 'quit' or u == 'q':
self.reset()
return "Thank you for using"
elif u == 'hello' or u == 'hi':
self.reset()
return "what can i do for you"
elif u == 'thank you' or u == 'thanks' or u == 'thank you very much':
self.reset()
return 'you are welcome'
else:
# encode
u_ent = self.et.extract_entities(u, intent, slot_values)
u_ent_features = self.et.context_features() # 5
u_emb = self.emb.encode(u) # 300
u_bow = self.bow_enc.encode(u) # 60
# concat features
features = np.concatenate((u_ent_features, u_emb, u_bow), axis=0)
# get action mask
action_mask = self.at.action_mask()
# action_mask = np.ones(self.net.action_size)
# forward
prediction = self.net.forward(features, action_mask)
response = self.action_templates[prediction]
if prediction == 0:
slot_values = copy.deepcopy(self.et.entities)
slot_values.pop('location')
memory = ', '.join(slot_values.values())
response = response.replace("memory", memory)
self.reset()
print('API CALL execute successfully and begin new session')
if prediction == 1:
response = response.replace("location",
self.et.entities['location'])
return response
class InteractiveSession():
def __init__(self):
self.bow_enc = BoW_encoder()
self.emb = UtteranceEmbed()
self.at = ActionTracker(interact=True)
print(self.at.dialog_acts)
_, _, self.other_da, self.other_da_id = self.at.get_data()
self.action_templates = self.at.action_templates
self.action_size = self.at.action_size
#self.responses = [r[0] for r in self.at.responses]
#print(self.emb.dim , self.bow_enc.vocab_size , self.action_size)
self.obs_size = self.emb.dim + self.bow_enc.vocab_size + self.action_size
nb_hidden = 128
self.exercises = list(self.at.action_templates.keys())
self.net = LSTM_net(obs_size=self.obs_size,
action_size=self.action_size,
nb_hidden=nb_hidden)
# restore checkpoint
self.net.restore()
# setup remote interaction via zeromq
context = zmq.Context.instance()
# PUB socket for sending system-output
self.pubSocket: zmq.Socket = context.socket(zmq.PUB)
self.pubSocket.bind(Config.SYSTEM_OUTPUT_ADDR)
# SUB socket for receiving user-input
self.subSocket: zmq.Socket = context.socket(zmq.SUB)
self.subSocket.connect(Config.USER_INPUT_ADDR)
self.subSocket.setsockopt_string(zmq.SUBSCRIBE,
Config.USER_INPUT_SUBJECT)
self.lastOutputMessage: str = None
self.requestHandler: threading.Thread = None
self.requestInterrupted: threading.Lock = None
def has_exercise(self, exercise_name: str) -> bool:
return exercise_name in self.at.action_templates
def get_current_exercise(self) -> str:
return self.at.exercise
def send(self, message, utteranceType, state, dialogAct):
print(">>", message)
data = SystemOutputData(text=message,
utterance_type=utteranceType,
state=state,
dialogue_act=dialogAct)
msg = SystemOutputMessage(type=MessageType.OUTPUT,
timestamp=int(time.time() * 1000),
data=data)
jsonMsg = json.dumps(msg.to_dict())
self.lastOutputMessage = jsonMsg
rawMsg = "%s %s" % (to_enum(MessageType, msg.type), jsonMsg)
# print(" ", rawMsg)#DEBUG
self.pubSocket.send_string(rawMsg)
def receive(self) -> 'UserInputMessage':
rawMsg = self.subSocket.recv_string()
# print(" ", rawMsg)#DEBUG
# FIXME detect & handle when message has other type than "input"
# TODO handle message types other than "input"
msgTypeStr = to_enum(MessageType, MessageType.INPUT)
# extract JSON message by removing subject/subscription string from beginning of message:
msgStr = rawMsg.replace(msgTypeStr, "", 1).strip()
return user_input_message_from_dict(json.loads(msgStr))
def start_processing_requests(self, verbose: bool = False):
self.stop_processing_requests()
self.requestInterrupted = threading.Lock()
self.requestInterrupted.acquire()
self.requestHandler = threading.Thread(
target=request_worker,
args=(Config.QUERY_SYSTEM_OUTPUT_ADDR, self,
self.requestInterrupted, verbose))
self.requestHandler.start()
def stop_processing_requests(self):
if self.requestInterrupted is not None and self.requestHandler is not None and self.requestHandler.isAlive(
):
self.requestInterrupted.release()
# print("should stop request_worker thread now!")
self.requestHandler.join()
print("stopped request_worker thread.")
def interact_anew(self,
verbose,
update,
out_file,
new_state_name: str = None):
prev_prediction = -1
self.net.reset_state()
if new_state_name is None:
new_state_name = random.choice(self.exercises)
self.at.set_exercise_string(new_state_name)
self.at.memory.reset_memory(self.at.exercise)
self.at.initialize_am()
u = "start"
if verbose == True:
print("\n*******************\nÄUßERUNGSTYP: ",
self.at.automaton.get_utterance_type(u))
action_mask = self.at.am
u_emb = self.emb.encode(u) # HIER
u_bow = self.bow_enc.encode(u) # HIER
features = np.concatenate((u_emb, u_bow, action_mask), axis=0)
# forward
prediction = self.net.forward(features,
action_mask,
self.other_da,
print_top=True)
self.at.automaton.current_state, response = self.at.walk(u, prediction)
prev_prediction = prediction
if verbose == True:
print("AKTUELLER ZUSTAND: ", self.at.automaton.current_state.name,
"\nDIALOGAKT: ", self.other_da[prediction],
"\n*******************\n"
) # print("CURRENT Exercise: ", self.at.exercise)
# self.net.train_step(features, prediction, action_mask)
# response = self.at.get_text(prediction)
self.send(response, self.at.automaton.get_utterance_type(u),
self.at.automaton.current_state.name,
self.at.dialog_acts[prediction]) # print('>>', response)
if out_file:
out_file.write(u + '\t' + response + '\t' +
self.other_da[prediction] + '\n')
return prev_prediction
def interact(self, verbose, update, write_to_file):
print(
"How do you do? I'm glad you're here. Let's do some exercises together."
)
# open file to write around here
# only if write_to_file !
with open('data/dialogs_from_interaction', 'a') as out:
with open("data/utterances_from_interaction", "a") as uttout:
self.start_processing_requests(verbose)
out_file = None
if write_to_file:
out_file = out
prev_prediction = self.interact_anew(verbose, update, out_file)
# begin interaction loop
while True:
if verbose == True:
print([
self.other_da[i] for i in range(len(self.at.am))
if self.at.am[i] == 1
])
#print("prediction: ",self.at.dialog_acts[prediction])
#print("response: ",response)
# get input from user; utterance isn't saved!
message = self.receive() # u = input(':: ')
u = message.data.strip()
print(":: %s" % (u))
# check if user wants to begin new session
if u == 'clear' or u == 'reset' or u == 'restart' or u == 'von vorne' or u == 'neustart':
if write_to_file:
out.write("WARNING: Interaction was reset.\n\n")
prev_prediction = self.interact_anew(
verbose, update, out_file)
if write_to_file:
uttout.write(
u + "\t" +
self.at.automaton.get_utterance_type(u) + "\n")
elif self.other_da[prev_prediction] in [
"closing", "finish"
]:
if write_to_file:
out.write('\n')
prev_prediction = self.interact_anew(
verbose, update, out_file)
# check for exit command
elif u == 'exit' or u == 'quit':
if update == True:
self.net.save()
self.send(
'Ok, wir brechen ab. Bis zum nächsten Mal!',
'EXIT COMMAND',
self.at.automaton.current_state.name, "UNKNOWN"
) # print('>> Ok, wir brechen ab. Bis zum nächsten Mal!')
if write_to_file:
out.write("WARNING: Dialog may not be finished.\n")
break
else:
if u == "\n" or u == "":
u = '<SILENCE>'
#self.at.reset_action_mask()
if self.at.automaton.current_state.name == "all_one":
if verbose == True:
print("\n*******************\nÄUßERUNGSTYP: ",
self.at.automaton.get_utterance_type(u))
##print(self.at.automaton.current_state.name, self.at.dialog_acts[prev_prediction])
else:
if verbose == True:
print("\n*******************\nÄUßERUNGSTYP: ",
self.at.automaton.get_utterance_type(u))
action_mask = self.at.am
u_emb = self.emb.encode(u) #HIER
u_bow = self.bow_enc.encode(u) #HIER
features = np.concatenate((u_emb, u_bow, action_mask),
axis=0)
# forward
prediction = self.net.forward(features,
action_mask,
self.other_da,
print_top=True)
self.at.automaton.current_state, response = self.at.walk(
u, prediction)
if prediction != "all_one":
prev_prediction = prediction
#self.at.memory.update(prediction)
if verbose == True:
print(
"AKTUELLER ZUSTAND: ",
self.at.automaton.current_state.name,
"\nDIALOGAKT: ", self.other_da[prediction],
"\n*******************\n"
) #print("CURRENT Exercise: ", self.at.exercise)
self.net.train_step(features, prediction, action_mask)
self.send(response,
self.at.automaton.get_utterance_type(u),
self.at.automaton.current_state.name,
self.at.dialog_acts[prediction]
) # print('>>', response)
if write_to_file:
out.write(u + '\t' + response + '\t' +
self.other_da[prediction] + '\n')
uttout.write(
u + "\t" +
self.at.automaton.get_utterance_type(u) + "\n")
if write_to_file:
out.write('\n')
self.stop_processing_requests()