def createDelexData():
"""Main function of the script - loads delexical dictionary,
goes through each dialogue and does:
1) data normalization
2) delexicalization
3) addition of database pointer
4) saves the delexicalized data
"""
# download the data
loadData()
# create dictionary of delexicalied values that then we will search against, order matters here!
dic = delexicalize.prepareSlotValuesIndependent()
delex_data = {}
# fin1 = file('data/multi-woz/data.json')
fin1 = open('data/multi-woz/data.json')
data = json.load(fin1)
# fin2 = file('data/multi-woz/dialogue_acts.json')
fin2 = open('data/multi-woz/dialogue_acts.json')
data2 = json.load(fin2)
num = 0
for dialogue_name in tqdm(data):
dialogue = data[dialogue_name]
#print dialogue_name
idx_acts = 1
for idx, turn in enumerate(dialogue['log']):
# normalization, split and delexicalization of the sentence
sent = normalize(turn['text'])
words = sent.split()
sent = delexicalize.delexicalise(' '.join(words), dic)
# parsing reference number GIVEN belief state
sent = delexicaliseReferenceNumber(sent, turn)
# changes to numbers only here
digitpat = re.compile('\d+')
sent = re.sub(digitpat, '[value_count]', sent)
# delexicalized sentence added to the dialogue
dialogue['log'][idx]['text'] = sent
if idx % 2 == 1: # if it's a system turn
# add database pointer
pointer_vector = addDBPointer(turn)
# add booking pointer
pointer_vector = addBookingPointer(dialogue, turn,
pointer_vector)
#print pointer_vector
dialogue['log'][idx -
1]['db_pointer'] = pointer_vector.tolist()
# FIXING delexicalization:
dialogue = fixDelex(dialogue_name, dialogue, data2, idx, idx_acts)
idx_acts += 1
delex_data[dialogue_name] = dialogue
# num += 1
# if num > 100:
# break
with open('data/multi-woz/delex.json', 'w') as outfile:
json.dump(delex_data, outfile)
return delex_data
def predict(model, prev_state, prev_active_domain, state, dic):
start_time = time.time()
model.beam_search = False
input_tensor = []
bs_tensor = []
db_tensor = []
usr = state['history'][-1][-1]
prev_state = deepcopy(prev_state['belief_state'])
state = deepcopy(state['belief_state'])
mark_not_mentioned(prev_state)
mark_not_mentioned(state)
words = usr.split()
usr = delexicalize.delexicalise(' '.join(words), dic)
# parsing reference number GIVEN belief state
usr = delexicaliseReferenceNumber(usr, state)
# changes to numbers only here
digitpat = re.compile('\d+')
usr = re.sub(digitpat, '[value_count]', usr)
# dialogue = fixDelex(dialogue_name, dialogue, data2, idx, idx_acts)
# add database pointer
pointer_vector, top_results, num_results = addDBPointer(state)
# add booking pointer
pointer_vector = addBookingPointer(state, pointer_vector)
belief_summary = get_summary_bstate(state)
tensor = [
model.input_word2index(word)
for word in normalize(usr).strip(' ').split(' ')
] + [util.EOS_token]
input_tensor.append(torch.LongTensor(tensor))
bs_tensor.append(belief_summary) #
db_tensor.append(pointer_vector) # db results and booking
# bs_tensor.append([0.] * 94) #
# db_tensor.append([0.] * 30) # db results and booking
# create an empty matrix with padding tokens
input_tensor, input_lengths = util.padSequence(input_tensor)
bs_tensor = torch.tensor(bs_tensor, dtype=torch.float, device=device)
db_tensor = torch.tensor(db_tensor, dtype=torch.float, device=device)
output_words, loss_sentence = model.predict(input_tensor, input_lengths,
input_tensor, input_lengths,
db_tensor, bs_tensor)
active_domain = get_active_domain(prev_active_domain, prev_state, state)
if active_domain is not None and active_domain in num_results:
num_results = num_results[active_domain]
else:
num_results = 0
if active_domain is not None and active_domain in top_results:
top_results = {active_domain: top_results[active_domain]}
else:
top_results = {}
response = populate_template(output_words[0], top_results, num_results,
state)
return response, active_domain