def delexicaliseReferenceNumber(sent, turn): """Based on the belief state, we can find reference number that during data gathering was created randomly.""" domains = [ 'restaurant', 'hotel', 'attraction', 'train', 'taxi', 'hospital' ] # , 'police'] if turn['metadata']: for domain in domains: if turn['metadata'][domain]['book']['booked']: for slot in turn['metadata'][domain]['book']['booked'][0]: if slot == 'reference': val = '[' + domain + '_' + slot + ']' else: val = '[' + domain + '_' + slot + ']' key = normalize( turn['metadata'][domain]['book']['booked'][0][slot]) sent = (' ' + sent + ' ').replace(' ' + key + ' ', ' ' + val + ' ') # try reference with hashtag key = normalize( "#" + turn['metadata'][domain]['book']['booked'][0][slot]) sent = (' ' + sent + ' ').replace(' ' + key + ' ', ' ' + val + ' ') # try reference with ref# key = normalize( "ref#" + turn['metadata'][domain]['book']['booked'][0][slot]) sent = (' ' + sent + ' ').replace(' ' + key + ' ', ' ' + val + ' ') return sent
def delexicalize_one_sent0(sent, dic): sent, kv_dic = normalize(sent) words = sent.split() sent, kv_dic_tmp = delexicalize.delexicalise(' '.join(words), dic) kv_dic.update(kv_dic_tmp) # parsing reference number GIVEN belief state # sent = delexicaliseReferenceNumber(sent, turn) # changes to numbers only here # digitpat = re.compile('(?<!looking for)(?<=for) \d+ (?!of)|(?<=party of) \d+ | \d+ (?=people|person|of us)') # # digitpat = re.compile(' \d+ ') # value_count = re.findall(digitpat, sent) # while value_count: # # index = sent.find(value_count[0]) # # if not delexicalize.check_balance(sent[:index]): # # pdb.set_trace() # value_count.pop(0) # continue # # sent = sent[:index] + \ # ' [value_count|' + value_count[0][1:-1] + '] ' + \ # sent[index + len(value_count[0]):] # value_count = re.findall(digitpat, sent) digitpat = re.compile('\d+') digits = re.findall(digitpat, sent) if len(digits): kv_dic.update({'[value_count]': digits}) sent = re.sub(digitpat, '[value_count]', sent) return sent, kv_dic
def delexicalize_one_sent(sent): def normalize(text): def insertSpace(token, text): sidx = 0 while True: sidx = text.find(token, sidx) if sidx == -1: break if sidx + 1 < len(text) and re.match('[0-9]', text[sidx - 1]) and \ re.match('[0-9]', text[sidx + 1]): sidx += 1 continue if text[sidx - 1] != ' ': text = text[:sidx] + ' ' + text[sidx:] sidx += 1 if sidx + len(token) < len(text) and text[sidx + len(token)] != ' ': text = text[:sidx + 1] + ' ' + text[sidx + 1:] sidx += 1 return text # lower case every word text = text.lower() # replace white spaces in front and end text = re.sub(r'^\s*|\s*$', '', text) # hotel domain pfb30 text = re.sub(r"b&b", "bed and breakfast", text) text = re.sub(r"b and b", "bed and breakfast", text) # normalize phone number ms = re.findall('\(?(\d{3})\)?[-.\s]?(\d{3})[-.\s]?(\d{4,5})', text) if ms: sidx = 0 for m in ms: sidx = text.find(m[0], sidx) if text[sidx - 1] == '(': sidx -= 1 eidx = text.find(m[-1], sidx) + len(m[-1]) text = text.replace(text[sidx:eidx], ''.join(m)) # normalize postcode ms = re.findall( '([a-z]{1}[\. ]?[a-z]{1}[\. ]?\d{1,2}[, ]+\d{1}[\. ]?[a-z]{1}[\. ]?[a-z]{1}|[a-z]{2}\d{2}[a-z]{2})', text) if ms: sidx = 0 for m in ms: sidx = text.find(m, sidx) eidx = sidx + len(m) text = text[:sidx] + re.sub('[,\. ]', '', m) + text[eidx:] # weird unicode bug text = re.sub(u"(\u2018|\u2019)", "'", text) text = ' ' + text + ' ' # # replace time and and price timepat = re.compile( " \d{1,2}[:]\d{1,2}[ \.,\?]| \d{4}[ \.,\?]| \d{1,2}[ap][m\. ]+| \d{1,2} [ap][m\. ]+| \d{1,2}[:]\d{1,2}[ap]m[ \.,\?]" ) # # some utterances just miss the ":" # timepat_noise = re.compile(" \d{4}[ \.,\?]") pricepat = re.compile("\d{1,3}[\.]\d{1,2}") value_time = re.findall(timepat, text) # pdb.set_trace() while value_time: index = text.find(value_time[0]) text = text[:index] + \ ' [value_time|' + value_time[0][1:-1] + ']' + \ text[index + len(value_time[0]) - 1:] value_time = re.findall(timepat, text) value_price = re.findall(pricepat, text) if value_price: text = re.sub(pricepat, ' [value_price|' + value_price[0] + '] ', text) text = text[1:-1] # replace st. text = text.replace(';', ',') text = re.sub('$\/', '', text) text = text.replace('/', ' and ') # replace other special characters text = text.replace('-', ' ') text = re.sub('[\"\<>@\(\)]', '', text) # insert white space before and after tokens: for token in ['?', '.', ',', '!']: text = insertSpace(token, text) # insert white space for 's text = insertSpace('\'s', text) # replace it's, does't, you'd ... etc text = re.sub('^\'', '', text) text = re.sub('\'$', '', text) text = re.sub('\'\s', ' ', text) text = re.sub('\s\'', ' ', text) fin = open( '/data/qkun/simulator/data/multiwoz-master/utils/mapping.pair') replacements = [] for line in fin.readlines(): tok_from, tok_to = line.replace('\n', '').split('\t') replacements.append((' ' + tok_from + ' ', ' ' + tok_to + ' ')) for fromx, tox in replacements: text = ' ' + text + ' ' text = text.replace(fromx, tox)[1:-1] # remove multiple spaces text = re.sub(' +', ' ', text) # concatenate numbers tmp = text tokens = text.split() i = 1 while i < len(tokens): if re.match(u'^\d+$', tokens[i]) and \ re.match(u'\d+$', tokens[i - 1]): tokens[i - 1] += tokens[i] del tokens[i] else: i += 1 text = ' '.join(tokens) return text def check_balance(string): # open_tup = tuple('[') # close_tup = tuple(']') # map = dict(zip(open_tup, close_tup)) queue = 0 for i in string: if i == '[': queue += 1 elif i == ']': if not queue: return False else: queue -= 1 if not queue: return True else: return False def delexicalise(utt, dictionary): for key, val in dictionary: utt = (' ' + utt + ' ') if key in utt: idx = 0 while utt[idx:].find(' ' + key + ' ') != -1: idx += utt[idx:].find(' ' + key + ' ') # # to exclude the case that 'ask' is a verb if key == 'ask' and idx > 2 and utt[idx - 2:idx] == ' i': idx += 1 continue if check_balance(utt[:idx]): utt = utt[: idx] + ' ' + val[: -1] + '|' + key + '] ' + utt[ idx + len(key) + 2:] idx += len(key) + 4 + len(val[:-1]) else: idx += len(key) utt = utt[1:-1] return utt def delex_people_count(sent): sent = ' ' + sent + ' ' digitpat = re.compile( '(?<!looking for)(?<=for) \d+ (?!of)|(?<=party of) \d+ | \d+ (?=people|person|of us)' ) value_count = re.findall(digitpat, sent) while value_count: index = sent.find(value_count[0]) if not check_balance(sent[:index]): value_count.pop(0) continue sent = sent[:index] + \ ' [value_count|' + value_count[0][1:-1] + '] ' + \ sent[index + len(value_count[0]):] value_count = re.findall(digitpat, sent) sent = sent[1:-1] return sent # # # extract slots # # replace time, date, specific price sent = ' '.join(word_tokenize(sent)) sent = normalize(sent) # pdb.set_trace() # # replace info in db db_entity_file = open( '/data/qkun/simulator/data/multiwoz-master/db_entity_file.pkl', 'rb') db_entity_list = pkl.load(db_entity_file) db_entity_file.close() sent = delexicalise(sent, db_entity_list) # # replace # of people for reservation sent = delex_people_count(sent) # # # replace and generate dic slotpat = re.compile('\[.*?\]') slots = re.findall(slotpat, sent) slot_dic = {} for slot in slots: if '|' in slot: [slot_name, slot_val] = slot[1:-1].split('|') slot_name = '[' + slot_name + ']' slot_dic[slot_name] = slot_val sent = sent.replace(slot, slot_name) sent = sent.replace('[', "").replace(']', "") return sent, slot_dic
def createDelexData(): """Main function of the script - loads delexical dictionary, goes through each dialogue and does: 1) data normalization 2) delexicalization 3) addition of database pointer 4) saves the delexicalized data """ # download the data loadData() # create dictionary of delexicalied values that then we will search against, order matters here! dic = delexicalize.prepareSlotValuesIndependent() fin1 = file('data/multi-woz/data.json') data = json.load(fin1) fin2 = file('data/multi-woz/dialogue_acts.json') data2 = json.load(fin2) for dialogue_name in tqdm(data): dialogue = data[dialogue_name] # print dialogue_name idx_acts = 1 for idx, turn in enumerate(dialogue['log']): # normalization, split and delexicalization of the sentence sent = normalize(turn['text']) words = sent.split() sent = delexicalize.delexicalise(' '.join(words), dic) # parsing reference number GIVEN belief state sent = delexicaliseReferenceNumber(sent, turn) # changes to numbers only here digitpat = re.compile('\d+') sent = re.sub(digitpat, '[value_count]', sent) # delexicalized sentence added to the dialogue dialogue['log'][idx]['text'] = sent if idx % 2 == 1: # if it's a system turn # add database pointer pointer_vector = addDBPointer(turn) # add booking pointer pointer_vector = addBookingPointer(dialogue, turn, pointer_vector) # print pointer_vector dialogue['log'][idx - 1]['db_pointer'] = pointer_vector.tolist() # FIXING delexicalization: dialogue = fixDelex(dialogue_name, dialogue, data2, idx, idx_acts) idx_acts += 1 delex_data[dialogue_name] = dialogue with open('data/multi-woz/delex.json', 'w') as outfile: json.dump(delex_data, outfile) return delex_data
fin1 = file('data/multi-woz/data.json') data = json.load(fin1) fin2 = file('data/multi-woz/dialogue_acts.json') data2 = json.load(fin2) for dialogue_name in tqdm(data): dialogue = data[dialogue_name] #print dialogue_name idx_acts = 1 for idx, turn in enumerate(dialogue['log']): # normalization, split and delexicalization of the sentence sent = normalize(turn['text']) words = sent.split() sent = delexicalize.delexicalise(' '.join(words), dic) # parsing reference number GIVEN belief state sent = delexicaliseReferenceNumber(sent, turn) # changes to numbers only here digitpat = re.compile('\d+') sent = re.sub(digitpat, '[value_count]', sent) # delexicalized sentence added to the dialogue dialogue['log'][idx]['text'] = sent if idx % 2 == 1: # if it's a system turn
def queryResultVenues(domain, turn, real_belief=False): # query the db sql_query = "select * from {}".format(domain) if real_belief == True: items = turn.items() elif real_belief == 'tracking': for slot in turn[domain]: key = slot[0].split("-")[1] val = slot[0].split("-")[2] if key == "price range": key = "pricerange" elif key == "leave at": key = "leaveAt" elif key == "arrive by": key = "arriveBy" if val == "do n't care": pass else: if flag: sql_query += " where " val2 = val.replace("'", "''") val2 = normalize(val2) if key == 'leaveAt': sql_query += key + " > " + r"'" + val2 + r"'" elif key == 'arriveBy': sql_query += key + " < " + r"'" + val2 + r"'" else: sql_query += r" " + key + "=" + r"'" + val2 + r"'" flag = False else: val2 = val.replace("'", "''") val2 = normalize(val2) if key == 'leaveAt': sql_query += r" and " + key + " > " + r"'" + val2 + r"'" elif key == 'arriveBy': sql_query += r" and " + key + " < " + r"'" + val2 + r"'" else: sql_query += r" and " + key + "=" + r"'" + val2 + r"'" try: # "select * from attraction where name = 'queens college'" return dbs[domain].execute(sql_query).fetchall() except: return [] # TODO test it pass else: items = turn['metadata'][domain]['semi'].items() flag = True for key, val in items: if val == "" or val == "dontcare" or val == 'not mentioned' or val == "don't care" or val == "dont care" or val == "do n't care": pass else: if flag: sql_query += " where " val2 = val.replace("'", "''") val2 = normalize(val2) if key == 'leaveAt': sql_query += r" " + key + " > " + r"'" + val2 + r"'" elif key == 'arriveBy': sql_query += r" " + key + " < " + r"'" + val2 + r"'" else: sql_query += r" " + key + "=" + r"'" + val2 + r"'" flag = False else: val2 = val.replace("'", "''") val2 = normalize(val2) if key == 'leaveAt': sql_query += r" and " + key + " > " + r"'" + val2 + r"'" elif key == 'arriveBy': sql_query += r" and " + key + " < " + r"'" + val2 + r"'" else: sql_query += r" and " + key + "=" + r"'" + val2 + r"'" try: # "select * from attraction where name = 'queens college'" return dbs[domain].execute(sql_query).fetchall() except: return [] # TODO test it
def prepareSlotValuesIndependent(): domains = [ 'restaurant', 'hotel', 'attraction', 'train', 'taxi', 'hospital', 'police' ] requestables = ['phone', 'address', 'postcode', 'reference', 'id'] dic = [] dic_area = [] dic_food = [] dic_price = [] # read databases for domain in domains: try: fin = file('db/' + domain + '_db.json') db_json = json.load(fin) fin.close() for ent in db_json: for key, val in ent.items(): if val == '?' or val == 'free': pass elif key == 'address': dic.append((normalize(val), '[' + domain + '_' + 'address' + ']')) if "road" in val: val = val.replace("road", "rd") dic.append((normalize(val), '[' + domain + '_' + 'address' + ']')) elif "rd" in val: val = val.replace("rd", "road") dic.append((normalize(val), '[' + domain + '_' + 'address' + ']')) elif "st" in val: val = val.replace("st", "street") dic.append((normalize(val), '[' + domain + '_' + 'address' + ']')) elif "street" in val: val = val.replace("street", "st") dic.append((normalize(val), '[' + domain + '_' + 'address' + ']')) elif key == 'name': dic.append((normalize(val), '[' + domain + '_' + 'name' + ']')) if "b & b" in val: val = val.replace("b & b", "bed and breakfast") dic.append((normalize(val), '[' + domain + '_' + 'name' + ']')) elif "bed and breakfast" in val: val = val.replace("bed and breakfast", "b & b") dic.append((normalize(val), '[' + domain + '_' + 'name' + ']')) elif "hotel" in val and 'gonville' not in val: val = val.replace("hotel", "") dic.append((normalize(val), '[' + domain + '_' + 'name' + ']')) elif "restaurant" in val: val = val.replace("restaurant", "") dic.append((normalize(val), '[' + domain + '_' + 'name' + ']')) elif key == 'postcode': dic.append((normalize(val), '[' + domain + '_' + 'postcode' + ']')) elif key == 'phone': dic.append((val, '[' + domain + '_' + 'phone' + ']')) elif key == 'trainID': dic.append( (normalize(val), '[' + domain + '_' + 'id' + ']')) elif key == 'department': dic.append((normalize(val), '[' + domain + '_' + 'department' + ']')) # NORMAL DELEX elif key == 'area': dic_area.append((normalize(val), '[' + 'value' + '_' + 'area' + ']')) elif key == 'food': dic_food.append((normalize(val), '[' + 'value' + '_' + 'food' + ']')) elif key == 'pricerange': dic_price.append( (normalize(val), '[' + 'value' + '_' + 'pricerange' + ']')) else: pass # TODO car type? except: pass if domain == 'hospital': dic.append( (normalize('Hills Rd'), '[' + domain + '_' + 'address' + ']')) dic.append((normalize('Hills Road'), '[' + domain + '_' + 'address' + ']')) dic.append( (normalize('CB20QQ'), '[' + domain + '_' + 'postcode' + ']')) dic.append(('01223245151', '[' + domain + '_' + 'phone' + ']')) dic.append(('1223245151', '[' + domain + '_' + 'phone' + ']')) dic.append(('0122324515', '[' + domain + '_' + 'phone' + ']')) dic.append((normalize('Addenbrookes Hospital'), '[' + domain + '_' + 'name' + ']')) elif domain == 'police': dic.append( (normalize('Parkside'), '[' + domain + '_' + 'address' + ']')) dic.append( (normalize('CB11JG'), '[' + domain + '_' + 'postcode' + ']')) dic.append(('01223358966', '[' + domain + '_' + 'phone' + ']')) dic.append(('1223358966', '[' + domain + '_' + 'phone' + ']')) dic.append((normalize('Parkside Police Station'), '[' + domain + '_' + 'name' + ']')) # add at the end places from trains fin = file('db/' + 'train' + '_db.json') db_json = json.load(fin) fin.close() for ent in db_json: for key, val in ent.items(): if key == 'departure' or key == 'destination': dic.append( (normalize(val), '[' + 'value' + '_' + 'place' + ']')) # add specific values: for key in [ 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday' ]: dic.append((normalize(key), '[' + 'value' + '_' + 'day' + ']')) # more general values add at the end dic.extend(dic_area) dic.extend(dic_food) dic.extend(dic_price) return dic