def score_slot_realizations(path, filename): """Analyzes unrealized and hallucinated slot mentions in the utterances.""" errors = [] incorrect_slots = [] # slot_cnt = 0 # print('Analyzing missing slot realizations and hallucinations in ' + str(filename)) # Read in the data data_cont = data_loader.init_test_data(os.path.join(path, filename)) dataset_name = data_cont['dataset_name'] mrs_orig, utterances_orig = data_cont['data'] _, _, slot_sep, val_sep, val_sep_end = data_cont['separators'] # Preprocess the MRs and the utterances mrs = [data_loader.preprocess_mr(mr, data_cont['separators']) for mr in mrs_orig] utterances = [data_loader.preprocess_utterance(utt) for utt in utterances_orig] # DEBUG PRINT # print('\n'.join([mr1 + '\n' + mr2 + '\n' for mr1, mr2 in zip(mrs_orig, mrs)])) for i, mr in enumerate(mrs): mr_dict = OrderedDict() # Extract the slot-value pairs into a dictionary for slot_value in mr.split(slot_sep): slot, value, _, _ = data_loader.parse_slot_and_value(slot_value, val_sep, val_sep_end) mr_dict[slot] = value # Count auxiliary slots # if not re.match(r'<!.*?>', slot): # slot_cnt += 1 # TODO: get rid of this hack # Move the food-slot to the end of the dict (because of delexing) if 'food' in mr_dict: food_val = mr_dict['food'] del(mr_dict['food']) mr_dict['food'] = food_val # Delexicalize the MR and the utterance utterances[i] = data_loader.delex_sample(mr_dict, utterances[i], dataset=dataset_name) # Count the missing and hallucinated slots in the utterance cur_errors, cur_incorrect_slots = count_errors(utterances[i], mr_dict) errors.append(cur_errors) incorrect_slots.append(', '.join(cur_incorrect_slots)) # DEBUG PRINT # print(slot_cnt) new_df = pd.DataFrame(columns=['mr', 'ref', 'errors', 'incorrect slots']) new_df['mr'] = mrs_orig new_df['ref'] = utterances_orig new_df['errors'] = errors new_df['incorrect slots'] = incorrect_slots filename_out = os.path.splitext(filename)[0] + ' [errors].csv' new_df.to_csv(os.path.join(path, filename_out), index=False, encoding='utf8')
def align_slots(dataset, filename): """Aligns slots of the MRs with their mentions in the corresponding utterances.""" alignments = [] alignment_strings = [] print('Aligning slots in ' + str(filename)) # Read in the data data_cont = data_loader.init_test_data(os.path.join(config.DATA_DIR, dataset, filename)) mrs_orig, utterances_orig = data_cont['data'] _, _, slot_sep, val_sep, val_sep_end = data_cont['separators'] # Preprocess the MRs and the utterances mrs = [data_loader.preprocess_mr(mr, data_cont['separators']) for mr in mrs_orig] utterances = [data_loader.preprocess_utterance(utt) for utt in utterances_orig] for i, mr in enumerate(mrs): mr_dict = OrderedDict() # Extract the slot-value pairs into a dictionary for slot_value in mr.split(slot_sep): slot, value, _, _ = data_loader.parse_slot_and_value(slot_value, val_sep, val_sep_end) mr_dict[slot] = value alignments.append(find_alignment(utterances[i], mr_dict)) for i in range(len(utterances)): alignment_strings.append(' '.join(['({0}: {1})'.format(pos, slot) for pos, slot, _ in alignments[i]])) new_df = pd.DataFrame(columns=['mr', 'ref', 'alignment']) new_df['mr'] = mrs_orig new_df['ref'] = utterances_orig new_df['alignment'] = alignment_strings filename_out = os.path.splitext(filename)[0] + ' [aligned].csv' new_df.to_csv(os.path.join(config.DATA_DIR, dataset, filename_out), index=False, encoding='utf8')
def analyze_contrast_relations(dataset, filename): """Identifies the slots involved in a contrast relation.""" contrast_connectors = ['but', 'however', 'yet'] slots_before = [] slots_after = [] print('Analyzing contrast relations in ' + str(filename)) # Read in the data data_cont = data_loader.init_test_data(os.path.join(config.DATA_DIR, dataset, filename)) mrs_orig, utterances_orig = data_cont['data'] _, _, slot_sep, val_sep, val_sep_end = data_cont['separators'] # Preprocess the MRs mrs = [data_loader.preprocess_mr(mr, data_cont['separators']) for mr in mrs_orig] for mr, utt in zip(mrs, utterances_orig): mr_dict = OrderedDict() mr_list_augm = [] # Extract the slot-value pairs into a dictionary for slot_value in mr.split(slot_sep): slot, value, slot_orig, value_orig = data_loader.parse_slot_and_value(slot_value, val_sep, val_sep_end) mr_dict[slot] = value mr_list_augm.append((slot, value_orig)) # Find the slot alignment alignment = find_alignment(utt, mr_dict) slot_before = None slot_after = None for contrast_conn in contrast_connectors: contrast_pos = utt.find(contrast_conn) if contrast_pos >= 0: slot_before = None slot_after = None for pos, slot, value in alignment: slot_before = slot_after slot_after = slot if pos > contrast_pos: break break slots_before.append(slot_before if slot_before is not None else '') slots_after.append(slot_after if slot_after is not None else '') # Calculate the frequency distribution of slots involved in a contrast relation contrast_slot_cnt = Counter() contrast_slot_cnt.update(slots_before + slots_after) del contrast_slot_cnt[''] print('\n---- Slot distribution in contrast relations ----\n') print('\n'.join(slot + ': ' + str(freq) for slot, freq in contrast_slot_cnt.most_common())) # Calculate the frequency distribution of slot pairs involved in a contrast relation contrast_slot_cnt = Counter() slot_pairs = [tuple(sorted(slot_pair)) for slot_pair in zip(slots_before, slots_after) if slot_pair != ('', '')] contrast_slot_cnt.update(slot_pairs) print('\n---- Slot pair distribution in contrast relations ----\n') print('\n'.join(slot_pair[0] + ', ' + slot_pair[1] + ': ' + str(freq) for slot_pair, freq in contrast_slot_cnt.most_common())) new_df = pd.DataFrame(columns=['mr', 'ref', 'slot before contrast', 'slot after contrast']) new_df['mr'] = mrs_orig new_df['ref'] = utterances_orig new_df['slot before contrast'] = slots_before new_df['slot after contrast'] = slots_after filename_out = os.path.splitext(filename)[0] + ' [contrast relations].csv' new_df.to_csv(os.path.join(config.DATA_DIR, dataset, filename_out), index=False, encoding='utf8')
def score_contrast(dataset, filename): """Determines whether the indicated contrast relation is correctly realized in the utterance.""" contrast_connectors = ['but', 'however', 'yet'] contrast_missed = [] contrast_incorrectness = [] contrast_total = [] print('Analyzing contrast realizations in ' + str(filename)) # Read in the data data_cont = data_loader.init_test_data(os.path.join(config.EVAL_DIR, dataset, filename)) dataset_name = data_cont['dataset_name'] mrs_orig, utterances_orig = data_cont['data'] _, _, slot_sep, val_sep, val_sep_end = data_cont['separators'] # Preprocess the MRs and the utterances mrs = [data_loader.preprocess_mr(mr, data_cont['separators']) for mr in mrs_orig] utterances = [data_loader.preprocess_utterance(utt) for utt in utterances_orig] for i, mr in enumerate(mrs): contrast_found = False contrast_correct = False contrast_slots = [] mr_dict = OrderedDict() # Extract the slot-value pairs into a dictionary for slot_value in mr.split(slot_sep): slot, value, _, _ = data_loader.parse_slot_and_value(slot_value, val_sep, val_sep_end) # Extract slots to be contrasted if slot in [config.CONTRAST_TOKEN, config.CONCESSION_TOKEN]: contrast_slots.extend(value.split()) else: mr_dict[slot] = value # Delexicalize the MR and the utterance utterances[i] = data_loader.delex_sample(mr_dict, utterances[i], dataset=dataset_name) # Determine the slot alignment in the utterance alignment = find_alignment(utterances[i], mr_dict) contrast_total.append(1 if len(contrast_slots) > 0 else 0) if len(contrast_slots) > 0: for contrast_conn in contrast_connectors: contrast_pos = utterances[i].find(contrast_conn) if contrast_pos < 0: continue slot_left_pos = -1 slot_right_pos = -1 dist = 0 contrast_found = True # Check whether the correct pair of slots was contrasted for pos, slot, _ in alignment: # DEBUG PRINT # print(alignment) # print(contrast_slots) # print() if slot_left_pos > -1: dist += 1 if slot in contrast_slots: if slot_left_pos == -1: slot_left_pos = pos else: slot_right_pos = pos break if slot_left_pos > -1 and slot_right_pos > -1: if slot_left_pos < contrast_pos < slot_right_pos and dist <= 2: contrast_correct = True break else: contrast_found = True contrast_correct = True contrast_missed.append(0 if contrast_found else 1) contrast_incorrectness.append(0 if contrast_correct else 1) new_df = pd.DataFrame(columns=['mr', 'ref', 'missed contrast', 'incorrect contrast', 'total contrast']) new_df['mr'] = mrs_orig new_df['ref'] = utterances_orig new_df['missed contrast'] = contrast_missed new_df['incorrect contrast'] = contrast_incorrectness new_df['total contrast'] = contrast_total filename_out = os.path.splitext(filename)[0] + ' [contrast eval].csv' new_df.to_csv(os.path.join(config.EVAL_DIR, dataset, filename_out), index=False, encoding='utf8')
def score_emphasis(dataset, filename): """Determines how many of the indicated emphasis instances are realized in the utterance.""" emph_missed = [] emph_total = [] print('Analyzing emphasis realizations in ' + str(filename)) # Read in the data data_cont = data_loader.init_test_data(os.path.join(config.EVAL_DIR, dataset, filename)) dataset_name = data_cont['dataset_name'] mrs_orig, utterances_orig = data_cont['data'] _, _, slot_sep, val_sep, val_sep_end = data_cont['separators'] # Preprocess the MRs and the utterances mrs = [data_loader.preprocess_mr(mr, data_cont['separators']) for mr in mrs_orig] utterances = [data_loader.preprocess_utterance(utt) for utt in utterances_orig] for i, mr in enumerate(mrs): expect_emph = False emph_slots = set() mr_dict = OrderedDict() # Extract the slot-value pairs into a dictionary for slot_value in mr.split(slot_sep): slot, value, _, _ = data_loader.parse_slot_and_value(slot_value, val_sep, val_sep_end) # Extract slots to be emphasized if slot == config.EMPH_TOKEN: expect_emph = True else: mr_dict[slot] = value if expect_emph: emph_slots.add(slot) expect_emph = False # Delexicalize the MR and the utterance utterances[i] = data_loader.delex_sample(mr_dict, utterances[i], dataset=dataset_name) # Determine the slot alignment in the utterance alignment = find_alignment(utterances[i], mr_dict) emph_total.append(len(emph_slots)) # Check how many emphasized slots were not realized before the name-slot for pos, slot, _ in alignment: # DEBUG PRINT # print(alignment) # print(emph_slots) # print() if slot == 'name': break if slot in emph_slots: emph_slots.remove(slot) emph_missed.append(len(emph_slots)) new_df = pd.DataFrame(columns=['mr', 'ref', 'missed emphasis', 'total emphasis']) new_df['mr'] = mrs_orig new_df['ref'] = utterances_orig new_df['missed emphasis'] = emph_missed new_df['total emphasis'] = emph_total filename_out = os.path.splitext(filename)[0] + ' [emphasis eval].csv' new_df.to_csv(os.path.join(config.EVAL_DIR, dataset, filename_out), index=False, encoding='utf8')
def augment_with_aux_indicators(dataset, filename, indicators, mode='all', alt_contrast_mode=False): """Augment MRs in a dataset with auxiliary tokens indicating desired discourse phenomena in the corresponding utterances. Depending on the mode, the augmented dataset will only contain samples which exhibit 1.) at most one of the desired indicators ('single'), 2.) the one selected indicator only ('only'), or 3.) all the desired indicators at once ('combo'). The default mode ('all') keeps all samples in the dataset. """ if indicators is None or len(indicators) == 0: return mrs_augm = [] mrs_single = [] utterances_single = [] mrs_emph_only = [] utterances_emph_only = [] mrs_contrast_only = [] utterances_contrast_only = [] mrs_combo = [] utterances_combo = [] emph_only_ctr = 0 contrast_only_ctr = 0 combo_ctr = 0 print('Augmenting MRs with ' + ' + '.join(indicators) + ' in ' + str(filename)) # Read in the data data_cont = data_loader.init_test_data( os.path.join(config.DATA_DIR, dataset, filename)) mrs, utterances = data_cont['data'] _, _, slot_sep, val_sep, val_sep_end = data_cont['separators'] for mr, utt in zip(mrs, utterances): mr_dict = OrderedDict() mr_list_augm = [] # Extract the slot-value pairs into a dictionary for slot_value in mr.split(slot_sep): slot, value, slot_orig, value_orig = data_loader.parse_slot_and_value( slot_value, val_sep, val_sep_end) mr_dict[slot] = value mr_list_augm.append((slot, value_orig)) # mrs[i] = mrs[i].replace(slot_orig, slot) # Find the slot alignment alignment = find_alignment(utt, mr_dict) # Augment the MR with auxiliary tokens if 'emphasis' in indicators: __add_emphasis_tokens(mr_list_augm, alignment) if 'contrast' in indicators: __add_contrast_tokens(mr_list_augm, utt, alignment, alt_mode=alt_contrast_mode) # Convert augmented MR from list to string representation mr_augm = (slot_sep + ' ').join([ s + val_sep + v + (val_sep_end if val_sep_end is not None else '') for s, v in mr_list_augm ]) mrs_augm.append(mr_augm) # Count and separate the different augmentation instances slots_augm = set([s for s, v in mr_list_augm]) if config.EMPH_TOKEN in slots_augm and ( config.CONTRAST_TOKEN in slots_augm or config.CONCESSION_TOKEN in slots_augm): mrs_combo.append(mr_augm) utterances_combo.append(utt) combo_ctr += 1 else: mrs_single.append(mr_augm) utterances_single.append(utt) if config.EMPH_TOKEN in slots_augm: mrs_emph_only.append(mr_augm) utterances_emph_only.append(utt) emph_only_ctr += 1 elif config.CONTRAST_TOKEN in slots_augm or config.CONCESSION_TOKEN in slots_augm: mrs_contrast_only.append(mr_augm) utterances_contrast_only.append(utt) contrast_only_ctr += 1 print('# of MRs with emphasis only:', emph_only_ctr) print('# of MRs with contrast/concession only:', contrast_only_ctr) print('# of MRs with emphasis & contrast/concession:', combo_ctr) new_df = pd.DataFrame(columns=['mr', 'ref']) if mode == 'single': new_df['mr'] = mrs_single new_df['ref'] = utterances_single elif mode == 'only': if 'emphasis' in indicators: new_df['mr'] = mrs_emph_only new_df['ref'] = utterances_emph_only elif 'contrast' in indicators: new_df['mr'] = mrs_contrast_only new_df['ref'] = utterances_contrast_only elif mode == 'combo': new_df['mr'] = mrs_combo new_df['ref'] = utterances_combo else: new_df['mr'] = mrs_augm new_df['ref'] = utterances # Store augmented dataset to a new file filename_out = os.path.splitext(filename)[0] + '_augm_' + '_'.join(indicators) \ + (('_' + mode) if mode != 'all' else '') + ('_alt' if alt_contrast_mode else '') + '.csv' new_df.to_csv(os.path.join(config.DATA_DIR, dataset, filename_out), index=False, encoding='utf8')
def augment_with_contrast_tgen(dataset, filename): """Augments the MRs with auxiliary tokens indicating a pair of slots that should be contrasted in the corresponding generated utterance. The output is in the format accepted by TGen. """ contrast_connectors = ['but', 'however', 'yet'] scalar_slots = get_scalar_slots() alignments = [] contrasts = [] print('Augmenting MRs with contrast in ' + str(filename)) # Read in the data data_cont = data_loader.init_test_data( os.path.join(config.DATA_DIR, dataset, filename)) mrs, utterances = data_cont['data'] _, _, slot_sep, val_sep, val_sep_end = data_cont['separators'] for i, mr in enumerate(mrs): mr_dict = OrderedDict() # Extract the slot-value pairs into a dictionary for slot_value in mr.split(slot_sep): slot, value, slot_orig, _ = data_loader.parse_slot_and_value( slot_value, val_sep, val_sep_end) mr_dict[slot] = value mrs[i] = mrs[i].replace(slot_orig, slot) alignments.append(find_alignment(utterances[i], mr_dict)) for i in range(len(utterances)): contrasts.append(['none', 'none', 'none', 'none']) for contrast_conn in contrast_connectors: contrast_pos = utterances[i].find(contrast_conn) if contrast_pos >= 0: slot_before = None value_before = None slot_after = None value_after = None for pos, slot, value in alignments[i]: if pos > contrast_pos: if not slot_before: break if slot in scalar_slots: slot_after = slot value_after = value break else: if slot in scalar_slots: slot_before = slot value_before = value if slot_before and slot_after: if scalar_slots[slot_before][value_before] - scalar_slots[ slot_after][value_after] == 0: contrasts[i][2] = slot_before contrasts[i][3] = slot_after else: contrasts[i][0] = slot_before contrasts[i][1] = slot_after break new_df = pd.DataFrame(columns=[ 'mr', 'ref', 'contrast1', 'contrast2', 'concession1', 'concession2' ]) new_df['mr'] = mrs new_df['ref'] = utterances new_df['contrast1'] = [tup[0] for tup in contrasts] new_df['contrast2'] = [tup[1] for tup in contrasts] new_df['concession1'] = [tup[2] for tup in contrasts] new_df['concession2'] = [tup[3] for tup in contrasts] filename_out = ''.join( filename.split('.')[:-1]) + '_augm_contrast_tgen.csv' new_df.to_csv(os.path.join(config.DATA_DIR, dataset, filename_out), index=False, encoding='utf8')
def augment_by_utterance_splitting(dataset, filename, denoise_only=False): """Performs utterance splitting and augments the dataset with new pseudo-samples whose utterances are one sentence long. The MR of each pseudo-sample contains only slots mentioned in the corresponding sentence. Assumes a CSV or JSON file as input. """ if not filename.lower().endswith(('.csv', '.json')): raise ValueError( 'Unexpected file type. Please provide a CSV or JSON file as input.' ) mrs_dicts = [] data_new = [] print('Performing utterance splitting on ' + str(filename)) # Read in the data data_cont = data_loader.init_test_data( os.path.join(config.DATA_DIR, dataset, filename)) mrs, utterances = data_cont['data'] _, _, slot_sep, val_sep, val_sep_end = data_cont['separators'] for i, mr in enumerate(mrs): mr_dict = OrderedDict() # Extract the slot-value pairs into a dictionary for slot_value in mr.split(slot_sep): slot, _, _, value_orig = data_loader.parse_slot_and_value( slot_value, val_sep, val_sep_end) mr_dict[slot] = value_orig mrs_dicts.append(mr_dict) new_mrs, new_utterances = split_content(mrs_dicts, utterances, filename, permute=False, denoise_only=denoise_only) suffix = ' [' + ('denoised' if denoise_only else 'utt. split') + ']' filename_out = os.path.splitext(filename)[0] + suffix + os.path.splitext( filename)[1] if filename.lower().endswith('.csv'): for row, mr in enumerate(new_mrs): if len(mr) == 0: continue mr_str = ', '.join( ['{0}[{1}]'.format(slot, value) for slot, value in mr.items()]) data_new.append([mr_str, new_utterances[row]]) # Write the augmented dataset to a new file pd.DataFrame(data_new).to_csv(os.path.join(config.DATA_DIR, dataset, filename_out), header=['mr', 'ref'], index=False, encoding='utf8') elif filename.lower().endswith('.json'): for row, mr in enumerate(new_mrs): if len(mr) == 0: continue mr_str = mr.pop('da') mr_str += '(' + slot_sep.join([ '{0}{1}{2}'.format(key.rstrip(string.digits), val_sep, value) for key, value in mr.items() ]) + ')' data_new.append([mr_str, new_utterances[row]]) # Write the augmented dataset to a new file with io.open(os.path.join(config.DATA_DIR, dataset, filename_out), 'w', encoding='utf8') as f_data_new: json.dump(data_new, f_data_new, indent=4)