def process_instance(da, conc, mr, multi_ref_id): original_da = deepcopy(da) # why do the das need to be sorted? This seems weird # Anyway, we checked it gets sorted in delex_sent anyway so nothing to # do about it until later da.sort() conc_das.append(da) text, da, abst = delex_sent(da, tokenize(conc), slots_to_abstract, args.slot_names, repeated=True) # Originall we didn't want to lower case because it will make things # easier for udpipe later on, however ... # we changed our mind on this because the upper case characters are # messing with udpipe's ability to properly sentence tokenize. # we need underscores instead of dashes or else udpipe breaks it apart # text = re.sub(r"X-", r"X_", text) # Again running into problems with leaving x and as a capital letter # and also with udpipe randomly segmenting it but sometimes not. We # really need to find a more reliable sentence tokenizer / word # tokenizer text = text.lower().replace('x-', 'x') # We're testing out making xnear upper case to see if it reduces the # incorrect dropping of it by the deep parser text = text.replace('xnear', 'Xnear') # detokenize some of the apostrophe stuff because udpipe does it # differently. Namely removing spaces between letters and apostrophes text = re.sub(find_apostrophes, r"\1\2", text) da.sort() da_keys[unicode(da)] = da_keys.get(unicode(da), 0) + 1 das.append(da) concs.append(conc) absts.append(abst) texts.append(text) # now for our own bastardized sentence tokenization and human eval # required stuff this_conc_sents = sent_tokenize(conc) num_sents = len(this_conc_sents) this_delex_sents = [] for i, this_conc_sent in enumerate(this_conc_sents): text, _, _ = delex_sent(original_da, tokenize(this_conc_sent), slots_to_abstract, args.slot_names, repeated=True) text = text.lower().replace('x-', 'x') # We're testing out making xnear upper case to see if it reduces the # incorrect dropping of it by the deep parser text = text.replace('xnear', 'Xnear') # detokenize some of the apostrophe stuff because udpipe does it # differently. Namely removing spaces between letters and apostrophes text = re.sub(find_apostrophes, r"\1\2", text) this_delex_sents.append(text) # start appending the sentence specific ones sent_ids.append('_'.join([mr.replace(' ', ''), str(multi_ref_id), str(i)])) mrs_for_delex.append(mr) # now we're onto something else original_sents.append('\n'.join(this_conc_sents)) delexicalised_sents.append('\n'.join(this_delex_sents))
def process_instance(da, conc): da.sort() conc_das.append(da) text, da, abst = delex_sent(da, tokenize(conc), slots_to_abstract, args.slot_names, repeated=True) text = text.lower().replace('x-', 'X-') # lowercase all but placeholders da.sort() da_keys[unicode(da)] = da_keys.get(unicode(da), 0) + 1 das.append(da) concs.append(conc) absts.append(abst) texts.append(text)
def process_instance(conc_da, conc): # sort the DA using the same order as in E2E NLG data conc_da.dais.sort(key=lambda dai: (['name', 'eat_type', 'food', 'price_range', 'rating', 'area', 'family_friendly', 'near'].index(dai.slot), dai.value)) conc_das.append(conc_da) text, da, abst = delex_sent(conc_da, tokenize(conc), slots_to_abstract, args.slot_names, repeated=True) text = text.lower().replace('x-', 'X-') # lowercase all but placeholders da.dais.sort(key=lambda dai: (['name', 'eat_type', 'food', 'price_range', 'rating', 'area', 'family_friendly', 'near'].index(dai.slot), dai.value)) da_keys[str(da)] = da_keys.get(str(da), 0) + 1 das.append(da) concs.append(conc) absts.append(abst) texts.append(text)
def process_instance(da, conc): da.sort() conc_das.append(da) # store the non-delexicalized version of the DA # delexicalize text, da, abst = delex_sent(da, conc, slots_to_abstract, args.slot_names) da.sort() # delexicalization does not keep DAI order, need to sort again # store the DA text = fix_capitalization(text) conc = fix_capitalization(conc) da_keys[str(da)] = da_keys.get(str(da), 0) + 1 das.append(da) concs.append(conc) absts.append(abst) texts.append(text)
def process_instance(da, conc): da.sort() conc_das.append(da) # store the non-delexicalized version of the DA # delexicalize text, da, abst = delex_sent(da, conc, slots_to_abstract, args.slot_names) da.sort() # delexicalization does not keep DAI order, need to sort again # store the DA text = fix_capitalization(text) conc = fix_capitalization(conc) da_keys[unicode(da)] = da_keys.get(unicode(da), 0) + 1 das.append(da) concs.append(conc) absts.append(abst) texts.append(text)
def process_instance(da, conc): da.sort() conc_das.append(da) text, da, abst = delex_sent(da, tokenize(conc), slots_to_abstract, args.slot_names, repeated=True) text = text.lower().replace('x-', 'X-') # lowercase all but placeholders da.sort() da_keys[unicode(da)] = da_keys.get(unicode(da), 0) + 1 das.append(da) concs.append(conc) absts.append(abst) texts.append(text)
def delexicalize_refs(mrs, refs, delex_slots, delex_output_file): delex_refs = [] print('Delexicalizing...', end=' ', file=sys.stderr) for pos, (mr, ref) in enumerate(zip(mrs, refs)): delex_ref = [] print(pos, end=' ', file=sys.stderr) sys.stderr.flush() for sent in ref: delex, _, absts = delex_sent(mr, [tok[0] for tok in sent], delex_slots, repeated=True) off = 0 shift = 0 delex_tagged = [] for abst in absts: if abst.start == -1: # skip delex instances not actually occurring in this sentence continue delex_tagged.extend(sent[off + shift:abst.start + shift]) # POS tag for all delex'd slots is usually NNP, except for phone numbers and counts delex_pos = { 'count': 'CD', 'phone': 'CD' }.get(abst.slot, 'NNP') delex_tagged.append( [delex[abst.start], delex[abst.start], delex_pos]) off = abst.end shift += abst.surface_form.count(' ') delex_tagged.extend(sent[off + shift:]) delex_ref.append(delex_tagged) delex_refs.append(delex_ref) with codecs.open(delex_output_file + '.delex.txt', 'w', 'UTF-8') as fh: fh.write('\n'.join([ ' '.join([tok[0] for sent in ref for tok in sent]) for ref in delex_refs ])) write_output(delex_refs, 'lca', delex_output_file + '.delex.tag.lca.txt') write_output(delex_refs, 'collins', delex_output_file + '.delex.tag.collins.txt') return delex_refs
def create_fake_data(real_data, columns, score_type='nlg'): """Given some real data, create additional fake data, using human references and distorting them. Will start from scores provided, or default to best possible score. @param real_data: a real data set, as pd.DataFrame @param columns: list of columns for the fake data set @param score_type: switch between Likert scale 1-6 ('nlg') and HTER ('hter') @return: a fake data set, with the given columns, some of them empty """ def target_score(src_score, distort_step): if score_type == 'hter': return src_score + distort_step elif score_type == 'rank': return 1. # ignore scores for ranks return max(1, min(4., src_score - distort_step)) normalize = False best_score = 6. num_steps = 4 if score_type == 'hter': normalize = True best_score = 0. num_steps = 5 elif score_type == 'rank': best_score = 1. fake_data = pd.DataFrame(index=np.arange(len(real_data) * (num_steps + 1)), columns=columns) vocab = {} # add references as perfect data items for idx, row in enumerate(real_data.itertuples()): fake_data.loc[idx]['orig_ref'] = row.orig_ref fake_data.loc[idx]['system_ref'] = row.orig_ref fake_data.loc[idx]['mr'] = row.mr fake_data.loc[idx]['is_real'] = 0 for quant in ['naturalness', 'quality', 'informativeness']: fake_data.loc[idx][quant] = (getattr(row, quant) if ( hasattr(row, quant) and getattr(row, quant) is not None and not np.isnan(getattr(row, quant))) else best_score) for tok in tokenize(row.orig_ref).split(' '): vocab[tok] = vocab.get(tok, 0) + 1 lexicalizer = Lexicalizer(cfg={'mode': 'tokens'}) # default lexicalizer vocab = build_vocab(vocab) for distort_step in xrange(1, num_steps + 1): for idx, row in enumerate(real_data.itertuples(), start=distort_step * len(real_data)): fake_data.loc[idx]['orig_ref'] = row.orig_ref fake_data.loc[idx]['mr'] = row.mr fake_data.loc[idx]['is_real'] = 0 # delexicalize data da = DA.parse_cambridge_da(row.mr) sent, _, lex_instr = delex_sent(da, tokenize(row.orig_ref).split(' '), DELEX_SLOTS) ref_len = len(sent) # distort sent = distort_sent(sent, distort_step, vocab) # lexicalize again sent = lexicalizer._tree_to_sentence([(tok, None) for tok in sent], lex_instr) fake_data.loc[idx]['system_ref'] = ' '.join(sent) for quant in ['naturalness', 'quality', 'informativeness']: score = (getattr(row, quant) if ( hasattr(row, quant) and getattr(row, quant) is not None and not np.isnan(getattr(row, quant))) else best_score) score = target_score(score, distort_step) fake_data.loc[idx][quant] = (((score / ref_len) * 100) if normalize else score) return fake_data
def preprocess_sent(da, sent, delex_slots, delex_slot_names): sent = tokenize(sent.lower()).split(' ') if delex_slots: return delex_sent(da, sent, delex_slots, not delex_slot_names, delex_slot_names)[0] return sent