def build_test_file(self, out_path=OUT_PATH, remove_singleton=True, print_all_mentions=False, debug=None): """ Build a test file to supply to the coreference scoring perl script """ print("๐ Building test file") self._prepare_clusters() self.dataloader.dataset.no_targets = True if not print_all_mentions: print("๐ Build coreference clusters") cur_m = 0 for sample_batched, mentions_idx, n_pairs_l in zip(self.dataloader, self.mentions_idx, self.n_pairs): scores, max_i = self.get_max_score(sample_batched) for m_idx, ind, n_pairs in zip(mentions_idx, max_i, n_pairs_l): if ind < n_pairs : # the single score is not the highest, we have a match ! prev_idx = m_idx - n_pairs + ind if debug is not None and (debug == -1 or debug == prev_idx or debug == m_idx): m1_doc, m1_idx = self.flat_m_idx[m_idx] m1 = self.docs[m1_doc]['mentions'][m1_idx] m2_doc, m2_idx = self.flat_m_idx[prev_idx] m2 = self.docs[m2_doc]['mentions'][m2_idx] print("We have a match between:", m1, "(" + str(m1_idx) + ")", "and:", m2, "(" + str(m2_idx) + ")") self._merge_coreference_clusters(prev_idx, m_idx) if remove_singleton: self.remove_singletons_clusters() self.dataloader.dataset.no_targets = False print("๐ Construct test file") out_str = "" for doc, d_tokens, d_lookup, d_m_loc, d_m_to_c in zip(self.docs, self.tokens, self.lookup, self.m_loc, self.mention_to_cluster): out_str += u"#begin document (" + doc['name'] + u"); part " + doc['part'] + u"\n" for utt_idx, (c_tokens, c_lookup) in enumerate(zip(d_tokens, d_lookup)): for i, (token, lookup) in enumerate(zip(c_tokens, c_lookup)): out_coref = u"" for m_str, mention, mention_cluster in zip(doc['mentions'], d_m_loc, d_m_to_c): m_start, m_end, m_utt, m_idx, m_doc = mention if mention_cluster is None: pass elif m_utt == utt_idx: if m_start in lookup: out_coref += u"|" if out_coref else u"" out_coref += u"(" + unicode_(mention_cluster) if (m_end - 1) in lookup: out_coref += u")" else: out_coref += u"" elif (m_end - 1) in lookup: out_coref += u"|" if out_coref else u"" out_coref += unicode_(mention_cluster) + u")" out_line = doc['name'] + u" " + doc['part'] + u" " + unicode_(i) \ + u" " + token + u" " out_line += u"-" if len(out_coref) == 0 else out_coref out_str += out_line + u"\n" out_str += u"\n" out_str += u"#end document\n" # Write test file print("Writing in", out_path) with io.open(out_path, 'w', encoding='utf-8') as out_file: out_file.write(out_str)
def __init__(self, speaker_id, speaker_names=None): self.mentions = [] self.speaker_id = speaker_id if speaker_names is None: self.speaker_names = [unicode_(speaker_id)] elif isinstance(speaker_names, string_types): self.speaker_names = [speaker_names] elif len(speaker_names) > 1: self.speaker_names = speaker_names else: self.speaker_names = unicode_(speaker_names) self.speaker_tokens = [tok.lower() for s in self.speaker_names for tok in re.split(WHITESPACE_PATTERN, s)]
def display_clusters(self): ''' Print clusters informations ''' print(self.clusters) for key, mentions in self.clusters.items(): print("cluster", key, "(", ", ".join(unicode_(self.data[m]) for m in mentions), ")")
def get_mention_embeddings(self, mention, doc_embedding): ''' Get span (averaged) and word (single) embeddings of a mention ''' st = mention.sent mention_lefts = mention.doc[max(mention.start - 5, st.start):mention.start] mention_rights = mention.doc[mention.end:min(mention.end + 5, st.end)] head = mention.root.head spans = [ self.get_average_embedding(mention), self.get_average_embedding(mention_lefts), self.get_average_embedding(mention_rights), self.get_average_embedding(st), (unicode_(doc_embedding[0:8]) + "...", doc_embedding) ] words = [ self.get_word_embedding(mention.root), self.get_word_embedding(mention[0]), self.get_word_embedding(mention[-1]), self.get_word_in_sentence(mention.start - 1, st), self.get_word_in_sentence(mention.end, st), self.get_word_in_sentence(mention.start - 2, st), self.get_word_in_sentence(mention.end + 1, st), self.get_word_embedding(head) ] spans_embeddings_ = { "00_Mention": spans[0][0], "01_MentionLeft": spans[1][0], "02_MentionRight": spans[2][0], "03_Sentence": spans[3][0], "04_Doc": spans[4][0] } words_embeddings_ = { "00_MentionHead": words[0][0], "01_MentionFirstWord": words[1][0], "02_MentionLastWord": words[2][0], "03_PreviousWord": words[3][0], "04_NextWord": words[4][0], "05_SecondPreviousWord": words[5][0], "06_SecondNextWord": words[6][0], "07_MentionRootHead": words[7][0] } return (spans_embeddings_, words_embeddings_, np.concatenate([em[1] for em in spans], axis=0), np.concatenate([em[1] for em in words], axis=0))
def __str__(self): return '<utterances, speakers> \n {}\n<mentions> \n {}' \ .format('\n '.join(unicode_(i) + " " + unicode_(s) for i, s in zip(self.utterances, self.utterances_speaker)), '\n '.join(unicode_(i) + " " + unicode_(i.speaker) for i in self.mentions))
def read_corpus(self, data_path, debug=False): # this function holds the key to constructing the memory module for the conll corpus # find the discourse end marker, that holds the key to forming stories for memory inference print("๐ Reading files") for dirpath, _, filenames in os.walk(data_path): print("In", dirpath, os.path.abspath(dirpath)) file_list = [os.path.join(dirpath, f) for f in filenames if f.endswith(".v4_auto_conll") \ or f.endswith(".v4_gold_conll")] cleaned_file_list = [] for f in file_list: fn = f.split('.') if fn[1] == "v4_auto_conll": gold = fn[0] + "." + "v4_gold_conll" if gold not in file_list: cleaned_file_list.append(f) else: cleaned_file_list.append(f) #doc_list = parallel_process(cleaned_file_list, load_file) # what is the doc list ? #for docs in doc_list:#executor.map(self.load_file, cleaned_file_list): for file in cleaned_file_list : docs = load_file(file) for utts_text, utt_tokens, utts_corefs, utts_speakers, name, part in docs: print("Imported", name) if debug: print("utts_text", utts_text) print("utt_tokens", utt_tokens) print("utts_corefs", utts_corefs) print("utts_speakers", utts_speakers) print("name, part", name, part) self.utts_text += utts_text self.utts_tokens += utt_tokens self.utts_corefs += utts_corefs self.utts_speakers += utts_speakers self.utts_doc_idx += [len(self.docs_names)] * len(utts_text) self.docs_names.append((name, part)) print("utts_text size", len(self.utts_text)) print("utts_tokens size", len(self.utts_tokens)) print("utts_corefs size", len(self.utts_corefs)) print("utts_speakers size", len(self.utts_speakers)) print("utts_doc_idx size", len(self.utts_doc_idx)) print("๐ Building docs") for name, part in self.docs_names: self.docs.append(ConllDoc(name=name, part=part, nlp=None, blacklist=False, consider_speakers=True, embedding_extractor=self.embed_extractor, conll=CONLL_GENRES[name[:2]])) print("๐ Loading spacy model") try: spacy.info('en_core_web_sm') model = 'en_core_web_sm' except IOError: print("No spacy 2 model detected, using spacy1 'en' model") spacy.info('en') model = 'en' nlp = spacy.load(model) print("๐ Parsing utterances and filling docs") doc_iter = (s for s in self.utts_text) for utt_tuple in tqdm(zip(nlp.pipe(doc_iter), self.utts_tokens, self.utts_corefs, self.utts_speakers, self.utts_doc_idx)): spacy_tokens, conll_tokens, corefs, speaker, doc_id = utt_tuple if debug: print(unicode_(self.docs_names[doc_id]), "-", spacy_tokens) doc = spacy_tokens if debug: out_str = "utterance " + unicode_(doc) + " corefs " + unicode_(corefs) + \ " speaker " + unicode_(speaker) + "doc_id" + unicode_(doc_id) print(out_str.encode('utf-8')) self.docs[doc_id].add_conll_utterance(doc, conll_tokens, corefs, speaker, use_gold_mentions=self.use_gold_mentions) del spacy_tokens, conll_tokens, corefs,speaker, doc_id del nlp, doc_iter