def process(f, np, head, text, heads2qp, stanford_deps): dep_key = "{0},{1}".format(np["HEAD_START"], np["HEAD_END"]) pos_tags = reconcile.getPOS(f) for dep in stanford_deps: if (dep["RELATION"] == "nsubj" or dep["RELATION"] == "nsubjpass") and dep_key == dep["DEP_SPAN"]: #we have a subj heads2qp[head].subj += 1 if dep["RELATION"] == "dobj" and dep_key == dep["DEP_SPAN"]: #direct obj heads2qp[head].dobj += 1 if dep["RELATION"] == "iobj" and dep_key == dep["DEP_SPAN"]: #indirect object heads2qp[head].iobj += 1 #apposition if dep["RELATION"] == "appos": if dep_key == dep["DEP_SPAN"]: heads2qp[head].appos_dep += 1 elif dep_key == dep["GOV_SPAN"]: heads2qp[head].appos_gov += 1 if dep["RELATION"] == "agent": if dep_key == dep["DEP_SPAN"]: heads2qp[head].agent += 1 heads2qp[head].agent_verbs.append(dep["GOV"]) if dep["RELATION"] == "amod": if dep_key == dep["GOV_SPAN"]: heads2qp[head].adj_mod += 1 if dep["RELATION"] == "advmod": if dep_key == dep["GOV_SPAN"]: heads2qp[head].adv_mod += 1 if dep["RELATION"] == "nn": if dep_key == dep["GOV_SPAN"]: heads2qp[head].nn_mod += 1 dep_start = int(dep["DEP_SPAN"].split(",")[0]) dep_end = int(dep["DEP_SPAN"].split(",")[1]) tag = pos_tags.getAnnotBySpan(dep_start, dep_end) if tag is not None: if tag["TAG"] in ("NNP", "NNPS"): heads2qp[head].prp_mod += 1 elif tag["TAG"] in ("NN", "NNS"): heads2qp[head].nom_mod += 1 if dep["RELATION"] == "num": if dep_key == dep["GOV_SPAN"]: heads2qp[head].num_mod += 1 if dep["RELATION"] == "poss": #the possessed if dep_key == dep["GOV_SPAN"]: heads2qp[head].poss_mod += 1 elif dep_key == dep["DEP_SPAN"]: #the possessor heads2qp[head].is_poss += 1 #NOTE prep_ can appear more than once for a single noun, which throws #off percentages if dep["RELATION"] in ("prep_of"): if dep_key == dep["GOV_SPAN"]: heads2qp[head].prep_mod += 1 if dep["RELATION"] in ("rcmod"): if dep_key == dep["GOV_SPAN"]: heads2qp[head].rc_mod += 1 #NOTE: modification levels should be low if (text == "the " + head) or \ (text == "that " + head) or \ (text == "this " + head) or \ (text == "those " + head) or \ (text == "these " + head): heads2qp[head].bare_definite += 1 if (text.startswith("the ")): heads2qp[head].definite += 1 if (text.startswith("a ")): heads2qp[head].indefinite += 1
def __str__(self): s = "{0} : [{1}]".format(self.text, ", ".join(self.gold_buddies)) return s if __name__ == "__main__": if len(sys.argv) < 2: print("Usage: %s <gold-document> <unannotated-documents-list>" % (sys.argv[0])) sys.exit(1) #read in gold chains gold_chains = reconcile.getGoldChains(sys.argv[1]) gold_chain_text = defaultdict(list) pos_tags = reconcile.getPOS(sys.argv[1]) tokens = reconcile.getTokens(sys.argv[1]) for key in list(gold_chains.keys()): for mention in gold_chains[key]: if mention.pprint() not in [ x.pprint() for x in gold_chain_text[key] ]: #np_pos = [x.getATTR("TAG") for x in \ # pos_tags.getOverlapping(mention)] #np_tok = [x.getText() for x in \ # tokens.getOverlapping(mention)] #mention.setProp("TAGS", np_pos) #mention.setProp("TOKENS", np_tok) gold_chain_text[key].append(mention) #print mention.pprint()
if len(sys.argv) < 2: print("Usage: %s <filelist>" % (sys.argv[0])) sys.exit(1) #let's look at some noun phrases for each domain. files = [] with open(sys.argv[1], 'r') as fileList: files.extend( [x for x in fileList.readlines() if not x.startswith("#")]) for f in files: f = f.strip() print("Working on file: {0}".format(f)) allNPs = {} nps = reconcile.getNPs(f) pos = reconcile.getPOS(f) stanford_nes = reconcile.getNEs(f) sundance_nes = reconcile.getSundanceNEs(f) for np in nps: key = "{0},{1}".format(np.getStart(), np.getEnd()) text = utils.textClean(np.getText().replace("\n", " ")).lower() tokens = text.split() if len(tokens) == 1: continue if key not in list(allNPs.keys()): allNPs[key] = NP(np.getText().replace("\n", " ")) allNPs[key].start = np.getStart() allNPs[key].end = np.getEnd()
def process_syntax(f, np, head, text, head2qp, stanford_deps): #TODO check for cases where dep_key isn't found might be an error in #finding the head bytespan dep_key = "{0},{1}".format(np["HEAD_START"], np["HEAD_END"]) pos_tags = reconcile.getPOS(f) already_subj_this_time = False already_dobj_this_time = False already_iobj_this_time = False already_bd_this_time = False already_appos_this_time = False already_appos_this_time2 = True already_modded_once = False already_poss_once = False already_of_once = False for dep in stanford_deps: if (dep["RELATION"] == "nsubj" or dep["RELATION"] == "nsubjpass") \ and dep_key == dep["DEP_SPAN"] and not already_subj_this_time: #we have a subj head2qp[head].subj += 1 already_subj_this_time = True if dep["RELATION"] == "dobj" and dep_key == dep[ "DEP_SPAN"] and not already_dobj_this_time: #direct obj head2qp[head].dobj += 1 already_dobj_this_time = True if dep["RELATION"] == "iobj" and dep_key == dep[ "DEP_SPAN"] and not already_iobj_this_time: #indirect object head2qp[head].iobj += 1 already_iobj_this_time = True #apposition if dep["RELATION"] == "appos": if dep_key == dep["DEP_SPAN"]: head2qp[head].appos_dep += 1 already_appos_this_time = True elif dep_key == dep["GOV_SPAN"]: head2qp[head].appos_gov += 1 already_appos_this_time2 = True if dep["RELATION"] == "agent": if dep_key == dep["DEP_SPAN"]: head2qp[head].agent += 1 head2qp[head].agent_verbs.append(dep["GOV"]) if dep["RELATION"] == "amod": if dep_key == dep["GOV_SPAN"]: head2qp[head].adj_mod += 1 #already_modded_once = True if dep["RELATION"] == "advmod": if dep_key == dep["GOV_SPAN"]: head2qp[head].adv_mod += 1 #already_modded_once = True if dep["RELATION"] == "nn": if dep_key == dep["GOV_SPAN"]: head2qp[head].nn_mod += 1 already_modded_once = True dep_start = int(dep["DEP_SPAN"].split(",")[0]) dep_end = int(dep["DEP_SPAN"].split(",")[1]) tag = pos_tags.getAnnotBySpan(dep_start, dep_end) if tag is not None: if tag["TAG"] in ("NNP", "NNPS"): head2qp[head].prp_mod += 1 elif tag["TAG"] in ("NN", "NNS"): head2qp[head].nom_mod += 1 if dep["RELATION"] == "num": if dep_key == dep["GOV_SPAN"]: head2qp[head].num_mod += 1 already_modded_once = True if dep["RELATION"] == "poss": #the possessed if dep_key == dep["GOV_SPAN"] and not already_poss_once: head2qp[head].poss_mod += 1 already_poss_once = True already_modded_once = True elif dep_key == dep["DEP_SPAN"]: #the possessor head2qp[head].is_poss += 1 #NOTE prep_ can appear more than once for a single noun, which throws #off percentages if dep["RELATION"] in ("prep_of") and not already_of_once: if dep_key == dep["GOV_SPAN"]: head2qp[head].prep_mod += 1 already_of_once = True if dep["RELATION"] in ("rcmod"): if dep_key == dep["GOV_SPAN"]: head2qp[head].rc_mod += 1 if already_modded_once: head2qp[head].one_premod += 1 #NOTE: modification levels should be low if (text == "the " + head) or \ (text == "that " + head) or \ (text == "this " + head) or \ (text == "those " + head) or \ (text == "these " + head): if not already_bd_this_time: head2qp[head].bare_definite += 1 already_bd_this_time = True if (text.startswith("the ")): head2qp[head].definite += 1 if (text.startswith("a ") or text.startswith("an ")): head2qp[head].indefinite += 1 if already_appos_this_time or already_appos_this_time2: head2qp[head].ind_no_app += 1
def process(f, head2qp, annotated_file): stanford_deps = reconcile.getStanfordDep(f) pos = reconcile.getPOS(f) if annotated_file: nps = reconcile.getNPs(f) for np in nps: head = None text = None if PRONOUNS: if qp_utils.isPronoun(np): head = np.getText().lower() text = np.getText() else: continue else: if qp_utils.isNominal(np, pos): text = utils.textClean(np.getText()) np_tags = pos.getSubset(np.getStart(), np.getEnd()) head = utils.textClean( qp_utils.getHead2(text.lower(), np_tags)) else: continue #bookkeeping if head not in list(head2qp.keys()): head2qp[head] = QuasiPronoun(head) head2qp[head].updateCount(True) head2qp[head].updateDocs(f, True) else: head2qp[head].updateDocs(f, True) head2qp[head].updateCount(True) if USE_GOLD: gold_chains = reconcile.getGoldChains(f) process_gold(f, np, head, text, head2qp, gold_chains) process_syntax(f, np, head, text, head2qp, stanford_deps) else: stanford_nps = reconcile.getStanfordNPs(f) for np in stanford_nps: if PRONOUNS: if np["is_pronoun"]: head = np.getText().lower() text = np.getText() else: continue else: #skip some problems with the parser or numbers if np["HEAD"].startswith("$") or np["HEAD"].endswith( "%") or np["HEAD"] == ".": continue if np["is_nominal"]: text = utils.textClean(np.getText()) head = np["HEAD"].lower() else: continue #bookkeeping if head not in list(head2qp.keys()): head2qp[head] = QuasiPronoun(head) head2qp[head].updateDocs(f, False) head2qp[head].updateCount(False) else: head2qp[head].updateDocs(f, False) head2qp[head].updateCount(False) process_syntax(f, np, head, text, head2qp, stanford_deps)
fileList = open(sys.argv[1], 'r') anaphoric_nominals = annotation_set.AnnotationSet("anaphoric_noms") existential_nominals = annotation_set.AnnotationSet("exist_noms") for f in fileList: f = f.strip() if f.startswith("#"): continue print("Working on %s" % f) gold_annots = reconcile.parseGoldAnnots(f) gold_chains = reconcile.getGoldChains(f) response_nps = reconcile.getNPs_annots(f) pos = reconcile.getPOS(f, True) reconcile.addSundanceProps(f, response_nps) utils.match_nps(gold_annots, response_nps) for g in gold_annots: if g.getATTR("GOLD_TYPE") != "NOM": continue for r in response_nps: if g.getATTR("MATCHED") == r.getID(): g.addProps(r.getProps()) if g.getATTR("GRAMMAR") == "SUBJECT" or g.getATTR( "SUN_ROLE") == "SUBJ": g.setProp("S_VERB", reconcile.getSubjVerb(g, pos))
def gold_annotations(f): """process the file with gold annotations""" global virtual_pronouns, total_counts, virtual_pronoun_heads, \ nominal_base_antecedent, distance_from_antecedent doc = Document(f) gold_chains = reconcile.getGoldChains(f) #adding in Sundance nes. nes = reconcile.getNEs(f, True) add_reconcile_semantic_class(gold_chains, nes) #adding in Reconcile pos too. pos = reconcile.getPOS(f, True) #getting the docs nps reconcile_nps = reconcile.getNPs_annots(f) #getting sundance nps sundance_nps = reconcile.getSundanceNPs(f) add_sundance_nps(gold_chains, sundance_nps) original_text_heads = {} # just getting the heads original_text = defaultdict(list) # for getting total doc counts later. nominal2chains = defaultdict( list) # the chains that a given nominal appears. for chain in list(gold_chains.keys()): base_antecedent = True prev_annot = None antecedents = 0 for mention in gold_chains[chain]: #if the first antecedent in a chain, do not list it as anaphoric. if base_antecedent: if mention.getATTR("is_nominal") and not \ mention.getATTR("GOLD_SINGLETON"): text = mention.getText() text_lower = mention.getATTR("TEXT_CLEAN").lower() docs_appeared[text_lower].append(f) nominal_base_antecedent[text_lower] = \ nominal_base_antecedent.get(text_lower, 0) + 1 original_text[text_lower].append(text) #take note that this chain contained this nominal nominal2chains[text_lower].append(chain) #take note of the gold semantic class gold_semantic_class[text_lower].append( mention.getATTR("GOLD_SEMANTIC")) #reconcile's semantic class reconcile_semantic_class[text_lower].append( mention.getATTR("NE_CLASS")) #sundance's semantic class sun_semantic_class[text_lower].append( mention.getATTR("SUN_SEMANTIC")) number_gold_antecedents[text_lower].append(antecedents) #get verb stats if mention.getATTR("ROLE") == "SUBJ": verb = reconcile.getSubjVerb(mention, pos) if verb != None: subj_verbs[text_lower].append(verb.lower()) elif mention.getATTR("ROLE") == "DOBJ": verb = reconcile.getObjVerb(mention, pos) if verb != None: obj_verbs[text_lower].append(verb.lower()) base_antecedent = False prev_annot = mention antecedents += 1 continue if mention.getATTR("is_nominal"): text = mention.getText() text_lower = mention.getATTR("TEXT_CLEAN").lower() head_text = mention.getATTR("HEAD_TEXT") original_text[text_lower].append(text) virtual_pronouns[text_lower] = \ virtual_pronouns.get(text_lower, 0) + 1 virtual_pronoun_heads[head_text.lower()] = \ virtual_pronoun_heads.get(head_text.lower(), 0) + 1 #the semantic class Reconcile puts this in. reconcile_semantic_class[text_lower].append( mention.getATTR("NE_CLASS")) #register this doc as containing this np. docs_appeared[text_lower].append(f) #take note that this chain contained this nominal nominal2chains[text_lower].append(chain) #take note of the gold semantic class gold_semantic_class[text_lower].append( mention.getATTR("GOLD_SEMANTIC")) #the number of possible correct antecedents for this anaphor number_gold_antecedents[text_lower].append(antecedents) #sundance's semantic class sun_semantic_class[text_lower].append( mention.getATTR("SUN_SEMANTIC")) # subject verb statistics if mention.getATTR("ROLE") == "SUBJ": verb = reconcile.getSubjVerb(mention, pos) subj_verbs[text_lower].append(verb.lower()) elif mention.getATTR("ROLE") == "DOBJ": verb = reconcile.getObjVerb(mention, pos) obj_verbs[text_lower].append(verb.lower()) #get the sentence distance from these two mentions. mention_sent = reconcile.getAnnotSentence(f, mention) prev_sent = reconcile.getAnnotSentence(f, prev_annot) if mention_sent > -1 and prev_sent > -1: distance_from_antecedent[text_lower].append(mention_sent - \ prev_sent) #get the TextTiling segment distance for the two mentions mention_seg = doc.getAnnotTile(mention) prev_seg = doc.getAnnotTile(prev_annot) if mention_seg > -1 and prev_seg > -1: focus_distance[text_lower].append(mention_seg - \ prev_seg) #getting the distribution of closest antecedent types for a #given nominal if prev_annot.getATTR("is_nominal"): nominals2type[text_lower]["nominal"] = \ nominals2type[text_lower].get("nominal",0) + 1 elif prev_annot.getATTR("is_pronoun"): nominals2type[text_lower]["pronoun"] = \ nominals2type[text_lower].get("pronoun",0) + 1 else: nominals2type[text_lower]["proper"] = \ nominals2type[text_lower].get("proper",0) + 1 prev_annot = mention antecedents += 1 #for key in nominal2chains.keys(): # print "%d : %s (doc: %s)" % (len(list(set(nominal2chains[key]))), key, # doc) #update the total counts. for key in list(original_text.keys()): for text in list(set(original_text[key])): total_counts[key] = total_counts.get(key, 0) + doc.getWordCounts(text) #the head counts for key in list(virtual_pronoun_heads.keys()): total_counts_heads[key] = total_counts_heads.get(key, 0) + \ doc.getWordCounts(key)