def evaluate_classifier_on_instances(inst_list, classifier, feat_list, pos_class_matrix, gold_tagmap=None): """ Given a list of instances, do the evaluation on them. :param inst_list: :param classifier: :param tagger: :return: """ pd = load_posdict() if (CLASS_FEATS_DICT in feat_list) or (CLASS_FEATS_PDICT in feat_list) or (CLASS_FEATS_NDICT in feat_list) else False matches = 0 compares = 0 for inst in inst_list: sup_postier = gloss_tag_tier(inst, tag_method=INTENT_POS_MANUAL) if sup_postier is None: continue gw_tier = gloss(inst) classify_gloss_pos(inst, classifier, posdict=pd, feat_prev_gram=CLASS_FEATS_PRESW in feat_list, feat_next_gram=CLASS_FEATS_NEXSW in feat_list, feat_dict=CLASS_FEATS_DICT in feat_list, feat_prev_gram_dict=CLASS_FEATS_PDICT in feat_list, feat_next_gram_dict=CLASS_FEATS_NDICT in feat_list, feat_suffix=CLASS_FEATS_SUF in feat_list, feat_prefix=CLASS_FEATS_PRE in feat_list, feat_morph_num=CLASS_FEATS_NUMSW in feat_list, feat_has_number=CLASS_FEATS_NUM in feat_list, feat_basic=CLASS_FEATS_SW in feat_list) cls_postier = gloss_tag_tier(inst, tag_method=INTENT_POS_CLASS) for cls_tag in cls_postier: word = xigt_find(gw_tier, id=cls_tag.alignment) sup_tag = xigt_find(sup_postier, alignment=cls_tag.alignment) if sup_tag is None: continue else: sup_tag_v = sup_tag.value() if gold_tagmap is not None: sup_tag_v = gold_tagmap.get(sup_tag_v) pos_class_matrix.add(sup_tag_v, cls_tag.value()) if cls_tag.value() == sup_tag_v: matches += 1 compares += 1 return matches, compares, matches/compares*100
def word_align_test(self): """ Test that the gloss has been automatically aligned at the word level correctly. """ at = Alignment() for gw in gloss(self.igt): gw_idx = item_index(gw) lw = xigt_find(self.igt, id=gw.alignment) if lw is not None: at.add((gw_idx, item_index(lw))) self.assertEqual(at, Alignment([(1,1),(2,2),(3,3),(4,4)]))
def gather_gloss_pos_stats(inst, subword_dict, feat_list): """ Given an instance, look for the gloss pos tags, and save the statistics about them, so that we can filter by the number of times each kind was seen later. :param inst: Instance to process. :type inst: RGIgt :param subword_dict: This dictionary will record the number of times each (word, TAG) pair has been seen. :type subword_dict: SubwordDict :param gram_tag_dict: This dictionary will record the number of times individual grams are seen. :type gram_tag_dict: TwoLevelCountDict """ # Grab the gloss POS tier... gpos_tier = gloss_tag_tier(inst) lpos_tier = lang_tag_tier(inst) gw_tier = gloss(inst) if CLASS_FEATS_ALN in feat_list: heur_align_inst(inst) get_trans_glosses_alignment(inst, aln_method=INTENT_ALN_HEUR) # If there are POS tags on the language line but not the gloss line... if gpos_tier is None and lpos_tier is not None: add_gloss_lang_alignments(inst) project_lang_to_gloss(inst) gpos_tier = gloss_tag_tier(inst) # If this tier exists, then let's process it. if gpos_tier is not None: # Iterate over each gloss POS tag... for i, gw in enumerate(gw_tier): tag = xigt_find(inst, alignment=gw.id) if tag is None: continue prev_word = gw_tier[i-1].value().lower() if i > 0 else None next_word = gw_tier[i+1].value().lower() if i < len(gw_tier)-1 else None if CLASS_FEATS_ALN in feat_list: subword_dict.add_word_tag(gw.value().lower(), tag.value(), prev_word, next_word)
def extract_sents_from_inst(inst: Igt, out_src, out_tgt, aln_method=None, no_alignment_heur = True, sent_type=SENT_TYPE_T_G): """ Extract parallel sentences from an instance. Either: 1) Translation--Gloss 2) Translation--Language """ # ------------------------------------------- # 1) Get the source string (translation) # ------------------------------------------- src_str = tier_text(trans(inst), remove_whitespace_inside_tokens=True).lower() # ------------------------------------------- # 2) Decide whether the target string is gloss or language. # ------------------------------------------- if sent_type == SENT_TYPE_T_L: tgt_str = tier_text(lang(inst), remove_whitespace_inside_tokens=True).lower() elif sent_type == SENT_TYPE_T_G: tgt_str = tier_text(gloss(inst), remove_whitespace_inside_tokens=True).lower() else: raise Exception("Invalid sent type") # ------------------------------------------- # 3) Write the choice out to disk. # ------------------------------------------- out_src.write(src_str + '\n') out_tgt.write(tgt_str + '\n') out_src.flush() out_tgt.flush() # ------------------------------------------- # 4) Add heuristic alignments, if asked for. # ------------------------------------------- if not no_alignment_heur: pairs = get_trans_aligned_wordpairs(inst, aln_method=aln_method, add_align=True, sent_type=sent_type) for src_word, tgt_word in pairs: out_src.write(src_word.lower() + '\n') out_tgt.write(tgt_word.lower() + '\n')
def naacl_to_xigt(naacl_path): """ Convert the NAACL format to XIGT. :param naacl_path: """ content = open(naacl_path, 'r').read() # First, collect all the instances. instances = re.findall('Igt_id[\s\S]+?Q6.*Answer', content) xc = XigtCorpus() for instance_txt in instances: # id = re.search('Igt_id=([\S]+)', instance_txt).group(1) inst = Igt(id='i{}'.format(len(xc))) lang_raw, gloss_raw, trans_raw = instance_txt.split('\n')[1:4] # Now, create the raw tier... raw_tier = Tier(id=gen_tier_id(inst, 'r'), type='odin', attributes={STATE_ATTRIBUTE:RAW_STATE}) raw_tier.append(Item(id=ask_item_id(raw_tier), text=lang_raw, attributes={ODIN_TAG_ATTRIBUTE:ODIN_LANG_TAG})) raw_tier.append(Item(id=ask_item_id(raw_tier), text=gloss_raw, attributes={ODIN_TAG_ATTRIBUTE:ODIN_GLOSS_TAG})) raw_tier.append(Item(id=ask_item_id(raw_tier), text=trans_raw, attributes={ODIN_TAG_ATTRIBUTE:ODIN_TRANS_TAG})) inst.append(raw_tier) xc.append(inst) # Generate the clean/normal tiers, but without any cleaning. generate_normal_tier(inst, clean=False) # Lang Dependency representation handling... lang_ds_str = re.search('Q6:([\s\S]+?)Q6:', instance_txt).group(1) lang_ds_lines = lang_ds_str.split('\n')[5:-3] try: lang_dt = parse_naacl_dep(lang(inst), lang_ds_lines) create_dt_tier(inst, lang_dt, lang(inst), parse_method=INTENT_POS_MANUAL) except TreeError as te: pass except IndexError as ie: pass # Eng DS handling... eng_ds_str = re.search('Q3:([\s\S]+?)Q3:', instance_txt).group(1) eng_ds_lines = eng_ds_str.split('\n')[2:-3] try: eng_dt = parse_naacl_dep(trans(inst), eng_ds_lines) create_dt_tier(inst, eng_dt, trans(inst), parse_method=INTENT_POS_MANUAL) except TreeError as te: pass except IndexError as ie: pass except ValueError as ve: pass # Add Alignment... biling_aln_str = re.search('Q5:([\s\S]+?)Q5:', instance_txt).group(1) biling_aln_lines = biling_aln_str.split('\n')[4:-3] trans_offset = trans_raw.startswith(' ') gloss_offset = gloss_raw.startswith(' ') try: a = Alignment() for line in biling_aln_lines: gloss_s, trans_s = line.split()[0:2] if '.' in gloss_s: continue gloss_i = int(gloss_s) for trans_token in trans_s.split(','): trans_i = int(trans_token) if trans_i == 0: continue else: if trans_offset: trans_i -= 1 if gloss_offset: gloss_i -= 1 a.add((trans_i, gloss_i)) except: pass set_bilingual_alignment(inst, trans(inst), gloss(inst), a, aln_method=INTENT_ALN_MANUAL) return xc
def line_test(self): """ Test that lines are rendered correctly. """ self.assertEqual(tier_text(gloss(self.igt)), 'I-Nom child-Dat rice-Acc eat-Caus-Pst-Dec') self.assertEqual(tier_text(trans(self.igt)), 'I made the child eat rice')
def convert_pml(aln_path, out_path, hindi=True): if hindi: igt_data = retrieve_hindi() else: igt_data = retrieve_naacl() a_root = load_xml(aln_path) doc_a = a_root.find(".//reffile[@name='document_a']").get('href') doc_b = a_root.find(".//reffile[@name='document_b']").get('href') doc_a = os.path.join(os.path.join(os.path.dirname(aln_path), doc_a)) doc_b = os.path.join(os.path.join(os.path.dirname(aln_path), doc_b)) # Load the sentences for each document. a_sents, a_glossed = load_sents(doc_a) b_sents, b_glossed = load_sents(doc_b) sent_alignments = a_root.findall(".//body/LM") assert (a_glossed and not b_glossed) or (b_glossed and not a_glossed), "Only one file should have glosses" xc = XigtCorpus() for sent_alignment in sent_alignments: # Get the sentence id... aln_id = sent_alignment.attrib.get('id') a_snt_id = re.search('^.+?-(.*)$', aln_id).group(1) if a_snt_id not in igt_data: continue # Get the text and tokens from the naacl data. pre_txt, lang_txt, gloss_txt, trans_txt = igt_data[a_snt_id] lang_tokens = lang_txt.split() gloss_tokens = gloss_txt.split() trans_tokens = trans_txt.split() a_snt_ref = sent_alignment.find('./tree_a.rf').text.split('#')[1] b_snt_ref = sent_alignment.find('./tree_b.rf').text.split('#')[1] word_alignments = sent_alignment.findall('./node_alignments/LM') a_snt, a_edges = a_sents[a_snt_ref] b_snt, b_edges = b_sents[b_snt_ref] assert isinstance(a_snt, Sentence) assert isinstance(b_snt, Sentence) # ------------------------------------------- # Skip sentences if they are not found for whatever reason # ------------------------------------------- if not a_snt or not b_snt: continue # ------------------------------------------- # Start constructing the IGT Instance. # ------------------------------------------- trans_snt, trans_indices = a_snt, a_edges gloss_snt, gloss_indices = b_snt, b_edges if a_glossed: trans_snt, trans_indices = b_snt, b_edges gloss_snt, gloss_indices = a_snt, a_edges # Hindi stuff... if hindi: lang_tokens = [w.text for w in gloss_snt] lang_postags = [w.pos for w in gloss_snt] lang_txt = ' '.join(lang_tokens) trans_tokens = [w.text for w in trans_snt] trans_postags = [w.pos for w in trans_snt] trans_txt = ' '.join(trans_tokens) gloss_tokens = [w.gloss if w.gloss else 'NULL' for w in gloss_snt] gloss_postags = lang_postags gloss_txt = ' '.join(gloss_tokens) inst = Igt(id=re.sub('s-', 'igt', a_snt_ref)) nt = Tier(type=ODIN_TIER_TYPE, id=NORM_ID, attributes={STATE_ATTRIBUTE:NORM_STATE}) ll = Item(id='n1', attributes={ODIN_TAG_ATTRIBUTE:ODIN_LANG_TAG}, text=lang_txt) gl = Item(id='n2', attributes={ODIN_TAG_ATTRIBUTE:ODIN_GLOSS_TAG}, text=gloss_txt) tl = Item(id='n3', attributes={ODIN_TAG_ATTRIBUTE:ODIN_TRANS_TAG}, text=trans_txt) nt.extend([ll,gl,tl]) inst.append(nt) # ------------------------------------------- # Handle the phrase tiers # ------------------------------------------- generate_lang_phrase_tier(inst) generate_trans_phrase_tier(inst) def process_postags(sent, tokens): postags = [] for i, token in enumerate(tokens): word = sent.getorder(i+1) if word is None: postags.append(None) else: postags.append(word.pos) return postags # ------------------------------------------- # Now, handle the translation words. # ------------------------------------------- tt = create_word_tier(ODIN_TRANS_TAG, trans_tokens, trans_phrase(inst)[0]) inst.append(tt) if not hindi: trans_postags = process_postags(trans_snt, trans_tokens) add_pos_tags(inst, tt.id, trans_postags, tag_method=INTENT_POS_MANUAL) # ------------------------------------------- # Handle the words tiers... # ------------------------------------------- wt = create_word_tier(ODIN_LANG_TAG, lang_tokens, lang_phrase(inst)[0]) gwt= create_word_tier(ODIN_GLOSS_TAG, gloss_tokens, gl) inst.extend([wt, gwt]) # Quickly set the alignment for the gloss words. for w, gw in zip(wt, gwt): gw.alignment = w.id if not hindi: lang_postags = process_postags(gloss_snt, gloss_tokens) gloss_postags = lang_postags add_pos_tags(inst, wt.id, lang_postags, tag_method=INTENT_POS_MANUAL) add_pos_tags(inst, gwt.id, gloss_postags, tag_method=INTENT_POS_MANUAL) create_dt_tier(inst, assemble_ds(gloss_snt, gloss_indices), wt, INTENT_DS_MANUAL) create_dt_tier(inst, assemble_ds(trans_snt, trans_indices), tt, INTENT_DS_MANUAL) # ------------------------------------------- # Now, the word alignments. # ------------------------------------------- a = Alignment() for word_alignment in word_alignments: a_ref = word_alignment.find('./a.rf').text.split('#')[1] b_ref = word_alignment.find('./b.rf').text.split('#')[1] a_word = a_snt.getid(a_ref) b_word = b_snt.getid(b_ref) if a_word is None or b_word is None: continue if not hindi: a_idx = a_word.order b_idx = b_word.order else: a_idx = a_snt.index(a_word)+1 b_idx = b_snt.index(b_word)+1 # Make sure the gloss is in the if a_glossed: trans_idx = b_idx lang_idx = a_idx else: trans_idx = a_idx lang_idx = b_idx a.add((trans_idx, lang_idx)) set_bilingual_alignment(inst, trans(inst), lang(inst), a, INTENT_ALN_MANUAL) set_bilingual_alignment(inst, trans(inst), gloss(inst), a, INTENT_ALN_MANUAL) xc.append(inst) with open(out_path, 'w', encoding='utf-8') as f: xigtxml.dump(f, xc)
def test_line_lengths(self): inst = self.xc[1] self.assertEqual(5, len(gloss(inst))) self.assertEqual(6, len(lang(inst)))
def evaluate_pos_projections_on_file(lang, xc, plma, pos_proj_matrix, tagger, gold_tagmap=None, trans_tagmap=None, outstream=sys.stdout): """ :type plma: PerLangMethodAccuracies :type pos_proj_matrix: POSMatrix """ new_xc = XigtCorpus(xc.id) for inst in xc: gtt = gloss_tag_tier(inst, INTENT_POS_MANUAL) ttt = trans_tag_tier(inst, INTENT_POS_MANUAL) m_aln = get_trans_gloss_alignment(inst, INTENT_ALN_MANUAL) # Only continue if we have manual gloss tags, trans tags, and manual alignment. if gtt is None or m_aln is None or ttt is None: continue # Get the heuristic alignment... h_aln = heur_align_inst(inst) # And tag the translation line. tag_trans_pos(inst, tagger=tagger) # Now, iterate through each alignment method and set of tags. for aln_method in [INTENT_ALN_MANUAL, INTENT_ALN_HEUR]: for trans_tag_method in [INTENT_POS_MANUAL, INTENT_POS_TAGGER]: project_trans_pos_to_gloss(inst, aln_method=aln_method, trans_tag_method=trans_tag_method) proj_gtt = gloss_tag_tier(inst, tag_method=INTENT_POS_PROJ) # Go through each word in the gloss line and, if it has a gold # tag, was it correct? matches = 0 compares = 0 for gw in gloss(inst): gold_tag = xigt_find(gtt, alignment=gw.id) proj_tag = xigt_find(proj_gtt, alignment=gw.id) if gold_tag is not None: gold_tag_v = gold_tag.value() # Remap the tags if asked... if gold_tagmap is not None: try: gold_tag_v = gold_tagmap.get(gold_tag_v) except TagMapException: pass if proj_tag is None: proj_str = '**UNK' else: proj_str = proj_tag.value() if trans_tagmap is not None: # Try to remap the tag, but keep it if it can't be remapped. try: proj_str = trans_tagmap.get(proj_str) except TagMapException: pass pos_proj_matrix.add(gold_tag_v, proj_str) if proj_tag is not None and proj_str == gold_tag_v: matches += 1 compares += 1 plma.add(lang, '{}:{}'.format(aln_method, trans_tag_method), matches, compares) outstream.write('{}\n'.format(plma)) return new_xc