def evaluate_instance(inst, classifier, tagger):
# Get the supervised POS tags...
"""
:param inst:
:type inst: RGIgt
:param classifier: MalletMaxent
:param tagger: StanfordPOSTagger
"""
sup_gloss_tier = pos_tag_tier(inst, GLOSS_WORD_ID) # We will incrementally build up the tag sequences...
sup_lang_tier = pos_tag_tier(inst, LANG_WORD_ID)
sup_tags = []
prj_tags = []
cls_tags = []
# If there are no supervised tags on the gloss line, but there are on the language line...
if sup_gloss_tier is None and sup_lang_tier is not None:
try:
add_gloss_lang_alignments(inst)
project_lang_to_gloss(inst)
sup_gloss_tier = pos_tag_tier(inst, GLOSS_WORD_ID)
except RGXigtException:
pass
if sup_gloss_tier:
# Do the classification
classify_gloss_pos(inst, classifier)
cls_tier = pos_tag_tier(inst, GLOSS_WORD_ID, tag_method=INTENT_POS_CLASS)
for sup_item in sup_gloss_tier:
word = xigt_find(inst, id=sup_item.alignment)
if not word:
continue
else:
word = word.value()
# prj_item = xigt_find(prj_tier, alignment=sup_item.alignment)
# if prj_item is None:
# prj_tag = 'UNK'
# else:
# prj_tag = prj_item.value()
cls_item = xigt_find(cls_tier, alignment=sup_item.alignment)
if cls_item is None:
cls_tag = 'UNK'
else:
cls_tag = cls_item.value()
sup_tags.append(POSToken(word, label=sup_item.value()))
# prj_tags.append(POSToken(word, label=prj_tag))
cls_tags.append(POSToken(word, label=cls_tag))
return sup_tags, cls_tags
def gather_gloss_pos_stats(inst, subword_dict, feat_list):
"""
Given an instance, look for the gloss pos tags, and save the statistics
about them, so that we can filter by the number of times each kind was
seen later.
:param inst: Instance to process.
:type inst: RGIgt
:param subword_dict: This dictionary will record the number of times each (word, TAG)
pair has been seen.
:type subword_dict: SubwordDict
:param gram_tag_dict: This dictionary will record the number of times individual grams are seen.
:type gram_tag_dict: TwoLevelCountDict
"""
# Grab the gloss POS tier...
gpos_tier = gloss_tag_tier(inst)
lpos_tier = lang_tag_tier(inst)
gw_tier = gloss(inst)
if CLASS_FEATS_ALN in feat_list:
heur_align_inst(inst)
get_trans_glosses_alignment(inst, aln_method=INTENT_ALN_HEUR)
# If there are POS tags on the language line but not the gloss line...
if gpos_tier is None and lpos_tier is not None:
add_gloss_lang_alignments(inst)
project_lang_to_gloss(inst)
gpos_tier = gloss_tag_tier(inst)
# If this tier exists, then let's process it.
if gpos_tier is not None:
# Iterate over each gloss POS tag...
for i, gw in enumerate(gw_tier):
tag = xigt_find(inst, alignment=gw.id)
if tag is None:
continue
prev_word = gw_tier[i-1].value().lower() if i > 0 else None
next_word = gw_tier[i+1].value().lower() if i < len(gw_tier)-1 else None
if CLASS_FEATS_ALN in feat_list:
subword_dict.add_word_tag(gw.value().lower(), tag.value(), prev_word, next_word)
def more_morph_align_test(self):
inst = self.xc[1]
# Align the gloss/lang words (Needed for aligning morphs)
add_gloss_lang_alignments(inst)
gloss_tokens = glosses(inst)
morph_tokens = morphemes(inst)
# Do the alignment
morph_align(gloss_tokens, morph_tokens)
# Assert that the glosses are aligned...
self.assertIsNotNone(gloss_tokens[0].alignment)
self.assertEquals(gloss_tokens[11].alignment, morph_tokens[6].id)
self.assertEquals(gloss_tokens[12].alignment, morph_tokens[6].id)
self.assertEquals(gloss_tokens[13].alignment, morph_tokens[7].id)
self.assertEquals(gloss_tokens[14].alignment, morph_tokens[7].id)
def enrich(**kwargs):
global classifier
if ARG_OUTFILE not in kwargs:
ENRICH_LOG.critical("No output file specified.")
sys.exit()
# =============================================================================
# Set up the alternate classifier path...
# =============================================================================
class_path = kwargs.get('class_path')
#===========================================================================
# Set up the different arguments...
#===========================================================================
inpath = kwargs.get(ARG_INFILE)
parse_args = kwargs.get(PARSE_VAR, [])
pos_args = kwargs.get(POS_VAR, [])
aln_args = kwargs.get(ALN_VAR, [])
max_parse_length = kwargs.get('max_parse_length', 10)
if not (parse_args or pos_args or aln_args):
ENRICH_LOG.warning("No enrichment specified. Basic processing only will be performed.")
#===========================================================================
# Sanity check the arguments.
#===========================================================================
# Check that alignment is asked for if projection is asked for.
if (ARG_POS_PROJ in pos_args or ARG_PARSE_PROJ in parse_args) and (not aln_args):
ENRICH_LOG.warn("You have asked for projection methods but have not requested " + \
"alignments to be generated. Projection may fail if alignment not already present in file.")
ENRICH_LOG.log(1000, 'Loading input file...')
with open(inpath, 'r', encoding='utf-8') as in_f:
corp = xigtxml.load(in_f, mode=INCREMENTAL)
# -------------------------------------------
# Initialize the English tagger if:
# A) "proj" option is selected for pos.
# B) "trans" option is given for pos.
# C) "heurpos" option is given for alignment.
# -------------------------------------------
s = None
if ARG_POS_PROJ in pos_args or ARG_POS_TRANS in pos_args or ARG_ALN_HEURPOS in aln_args:
ENRICH_LOG.log(1000, 'Initializing tagger...')
tagger = c.getpath('stanford_tagger_trans')
try:
s = StanfordPOSTagger(tagger)
except TaggerError as te:
ENRICH_LOG.critical(te)
sys.exit(2)
# -------------------------------------------
# Initialize the parser if:
# A) "trans" option is given for parse
# B) "proj" option is given for parse.
# -------------------------------------------
if ARG_PARSE_TRANS in parse_args or ARG_PARSE_PROJ in parse_args:
ENRICH_LOG.log(1000, "Intializing English parser...")
sp = stanford_parser.StanfordParser()
# -------------------------------------------
# Initialize the classifier if:
# A) "class" option is given for pos
# B) "heurpos" option is given for alignment.
# -------------------------------------------
m = None
if ARG_POS_CLASS in pos_args or ARG_ALN_HEURPOS in aln_args:
ENRICH_LOG.log(1000, "Initializing gloss-line classifier...")
p = load_posdict()
m = mallet_maxent.MalletMaxent(classifier)
# -- 1b) Giza Gloss to Translation alignment --------------------------------------
if ARG_ALN_GIZA in aln_args or ARG_ALN_GIZAHEUR in aln_args:
ENRICH_LOG.log(1000, 'Aligning gloss and translation lines using mgiza++...')
try:
if ARG_ALN_GIZAHEUR in aln_args:
giza_align_t_g(corp, resume=True, use_heur=True, symmetric=kwargs.get(ALN_SYM_VAR, SYMMETRIC_INTERSECT))
if ARG_ALN_GIZA in aln_args:
giza_align_t_g(corp, resume=True, use_heur=False, symmetric=kwargs.get(ALN_SYM_VAR, SYMMETRIC_INTERSECT))
except GizaAlignmentException as gae:
gl = logging.getLogger('giza')
gl.critical(str(gae))
raise gae
# -------------------------------------------
# Begin iterating through the corpus
# -------------------------------------------
for inst in corp:
feedback_string = 'Instance {:15s}: {{:20s}}{{}}'.format(inst.id)
reasons = []
inst_status = None
def fail(reason):
nonlocal inst_status, reasons
if reason not in reasons:
reasons.append(reason)
inst_status = 'WARN'
def success():
nonlocal inst_status
inst_status = 'OK'
# -------------------------------------------
# Define the reasons for failure
# -------------------------------------------
F_GLOSS_LINE = "NOGLOSS"
F_LANG_LINE = "NOLANG"
F_TRANS_LINE = "NOTRANS"
F_BAD_LINES = "BADLINES"
F_L_G_ALN = "L_G_ALIGN"
F_T_G_ALN = "G_T_ALIGN"
F_NO_TRANS_POS="NO_POS_TRANS"
F_PROJECTION = "PROJECTION"
F_UNKNOWN = "UNKNOWN"
F_PARSELEN = "OVER_MAX_LENGTH"
try:
# -------------------------------------------
# Get the different lines
# -------------------------------------------
def tryline(func):
nonlocal inst
try:
return func(inst)
except NoNormLineException as nnle:
return None
gl = tryline(gloss_line)
tls = tryline(trans_lines)
lls = tryline(lang_lines)
has_gl = gl is not None
has_tl = tls is not None
has_ll = lls is not None
has_all = lambda: (has_gl and has_tl and has_ll)
# -------------------------------------------
# Translation Line
# -------------------------------------------
if has_tl:
if ARG_POS_PROJ in pos_args or ARG_POS_TRANS in pos_args or ARG_ALN_HEURPOS in aln_args:
try:
tag_trans_pos(inst, s)
except CriticalTaggerError as cte:
ENRICH_LOG.critical(str(cte))
sys.exit(2)
if ARG_PARSE_PROJ in parse_args or ARG_PARSE_TRANS in parse_args:
if len(trans(inst)) <= max_parse_length:
parse_translation_line(inst, sp, pt=True, dt=True)
else:
fail(F_PARSELEN)
# 4) POS tag the gloss line --------------------------------------------
if has_gl:
if ARG_POS_CLASS in pos_args or ARG_ALN_HEURPOS in aln_args:
classify_gloss_pos(inst, m, posdict=p)
# -------------------------------------------
# Try getting alignments.
# -------------------------------------------
if has_gl and has_ll:
try:
add_gloss_lang_alignments(inst)
except GlossLangAlignException as glae:
fail(F_L_G_ALN)
if has_gl and has_tl:
if ARG_ALN_HEURPOS in aln_args:
heur_align_inst(inst, use_pos=True)
if ARG_ALN_HEUR in aln_args:
heur_align_inst(inst, use_pos=False)
# -------------------------------------------
# Now, do the necessary projection tasks.
# -------------------------------------------
# Project the classifier tags...
if has_ll and has_gl and ARG_POS_CLASS in pos_args:
try:
project_gloss_pos_to_lang(inst, tag_method=INTENT_POS_CLASS)
except GlossLangAlignException:
fail(F_L_G_ALN)
# -------------------------------------------
# Do the trans-to-lang projection...
# -------------------------------------------
if has_all():
proj_aln_method = ALN_ARG_MAP[kwargs.get('proj_aln', ARG_ALN_ANY)]
aln = get_trans_gloss_alignment(inst, aln_method=proj_aln_method)
if not aln or len(aln) == 0:
fail(F_T_G_ALN)
else:
# -------------------------------------------
# POS Projection
# -------------------------------------------
if ARG_POS_PROJ in pos_args:
trans_tags = trans_tag_tier(inst)
if not trans_tags:
fail(F_NO_TRANS_POS)
else:
project_trans_pos_to_gloss(inst)
try:
project_gloss_pos_to_lang(inst, tag_method=INTENT_POS_PROJ)
except GlossLangAlignException as glae:
fail(F_L_G_ALN)
# -------------------------------------------
# Parse projection
# -------------------------------------------
if ARG_PARSE_PROJ in parse_args:
try:
project_pt_tier(inst, proj_aln_method=proj_aln_method)
except PhraseStructureProjectionException as pspe:
fail(F_PROJECTION)
except NoAlignmentProvidedError as nape:
fail(F_T_G_ALN)
try:
project_ds_tier(inst, proj_aln_method=proj_aln_method)
except ProjectionException as pe:
fail(F_PROJECTION)
except NoAlignmentProvidedError as nape:
fail(F_T_G_ALN)
# Sort the tiers... ----------------------------------------------------
inst.sort_tiers()
except Exception as e:
# ENRICH_LOG.warn("Unknown Error occurred processing instance {}".format(inst.id))
ENRICH_LOG.debug(e)
# raise(e)
fail(F_UNKNOWN)
if not reasons:
success()
ENRICH_LOG.info(feedback_string.format(inst_status, ','.join(reasons)))
ENRICH_LOG.log(1000, 'Writing output file...')
if hasattr(kwargs.get(ARG_OUTFILE), 'write'):
xigtxml.dump(kwargs.get(ARG_OUTFILE), corp)
else:
xigtxml.dump(writefile(kwargs.get(ARG_OUTFILE)), corp)
ENRICH_LOG.log(1000, 'Done.')
ENRICH_LOG.log(1000, "{} instances written.".format(len(corp)))
def test_word_alignment(self):
inst = self.xc[1]
add_gloss_lang_alignments(inst)
rgp(inst)
