def test_ds_project(self):
xc = xc_load(os.path.join(testfile_dir, 'xigt/index_error.xml'), do_basic_processing=True)
inst = xc[0]
heur_align_inst(inst)
parse_translation_line(inst, dt=True)
project_ds_tier(inst)
proj_t = get_lang_ds(inst)
tgt_t = DepTree.fromstring("""(ROOT[0] (salli-i[2] (Jumala[1]) (sata-a[4] ([[3])) (rake-i-ta[5]) (ja[6]) (tuhka-a[7] (].[8]))))""", stype=DEPSTR_PTB)
self.assertTrue(tgt_t.similar(proj_t))
inst2 = xc[1]
heur_align_inst(inst2)
parse_translation_line(inst2, dt=True)
project_ds_tier(inst2)
print(inst2)
tgt2_t = DepTree.fromstring("""(ROOT[0]
(unohta-a[2] (*Minua[1])
(unohda-n[4]
(/Minä[3])
(/laula-tta-a[6] (pelo-tta-a[5]))
)
))
""", stype=DEPSTR_PTB)
self.assertTrue(get_lang_ds(inst2), tgt2_t)
def test_ds_cycle(self):
"""
The tree in the ds_cycle file has "woman" depend both
on "arriving" and "browse."
"""
xc = xc_load(ds_cycle)
inst = xc[0]
# 1 2 4 5 7 8 9
# The woman, (after) arriving, began to browse.
# (The commas count as words, hence the skipping)
tgt_t = DepTree.fromstring("""
(ROOT[0]
(began[7]
(woman[2]
(The[1])
(\(after\)[4] (arriving[5])))
(browse[9]
(woman[2])
(to[8])
)
))
""", stype=DEPSTR_PTB)
ds = get_ds(inst, trans(inst))
self.assertTrue(tgt_t.structurally_eq(ds))
self.assertIsNone(project_ds_tier(inst))
def eval_method(aln_method):
# Set up the gold instances
gold_edges = set(gold_ds.to_indices())
# Add the number of compares, (the gold edges)
# and currently 0 for matches...
plma.add(lang, aln_method, 0, len(gold_edges))
# Try to do the projection
try:
project_ds_tier(inst, proj_aln_method=aln_method, ds_source=INTENT_DS_PARSER)
ds = get_lang_ds(inst, parse_method=INTENT_DS_PROJ)
tgt_edges = set(ds.to_indices())
# Add the number of matches, with 0 compares, since we added
# those previously.
plma.add(lang, aln_method, len(gold_edges & tgt_edges), 0)
except TreeProjectionError:
pass
def enrich(**kwargs):
global classifier
if ARG_OUTFILE not in kwargs:
ENRICH_LOG.critical("No output file specified.")
sys.exit()
# =============================================================================
# Set up the alternate classifier path...
# =============================================================================
class_path = kwargs.get('class_path')
#===========================================================================
# Set up the different arguments...
#===========================================================================
inpath = kwargs.get(ARG_INFILE)
parse_args = kwargs.get(PARSE_VAR, [])
pos_args = kwargs.get(POS_VAR, [])
aln_args = kwargs.get(ALN_VAR, [])
max_parse_length = kwargs.get('max_parse_length', 10)
if not (parse_args or pos_args or aln_args):
ENRICH_LOG.warning("No enrichment specified. Basic processing only will be performed.")
#===========================================================================
# Sanity check the arguments.
#===========================================================================
# Check that alignment is asked for if projection is asked for.
if (ARG_POS_PROJ in pos_args or ARG_PARSE_PROJ in parse_args) and (not aln_args):
ENRICH_LOG.warn("You have asked for projection methods but have not requested " + \
"alignments to be generated. Projection may fail if alignment not already present in file.")
ENRICH_LOG.log(1000, 'Loading input file...')
with open(inpath, 'r', encoding='utf-8') as in_f:
corp = xigtxml.load(in_f, mode=INCREMENTAL)
# -------------------------------------------
# Initialize the English tagger if:
# A) "proj" option is selected for pos.
# B) "trans" option is given for pos.
# C) "heurpos" option is given for alignment.
# -------------------------------------------
s = None
if ARG_POS_PROJ in pos_args or ARG_POS_TRANS in pos_args or ARG_ALN_HEURPOS in aln_args:
ENRICH_LOG.log(1000, 'Initializing tagger...')
tagger = c.getpath('stanford_tagger_trans')
try:
s = StanfordPOSTagger(tagger)
except TaggerError as te:
ENRICH_LOG.critical(te)
sys.exit(2)
# -------------------------------------------
# Initialize the parser if:
# A) "trans" option is given for parse
# B) "proj" option is given for parse.
# -------------------------------------------
if ARG_PARSE_TRANS in parse_args or ARG_PARSE_PROJ in parse_args:
ENRICH_LOG.log(1000, "Intializing English parser...")
sp = stanford_parser.StanfordParser()
# -------------------------------------------
# Initialize the classifier if:
# A) "class" option is given for pos
# B) "heurpos" option is given for alignment.
# -------------------------------------------
m = None
if ARG_POS_CLASS in pos_args or ARG_ALN_HEURPOS in aln_args:
ENRICH_LOG.log(1000, "Initializing gloss-line classifier...")
p = load_posdict()
m = mallet_maxent.MalletMaxent(classifier)
# -- 1b) Giza Gloss to Translation alignment --------------------------------------
if ARG_ALN_GIZA in aln_args or ARG_ALN_GIZAHEUR in aln_args:
ENRICH_LOG.log(1000, 'Aligning gloss and translation lines using mgiza++...')
try:
if ARG_ALN_GIZAHEUR in aln_args:
giza_align_t_g(corp, resume=True, use_heur=True, symmetric=kwargs.get(ALN_SYM_VAR, SYMMETRIC_INTERSECT))
if ARG_ALN_GIZA in aln_args:
giza_align_t_g(corp, resume=True, use_heur=False, symmetric=kwargs.get(ALN_SYM_VAR, SYMMETRIC_INTERSECT))
except GizaAlignmentException as gae:
gl = logging.getLogger('giza')
gl.critical(str(gae))
raise gae
# -------------------------------------------
# Begin iterating through the corpus
# -------------------------------------------
for inst in corp:
feedback_string = 'Instance {:15s}: {{:20s}}{{}}'.format(inst.id)
reasons = []
inst_status = None
def fail(reason):
nonlocal inst_status, reasons
if reason not in reasons:
reasons.append(reason)
inst_status = 'WARN'
def success():
nonlocal inst_status
inst_status = 'OK'
# -------------------------------------------
# Define the reasons for failure
# -------------------------------------------
F_GLOSS_LINE = "NOGLOSS"
F_LANG_LINE = "NOLANG"
F_TRANS_LINE = "NOTRANS"
F_BAD_LINES = "BADLINES"
F_L_G_ALN = "L_G_ALIGN"
F_T_G_ALN = "G_T_ALIGN"
F_NO_TRANS_POS="NO_POS_TRANS"
F_PROJECTION = "PROJECTION"
F_UNKNOWN = "UNKNOWN"
F_PARSELEN = "OVER_MAX_LENGTH"
try:
# -------------------------------------------
# Get the different lines
# -------------------------------------------
def tryline(func):
nonlocal inst
try:
return func(inst)
except NoNormLineException as nnle:
return None
gl = tryline(gloss_line)
tls = tryline(trans_lines)
lls = tryline(lang_lines)
has_gl = gl is not None
has_tl = tls is not None
has_ll = lls is not None
has_all = lambda: (has_gl and has_tl and has_ll)
# -------------------------------------------
# Translation Line
# -------------------------------------------
if has_tl:
if ARG_POS_PROJ in pos_args or ARG_POS_TRANS in pos_args or ARG_ALN_HEURPOS in aln_args:
try:
tag_trans_pos(inst, s)
except CriticalTaggerError as cte:
ENRICH_LOG.critical(str(cte))
sys.exit(2)
if ARG_PARSE_PROJ in parse_args or ARG_PARSE_TRANS in parse_args:
if len(trans(inst)) <= max_parse_length:
parse_translation_line(inst, sp, pt=True, dt=True)
else:
fail(F_PARSELEN)
# 4) POS tag the gloss line --------------------------------------------
if has_gl:
if ARG_POS_CLASS in pos_args or ARG_ALN_HEURPOS in aln_args:
classify_gloss_pos(inst, m, posdict=p)
# -------------------------------------------
# Try getting alignments.
# -------------------------------------------
if has_gl and has_ll:
try:
add_gloss_lang_alignments(inst)
except GlossLangAlignException as glae:
fail(F_L_G_ALN)
if has_gl and has_tl:
if ARG_ALN_HEURPOS in aln_args:
heur_align_inst(inst, use_pos=True)
if ARG_ALN_HEUR in aln_args:
heur_align_inst(inst, use_pos=False)
# -------------------------------------------
# Now, do the necessary projection tasks.
# -------------------------------------------
# Project the classifier tags...
if has_ll and has_gl and ARG_POS_CLASS in pos_args:
try:
project_gloss_pos_to_lang(inst, tag_method=INTENT_POS_CLASS)
except GlossLangAlignException:
fail(F_L_G_ALN)
# -------------------------------------------
# Do the trans-to-lang projection...
# -------------------------------------------
if has_all():
proj_aln_method = ALN_ARG_MAP[kwargs.get('proj_aln', ARG_ALN_ANY)]
aln = get_trans_gloss_alignment(inst, aln_method=proj_aln_method)
if not aln or len(aln) == 0:
fail(F_T_G_ALN)
else:
# -------------------------------------------
# POS Projection
# -------------------------------------------
if ARG_POS_PROJ in pos_args:
trans_tags = trans_tag_tier(inst)
if not trans_tags:
fail(F_NO_TRANS_POS)
else:
project_trans_pos_to_gloss(inst)
try:
project_gloss_pos_to_lang(inst, tag_method=INTENT_POS_PROJ)
except GlossLangAlignException as glae:
fail(F_L_G_ALN)
# -------------------------------------------
# Parse projection
# -------------------------------------------
if ARG_PARSE_PROJ in parse_args:
try:
project_pt_tier(inst, proj_aln_method=proj_aln_method)
except PhraseStructureProjectionException as pspe:
fail(F_PROJECTION)
except NoAlignmentProvidedError as nape:
fail(F_T_G_ALN)
try:
project_ds_tier(inst, proj_aln_method=proj_aln_method)
except ProjectionException as pe:
fail(F_PROJECTION)
except NoAlignmentProvidedError as nape:
fail(F_T_G_ALN)
# Sort the tiers... ----------------------------------------------------
inst.sort_tiers()
except Exception as e:
# ENRICH_LOG.warn("Unknown Error occurred processing instance {}".format(inst.id))
ENRICH_LOG.debug(e)
# raise(e)
fail(F_UNKNOWN)
if not reasons:
success()
ENRICH_LOG.info(feedback_string.format(inst_status, ','.join(reasons)))
ENRICH_LOG.log(1000, 'Writing output file...')
if hasattr(kwargs.get(ARG_OUTFILE), 'write'):
xigtxml.dump(kwargs.get(ARG_OUTFILE), corp)
else:
xigtxml.dump(writefile(kwargs.get(ARG_OUTFILE)), corp)
ENRICH_LOG.log(1000, 'Done.')
ENRICH_LOG.log(1000, "{} instances written.".format(len(corp)))
def do_projection(**kwargs):
"""
(Re)project the
:param aln_method: The alignment method
"""
kwargs = ArgPasser(kwargs)
aln_method = ALN_ARG_MAP[kwargs.get('aln_method', ARG_ALN_ANY)]
successes = 0
failures = 0
in_path = kwargs.get(ARG_INFILE)
with open(in_path, 'r', encoding='utf-8') as f:
PROJ_LOG.log(1000, 'Loading file "{}"...'.format(os.path.basename(in_path)))
xc = xigtxml.load(f, mode=INCREMENTAL)
for inst in xc:
success_fail_string = 'Instance {:20s} {{:10s}}{{}}'.format('"'+inst.id+'"...')
def fail(reason):
nonlocal failures, success_fail_string
success_fail_string = success_fail_string.format('FAIL', reason)
failures += 1
def success():
nonlocal successes, success_fail_string
success_fail_string = success_fail_string.format('SUCCESS', '')
successes += 1
# Query whether we want to require to use only trees
# where the alignment is 100%.
completeness_requirement = kwargs.get('completeness', default=0, t=float)
# TODO: Find better way to do this?
try:
if kwargs.get('pos', True):
project_trans_pos_to_gloss(inst, aln_method=aln_method, completeness_requirement=completeness_requirement)
project_gloss_pos_to_lang(inst, tag_method=INTENT_POS_PROJ)
if kwargs.get('ds', True):
project_pt_tier(inst, proj_aln_method=aln_method)
project_ds_tier(inst, proj_aln_method=aln_method, completeness_requirement=completeness_requirement)
except (NoNormLineException) as ntle:
fail("Bad Lines")
except (NoAlignmentProvidedError, ProjectionException) as nape:
fail("Alignment")
except (GlossLangAlignException) as glae:
fail("Gloss-Lang")
except (ProjectionIncompleteAlignment) as pia:
fail("Alignment Incomplete")
except PhraseStructureProjectionException as pspe:
fail("Projection Failed")
else:
success()
finally:
PROJ_LOG.info(success_fail_string)
inst.sort_tiers()
out_path = kwargs.get(ARG_OUTFILE)
# Try to make the folder if it doesn't already exist.
os.makedirs(os.path.dirname(out_path), exist_ok=True)
PROJ_LOG.log(1000, 'Writing new file "{}"...'.format(os.path.basename(out_path)))
with open(out_path, 'w', encoding='utf-8') as out_f:
xigtxml.dump(out_f, xc)
PROJ_LOG.log(1000, '{} instances processed, {} successful, {} failed.'.format(len(xc), successes, failures))
def all_tokens_present_test(self):
project_ds_tier(self.inst)
ds = get_lang_ds(self.inst)
print(ds.to_conll())
def test_project_ds_tree(self):
"""
Test that performing projection works correctly.
"""
project_ds_tier(self.inst1)
project_ds_tier(self.inst2)