def train(self, prefix, e, f):
"""
Train the giza word alignments on the provided text files.
:param prefix: Prefix for where the giza output files will be stored.
:type prefix: path+prefix
:param e: Path to the "e" file
:type e: path
:param f: Path to the "f"
:type f: path
"""
GIZA_LOG.info("Starting mgiza training from scratch...")
self.tf = GizaFiles(prefix, e, f)
GIZA_LOG.info("Converting txt files to SNTS and VCB files...")
self.tf.txt_to_snt(ev = Vocab(), fv = Vocab())
# Now, do the aligning...
exe = c.getpath('mgiza')
if exe is None:
raise GizaAlignmentException('Path to mgiza binary not defined.')
elif not os.path.exists(exe):
raise GizaAlignmentException('Path to mgiza binary "%s" invalid.')
elts = [exe,
'-o', os.path.join(self.tf.prefix, self.tf.name),
'-S', self.tf.e_vcb,
'-T', self.tf.f_vcb,
'-C', self.tf.ef_snt,
'-CoocurrenceFile', self.tf.ef_cooc,
'-hmmiterations', '5',
'-model4iterations', '0',
'-ncpus', '0']
cmd = ' '.join(elts)
GIZA_LOG.debug('Command: "{}"'.format(cmd))
p = ProcessCommunicator(elts)
status = p.wait()
GIZA_LOG.debug("Exit code: {}".format(str(status)))
if status != 0:
raise GizaAlignmentException("mgiza exited abnormally with a return code of {}".format(str(status)))
self.tf.merge_a3()
# self.tf.clean()
return self.tf.aligned_sents()
def enrich(**kwargs):
global classifier
if ARG_OUTFILE not in kwargs:
ENRICH_LOG.critical("No output file specified.")
sys.exit()
# =============================================================================
# Set up the alternate classifier path...
# =============================================================================
class_path = kwargs.get('class_path')
#===========================================================================
# Set up the different arguments...
#===========================================================================
inpath = kwargs.get(ARG_INFILE)
parse_args = kwargs.get(PARSE_VAR, [])
pos_args = kwargs.get(POS_VAR, [])
aln_args = kwargs.get(ALN_VAR, [])
max_parse_length = kwargs.get('max_parse_length', 10)
if not (parse_args or pos_args or aln_args):
ENRICH_LOG.warning("No enrichment specified. Basic processing only will be performed.")
#===========================================================================
# Sanity check the arguments.
#===========================================================================
# Check that alignment is asked for if projection is asked for.
if (ARG_POS_PROJ in pos_args or ARG_PARSE_PROJ in parse_args) and (not aln_args):
ENRICH_LOG.warn("You have asked for projection methods but have not requested " + \
"alignments to be generated. Projection may fail if alignment not already present in file.")
ENRICH_LOG.log(1000, 'Loading input file...')
with open(inpath, 'r', encoding='utf-8') as in_f:
corp = xigtxml.load(in_f, mode=INCREMENTAL)
# -------------------------------------------
# Initialize the English tagger if:
# A) "proj" option is selected for pos.
# B) "trans" option is given for pos.
# C) "heurpos" option is given for alignment.
# -------------------------------------------
s = None
if ARG_POS_PROJ in pos_args or ARG_POS_TRANS in pos_args or ARG_ALN_HEURPOS in aln_args:
ENRICH_LOG.log(1000, 'Initializing tagger...')
tagger = c.getpath('stanford_tagger_trans')
try:
s = StanfordPOSTagger(tagger)
except TaggerError as te:
ENRICH_LOG.critical(te)
sys.exit(2)
# -------------------------------------------
# Initialize the parser if:
# A) "trans" option is given for parse
# B) "proj" option is given for parse.
# -------------------------------------------
if ARG_PARSE_TRANS in parse_args or ARG_PARSE_PROJ in parse_args:
ENRICH_LOG.log(1000, "Intializing English parser...")
sp = stanford_parser.StanfordParser()
# -------------------------------------------
# Initialize the classifier if:
# A) "class" option is given for pos
# B) "heurpos" option is given for alignment.
# -------------------------------------------
m = None
if ARG_POS_CLASS in pos_args or ARG_ALN_HEURPOS in aln_args:
ENRICH_LOG.log(1000, "Initializing gloss-line classifier...")
p = load_posdict()
m = mallet_maxent.MalletMaxent(classifier)
# -- 1b) Giza Gloss to Translation alignment --------------------------------------
if ARG_ALN_GIZA in aln_args or ARG_ALN_GIZAHEUR in aln_args:
ENRICH_LOG.log(1000, 'Aligning gloss and translation lines using mgiza++...')
try:
if ARG_ALN_GIZAHEUR in aln_args:
giza_align_t_g(corp, resume=True, use_heur=True, symmetric=kwargs.get(ALN_SYM_VAR, SYMMETRIC_INTERSECT))
if ARG_ALN_GIZA in aln_args:
giza_align_t_g(corp, resume=True, use_heur=False, symmetric=kwargs.get(ALN_SYM_VAR, SYMMETRIC_INTERSECT))
except GizaAlignmentException as gae:
gl = logging.getLogger('giza')
gl.critical(str(gae))
raise gae
# -------------------------------------------
# Begin iterating through the corpus
# -------------------------------------------
for inst in corp:
feedback_string = 'Instance {:15s}: {{:20s}}{{}}'.format(inst.id)
reasons = []
inst_status = None
def fail(reason):
nonlocal inst_status, reasons
if reason not in reasons:
reasons.append(reason)
inst_status = 'WARN'
def success():
nonlocal inst_status
inst_status = 'OK'
# -------------------------------------------
# Define the reasons for failure
# -------------------------------------------
F_GLOSS_LINE = "NOGLOSS"
F_LANG_LINE = "NOLANG"
F_TRANS_LINE = "NOTRANS"
F_BAD_LINES = "BADLINES"
F_L_G_ALN = "L_G_ALIGN"
F_T_G_ALN = "G_T_ALIGN"
F_NO_TRANS_POS="NO_POS_TRANS"
F_PROJECTION = "PROJECTION"
F_UNKNOWN = "UNKNOWN"
F_PARSELEN = "OVER_MAX_LENGTH"
try:
# -------------------------------------------
# Get the different lines
# -------------------------------------------
def tryline(func):
nonlocal inst
try:
return func(inst)
except NoNormLineException as nnle:
return None
gl = tryline(gloss_line)
tls = tryline(trans_lines)
lls = tryline(lang_lines)
has_gl = gl is not None
has_tl = tls is not None
has_ll = lls is not None
has_all = lambda: (has_gl and has_tl and has_ll)
# -------------------------------------------
# Translation Line
# -------------------------------------------
if has_tl:
if ARG_POS_PROJ in pos_args or ARG_POS_TRANS in pos_args or ARG_ALN_HEURPOS in aln_args:
try:
tag_trans_pos(inst, s)
except CriticalTaggerError as cte:
ENRICH_LOG.critical(str(cte))
sys.exit(2)
if ARG_PARSE_PROJ in parse_args or ARG_PARSE_TRANS in parse_args:
if len(trans(inst)) <= max_parse_length:
parse_translation_line(inst, sp, pt=True, dt=True)
else:
fail(F_PARSELEN)
# 4) POS tag the gloss line --------------------------------------------
if has_gl:
if ARG_POS_CLASS in pos_args or ARG_ALN_HEURPOS in aln_args:
classify_gloss_pos(inst, m, posdict=p)
# -------------------------------------------
# Try getting alignments.
# -------------------------------------------
if has_gl and has_ll:
try:
add_gloss_lang_alignments(inst)
except GlossLangAlignException as glae:
fail(F_L_G_ALN)
if has_gl and has_tl:
if ARG_ALN_HEURPOS in aln_args:
heur_align_inst(inst, use_pos=True)
if ARG_ALN_HEUR in aln_args:
heur_align_inst(inst, use_pos=False)
# -------------------------------------------
# Now, do the necessary projection tasks.
# -------------------------------------------
# Project the classifier tags...
if has_ll and has_gl and ARG_POS_CLASS in pos_args:
try:
project_gloss_pos_to_lang(inst, tag_method=INTENT_POS_CLASS)
except GlossLangAlignException:
fail(F_L_G_ALN)
# -------------------------------------------
# Do the trans-to-lang projection...
# -------------------------------------------
if has_all():
proj_aln_method = ALN_ARG_MAP[kwargs.get('proj_aln', ARG_ALN_ANY)]
aln = get_trans_gloss_alignment(inst, aln_method=proj_aln_method)
if not aln or len(aln) == 0:
fail(F_T_G_ALN)
else:
# -------------------------------------------
# POS Projection
# -------------------------------------------
if ARG_POS_PROJ in pos_args:
trans_tags = trans_tag_tier(inst)
if not trans_tags:
fail(F_NO_TRANS_POS)
else:
project_trans_pos_to_gloss(inst)
try:
project_gloss_pos_to_lang(inst, tag_method=INTENT_POS_PROJ)
except GlossLangAlignException as glae:
fail(F_L_G_ALN)
# -------------------------------------------
# Parse projection
# -------------------------------------------
if ARG_PARSE_PROJ in parse_args:
try:
project_pt_tier(inst, proj_aln_method=proj_aln_method)
except PhraseStructureProjectionException as pspe:
fail(F_PROJECTION)
except NoAlignmentProvidedError as nape:
fail(F_T_G_ALN)
try:
project_ds_tier(inst, proj_aln_method=proj_aln_method)
except ProjectionException as pe:
fail(F_PROJECTION)
except NoAlignmentProvidedError as nape:
fail(F_T_G_ALN)
# Sort the tiers... ----------------------------------------------------
inst.sort_tiers()
except Exception as e:
# ENRICH_LOG.warn("Unknown Error occurred processing instance {}".format(inst.id))
ENRICH_LOG.debug(e)
# raise(e)
fail(F_UNKNOWN)
if not reasons:
success()
ENRICH_LOG.info(feedback_string.format(inst_status, ','.join(reasons)))
ENRICH_LOG.log(1000, 'Writing output file...')
if hasattr(kwargs.get(ARG_OUTFILE), 'write'):
xigtxml.dump(kwargs.get(ARG_OUTFILE), corp)
else:
xigtxml.dump(writefile(kwargs.get(ARG_OUTFILE)), corp)
ENRICH_LOG.log(1000, 'Done.')
ENRICH_LOG.log(1000, "{} instances written.".format(len(corp)))
def resume(self, prefix, new_e, new_f):
"""
"Force" align a new set of data using the old
model, per the instructions at:
http://www.kyloo.net/software/doku.php/mgiza:forcealignment
"""
# First, initialize a new GizaFile container for
# the files we are going to create
new_gf = GizaFiles(prefix, new_e, new_f)
# Now, we're going to extend the old vocabulary files
# with the new text to align.
old_ev = Vocab.load(self.tf.e_vcb)
old_fv = Vocab.load(self.tf.f_vcb)
old_ev.add_from_txt(new_gf.e)
old_fv.add_from_txt(new_gf.f)
# Now that we've extended the vocabs, let's dump the
# now-extended vocabs into the new filepaths.
old_ev.dump(new_gf.e_vcb)
old_fv.dump(new_gf.f_vcb)
# Write out
new_gf.txt_to_snt(ev = old_ev, fv = old_fv)
exe = c.getpath('mgiza')
if exe is None:
raise GizaAlignmentException('Path to mgiza binary not defined.')
elif not os.path.exists(exe):
raise GizaAlignmentException('Path to mgiza binary "%s" invalid.' % exe)
args = [exe, #self.tf.cfg,
'-restart', '2',
'-o', os.path.join(new_gf.prefix, new_gf.name),
'-m2', '5',
'-previoust', self.tf.t,
'-previousa', self.tf.a,
'-previousn', self.tf.n,
'-previousd', self.tf.d3,
'-c', new_gf.ef_snt,
'-s', new_gf.e_vcb,
'-t', new_gf.f_vcb,
'-Coocurrencefile', new_gf.ef_cooc]
cmd = ' '.join(args)
GIZA_LOG.debug('Command: "{}"'.format(cmd))
p = ProcessCommunicator(args)
status = p.wait()
GIZA_LOG.debug("Exit status {}".format(str(status)))
if status != 0:
raise GizaAlignmentException("mgiza exited abnormally with a return code of {}".format(str(status)))
new_gf.merge_a3()
# new_gf.clean()
return new_gf.aligned_sents()