def tag(self, tagger: StanfordPOSTagger = None):
"""
Tag the given sentence.
"""
tags = tagger.tag(self.raw())
for i, tag in enumerate(tags):
self[i].cpostag = tag.label
self[i].postag = tag.label
def evaluate_intent(filelist, classifier_path=None, eval_alignment=None, eval_ds=None, eval_posproj=None,
classifier_feats=CLASS_FEATS_DEFAULT,
eval_tagger=None,
gold_tagmap=None, trans_tagmap=None, outpath=None):
"""
Given a list of files that have manual POS tags and manual alignment,
evaluate the various INTENT methods on that file.
:param filelist: List of paths to evaluate against.
:type filelist: list[str]
:param classifier_path: Path to the classifier model
:type classifier_path: str
:param eval_alignment:
"""
tagger = StanfordPOSTagger(tagger_model)
outstream = sys.stdout
if outpath is not None:
outstream = open(outpath, mode='w', encoding='utf-8')
# =============================================================================
# Set up the objects to run as "servers"
# =============================================================================
classifier_obj = MalletMaxent(classifier)
if classifier_path is not None:
classifier_obj = MalletMaxent(classifier_path)
class_matches, class_compares = 0, 0
e_tagger = None
if eval_tagger is not None:
e_tagger = StanfordPOSTagger(eval_tagger)
mas = MultAlignScorer()
ds_plma = PerLangMethodAccuracies()
pos_plma= PerLangMethodAccuracies()
pos_pla = POSEvalDict()
pos_proj_matrix = POSMatrix()
pos_class_matrix = POSMatrix()
# -------------------------------------------
# If a tag map is specified, let's load it.
# -------------------------------------------
g_tm = TagMap(gold_tagmap) if gold_tagmap is not None else None
t_tm = TagMap(trans_tagmap) if trans_tagmap is not None else None
# Go through all the files in the list...
for f in filelist:
outstream.write('Evaluating on file: {}\n'.format(f))
xc = xc_load(f, mode=FULL)
lang = os.path.basename(f)
# -------------------------------------------
# Test the classifier if evaluation is requested.
# -------------------------------------------
if classifier_path is not None:
matches, compares, acc = evaluate_classifier_on_instances(xc, classifier_obj, classifier_feats,
pos_class_matrix, gold_tagmap=g_tm)
outstream.write('{},{},{},{:.2f}\n'.format(lang, matches, compares, acc))
class_matches += matches
class_compares += compares
# -------------------------------------------
# Test alignment if requested.
# -------------------------------------------
if eval_alignment:
mas.add_corpus('gold', INTENT_ALN_MANUAL, lang, xc)
EVAL_LOG.log(NORM_LEVEL, "Evaluating heuristic methods...")
evaluate_heuristic_methods_on_file(f, xc, mas, classifier_obj, tagger, lang)
EVAL_LOG.log(NORM_LEVEL, "Evaluating statistical methods...")
evaluate_statistic_methods_on_file(f, xc, mas, classifier_obj, tagger, lang)
# -------------------------------------------
# Test DS Projection if requested
# -------------------------------------------
if eval_ds:
evaluate_ds_projections_on_file(lang, xc, ds_plma, outstream=outstream)
outstream.write('{}\n'.format(ds_plma))
# -------------------------------------------
# Test POS Projection
# -------------------------------------------
if eval_posproj:
evaluate_pos_projections_on_file(lang, xc, pos_plma, pos_proj_matrix, tagger, gold_tagmap=g_tm, trans_tagmap=t_tm, outstream=outstream)
if e_tagger is not None:
evaluate_lang_pos(lang, xc, e_tagger, pos_pla, gold_tagmap=g_tm, outstream=outstream)
if eval_alignment:
mas.eval_all(outstream=outstream)
if eval_ds:
outstream.write('{}\n'.format(ds_plma))
if e_tagger is not None:
outstream.write('{},{},{},{:.2f}\n'.format(lang, pos_pla.all_matches(), pos_pla.fulltotal(), pos_pla.accuracy()))
e_tagger.close()
# Report the POS tagging accuracy...
if classifier_path is not None:
outstream.write("ALL...\n")
outstream.write('{},{},{:.2f}\n'.format(class_matches, class_compares, class_matches/class_compares*100))
outstream.write('{}\n'.format(pos_class_matrix))
if eval_posproj:
outstream.write('{}\n'.format(pos_proj_matrix))
outstream.close()
