def cross_evaluate(corpus_fns, annotators=None, pickle_fn=None,
words_only=False):
"""
Performs a cross evaluation of each annotator against each of the other
ones. Returns an AlignEval object. Optionally pickles to a file.
"""
if annotators:
assert len(annotators) == len(corpus_fns)
else:
annotators = ["A%d" % (i+1)
for i in range(len(corpus_fns))]
corpora = read_corpora(corpus_fns, words_only)
align_eval = AlignEval()
for true_corp, true_annot in zip(corpora, annotators):
for pred_corp, pred_annot in zip(corpora, annotators):
if true_annot != pred_annot:
name = (true_annot, pred_annot)
align_eval.add(true_corp, pred_corp, name)
align_eval.run_eval()
if pickle_fn:
pickle.dump(align_eval, open(pickle_fn, "wb"))
return align_eval
def test_word_baseline_3(self):
print "Baseline: greedy_align_phrases"
pred_pgc = copy.deepcopy(self.true_pgc)
greedy_align_phrases(pred_pgc)
align_eval = AlignEval()
align_eval.add(self.true_pgc, pred_pgc)
align_eval.run_eval()
align_eval.write_alignment_overall()
def test_word_baseline_2(self):
# evaluation is incorrect, because true corpus also contains phrase
# alignments
print "Baseline: greedy_align_words"
pred_pgc = copy.deepcopy(self.true_pgc)
greedy_align_words(pred_pgc)
align_eval = AlignEval()
align_eval.add(self.true_pgc, pred_pgc)
align_eval.run_eval()
align_eval.write_alignment_overall()
def test_word_baseline_3(self):
print "Baseline: greedy_align_phrases"
pred_pgc = copy.deepcopy(self.true_pgc)
greedy_align_phrases(pred_pgc)
align_eval = AlignEval()
align_eval.add(self.true_pgc, pred_pgc)
align_eval.run_eval()
align_eval.write_alignment_overall()
def test_word_baseline_2(self):
# evaluation is incorrect, because true corpus also contains phrase
# alignments
print "Baseline: greedy_align_words"
pred_pgc = copy.deepcopy(self.true_pgc)
greedy_align_words(pred_pgc)
align_eval = AlignEval()
align_eval.add(self.true_pgc, pred_pgc)
align_eval.run_eval()
align_eval.write_alignment_overall()
def cross_evaluate(corpus_fns,
annotators=None,
pickle_fn=None,
words_only=False):
"""
Performs a cross evaluation of each annotator against each of the other
ones. Returns an AlignEval object. Optionally pickles to a file.
"""
if annotators:
assert len(annotators) == len(corpus_fns)
else:
annotators = ["A%d" % (i + 1) for i in range(len(corpus_fns))]
corpora = read_corpora(corpus_fns, words_only)
align_eval = AlignEval()
for true_corp, true_annot in zip(corpora, annotators):
for pred_corp, pred_annot in zip(corpora, annotators):
if true_annot != pred_annot:
name = (true_annot, pred_annot)
align_eval.add(true_corp, pred_corp, name)
align_eval.run_eval()
if pickle_fn:
pickle.dump(align_eval, open(pickle_fn, "wb"))
return align_eval
def test_pickle(self):
true_corpus = pred_corpus = ParallelGraphCorpus(inf="data/corpus-1.pgc")
align_eval = AlignEval()
align_eval.add(true_corpus, pred_corpus, "corpus-1")
align_eval.run_eval()
pickle_file = tempfile.TemporaryFile()
pickle.dump(align_eval, pickle_file, 2)
pickle_file.seek(0)
align_eval_2 = pickle.load(pickle_file)
align_eval_2.write()
def eval_corpora(true_corpora, pred_corpora, names, eval_fname,
align_eval=None, n=None):
"""
Evaluate predicted against true parallel graph corpora.
@param true_fns: iterable of true corpora
@param pred_fns: iterable of predicted corpora
@param names: iterable of labels for true/predicted pairs
@param eval_fname: name of file to which evaluation output is written
@keyword align_eval: AlignEval instance
@keyword n: limit evaluation to the first n files
"""
if align_eval:
assert isinstance(align_eval, AlignEval)
# reset evaluator to prevent accidents
align_eval.__init__()
else:
align_eval = AlignEval()
count = 0
for true_corpus, pred_corpus, name in itertools.izip(true_corpora,
pred_corpora,
names):
align_eval.add(true_corpus, pred_corpus, name)
count += 1
if count == n:
break
align_eval.run_eval()
log.info("saving evaluation report {0}".format(eval_fname))
makedirs(os.path.dirname(eval_fname))
align_eval.write(eval_fname)
return align_eval
"""
calculate simple baselines based on greedy alignment of equal words/roots
run from the exp dir which contains a data subdir with the true pgc
files and an eval subdir for evaluation results
"""
import copy
import glob
import os
from daeso.pgc.corpus import ParallelGraphCorpus
from daeso.pgc.evaluate import AlignEval
from daeso_nl.ga.kb.baseline import greedy_align_equal_words, greedy_align_equal_words_roots
eval1 = AlignEval()
eval2 = AlignEval()
for pgc_fn in glob.glob("data/part*true.pgc"):
true_corpus = ParallelGraphCorpus(inf=pgc_fn)
pred_corpus = copy.deepcopy(true_corpus)
greedy_align_equal_words(pred_corpus)
eval1.add(true_corpus, pred_corpus, os.path.basename(pgc_fn))
greedy_align_equal_words_roots(pred_corpus)
eval2.add(true_corpus, pred_corpus, os.path.basename(pgc_fn))
eval1.run_eval()
eval1.write("eval/greedy_align_equals_words.txt")
"-r", "--relations",
metavar="REL",
nargs="*",
help="limit output to given relations")
args = parser.parse_args()
corpus1 = ParallelGraphCorpus(inf=args.corpus1)
corpus2 = ParallelGraphCorpus(inf=args.corpus2)
pgc_diff(corpus1, corpus2,
corpus_name1=args.corpus1,
corpus_name2=args.corpus2,
annot1=args.first_annotator,
annot2=args.second_annotator,
show_comments=args.with_comments,
show_ident=args.with_ident,
relations=args.relations)
if args.evaluate:
from daeso.pgc.evaluate import AlignEval
align_eval = AlignEval()
align_eval.add(corpus1, corpus2)
align_eval.run_eval()
align_eval.write()
calculate simple baselines based on greedy alignment of equal words/roots
run from the exp dir which contains a data subdir with the true pgc
files and an eval subdir for evaluation results
"""
import copy
import glob
import os
from daeso.pgc.corpus import ParallelGraphCorpus
from daeso.pgc.evaluate import AlignEval
from daeso_nl.ga.kb.baseline import greedy_align_equal_words, greedy_align_equal_words_roots
eval1 = AlignEval()
eval2 = AlignEval()
for pgc_fn in glob.glob("data/part*true.pgc"):
true_corpus = ParallelGraphCorpus(inf=pgc_fn)
pred_corpus = copy.deepcopy(true_corpus)
greedy_align_equal_words(pred_corpus)
eval1.add(true_corpus, pred_corpus, os.path.basename(pgc_fn))
greedy_align_equal_words_roots(pred_corpus)
eval2.add(true_corpus, pred_corpus, os.path.basename(pgc_fn))
eval1.run_eval()
eval1.write("eval/greedy_align_equals_words.txt")
def init_procs(self):
self.develop = True
self.validate = False
# parting
self.part = False
self.dev_parts = {}
self.val_parts = {}
self.part_max_size = None
# feature description
self.features = ()
# feature extraction
self.extract = True
self.graph_selector = select_aligned_graph_pair
self.node_selector = select_visible_node
self.binary = False
# sampling
self.sample = False
self.class_fracts = {}
#self.sampler = None
# classification
self.classify = True
# Normally self.classifier will be None, and a TimblFile instance with
# appropriate settings will be created on the fly. If you want to set
# a classifier explicitly, notice that it must at least set the
# verbosity options +vo, +vdb, +vdi, the -f option to specify the
# instances file, and the -m options to specify that the
# administrative features must be ignored.
self.classifier = None
self.timbl_opts = ""
self.timbl_log = True
self.feat_weight_graphs = False
self.train_sample = False
# weighting
self.weight = True
self.weight_func = entropy_weight
# matching
self.match = True
self.matcher = Matcher()
# merging
self.merge = True
self.merger = Merger()
# evaluation
self.evaluate = True
self.evaluator = AlignEval()
# AlignEval instances for develop and validation exps,
# which are saved in a pickled seting
self.dev_eval = None
self.val_eval = None
# iteraration limit during cross-validation
self.n = None
# store settings including evaluation as a pickle
self.pickle = False
