def __init__(self, alpha=0.5):
self.simple_meteor = SimpleMeteor(alpha=alpha, beta=0.16)
self.tri_bleu = Bleu(3)
self.four_bleu = Bleu(4, beta=0.13)
self.p = Preprocessor()
def __init__(self, alpha=0.5): self.simple_meteor = SimpleMeteor(alpha=alpha, beta=0.16) self.tri_bleu = Bleu(3) self.four_bleu = Bleu(4, beta=0.13) self.p = Preprocessor()
class MeteorBleu:
""" Prints features for all versions of Meteor and BLEU for every
input sentence
"""
def __init__(self, alpha=0.5):
self.simple_meteor = SimpleMeteor(alpha=alpha, beta=0.16)
self.tri_bleu = Bleu(3)
self.four_bleu = Bleu(4, beta=0.13)
self.p = Preprocessor()
def features(self, tokline, posline):
""" The workhouse function
Takes lists of tokens and postags for [h1, h2, ref]
Returns feature values for h1, h2, h1-h2
"""
features = []
# Simple Meteor
h1p, h2p, refp = self.p.preprocess(tokline,
stem=False,
lowercase=False)
h1score = self.simple_meteor.score(h1p, refp)
h2score = self.simple_meteor.score(h2p, refp)
h1_h2 = h1score - h2score
features += [h1score, h2score, h1_h2]
# Simple Meteor lowercase
h1p, h2p, refp = self.p.preprocess(tokline, stem=False, lowercase=True)
h1score = self.simple_meteor.score(h1p, refp)
h2score = self.simple_meteor.score(h2p, refp)
h1_h2 = h1score - h2score
features += [h1score, h2score, h1_h2]
# Simple Meteor lowercase, stemmed
h1p, h2p, refp = self.p.preprocess(tokline, stem=True, lowercase=True)
h1score = self.simple_meteor.score(h1p, refp)
h2score = self.simple_meteor.score(h2p, refp)
h1_h2 = h1score - h2score
features += [h1score, h2score, h1_h2]
# Simple Meteor referencing sequence of postags, lowercase, stemmed
h1p, h2p, refp = self.p.preprocess(tokline, stem=True, lowercase=True)
h1pos, h2pos, refpos = self.p.preprocess(posline)
h1score = self.simple_meteor.score(h1p,
refp,
postags=True,
hpos=h1pos,
refpos=refpos)
h2score = self.simple_meteor.score(h2p,
refp,
postags=True,
hpos=h2pos,
refpos=refpos)
h1_h2 = h1score - h2score
features += [h1score, h2score, h1_h2]
# trigram BLEU, lowercased, stemmed
h1p, h2p, refp = self.p.preprocess(tokline, stem=True, lowercase=True)
h1score = self.tri_bleu.score(h1p, refp)
h2score = self.tri_bleu.score(h2p, refp)
h1_h2 = h1score - h2score
features += [h1score, h2score, h1_h2]
# postag-smoothed 4-gram BLEU
h1p, h2p, refp = self.p.preprocess(tokline,
stem=False,
lowercase=False)
h1pos, h2pos, refpos = self.p.preprocess(posline)
h1score = self.four_bleu.score(h1p,
refp,
postag=True,
hpos=h1pos,
refpos=refpos)
h2score = self.four_bleu.score(h2p,
refp,
postag=True,
hpos=h2pos,
refpos=refpos)
h1_h2 = h1score - h2score
features += [h1score, h2score, h1_h2]
# postag-smoothed 4-gram BLEU, lowercased
h1p, h2p, refp = self.p.preprocess(tokline, stem=False, lowercase=True)
h1pos, h2pos, refpos = self.p.preprocess(posline)
h1score = self.four_bleu.score(h1p,
refp,
postag=True,
hpos=h1pos,
refpos=refpos)
h2score = self.four_bleu.score(h2p,
refp,
postag=True,
hpos=h2pos,
refpos=refpos)
h1_h2 = h1score - h2score
features += [h1score, h2score, h1_h2]
# postag-smoothed 4-gram BLEU, lowercased, stemmed
h1p, h2p, refp = self.p.preprocess(tokline, stem=True, lowercase=True)
h1pos, h2pos, refpos = self.p.preprocess(posline)
h1score = self.four_bleu.score(h1p,
refp,
postag=True,
hpos=h1pos,
refpos=refpos)
h2score = self.four_bleu.score(h2p,
refp,
postag=True,
hpos=h2pos,
refpos=refpos)
h1_h2 = h1score - h2score
features += [h1score, h2score, h1_h2]
# postag-smoothed 4-gram BLEU, lowercased, stemmed, weighted
w = [10, 5, 2, 1]
h1p, h2p, refp = self.p.preprocess(tokline, stem=True, lowercase=True)
h1pos, h2pos, refpos = self.p.preprocess(posline)
h1score = self.four_bleu.score(h1p,
refp,
postag=True,
hpos=h1pos,
refpos=refpos,
wts=w)
h2score = self.four_bleu.score(h2p,
refp,
postag=True,
hpos=h2pos,
refpos=refpos,
wts=w)
h1_h2 = h1score - h2score
features += [h1score, h2score, h1_h2]
return features
def evaluate(self, h1score, h2score):
""" Scores hypothesis sentences based on scores
Prints output
"""
if h1score > h2score:
print -1
elif h1score == h2score:
print 0
else:
print 1
class MeteorBleu: """ Prints features for all versions of Meteor and BLEU for every input sentence """ def __init__(self, alpha=0.5): self.simple_meteor = SimpleMeteor(alpha=alpha, beta=0.16) self.tri_bleu = Bleu(3) self.four_bleu = Bleu(4, beta=0.13) self.p = Preprocessor() def features(self, tokline, posline): """ The workhouse function Takes lists of tokens and postags for [h1, h2, ref] Returns feature values for h1, h2, h1-h2 """ features = [] # Simple Meteor h1p, h2p, refp = self.p.preprocess(tokline, stem=False, lowercase=False) h1score = self.simple_meteor.score(h1p, refp) h2score = self.simple_meteor.score(h2p, refp) h1_h2 = h1score - h2score features += [h1score, h2score, h1_h2] # Simple Meteor lowercase h1p, h2p, refp = self.p.preprocess(tokline, stem=False, lowercase=True) h1score = self.simple_meteor.score(h1p, refp) h2score = self.simple_meteor.score(h2p, refp) h1_h2 = h1score - h2score features += [h1score, h2score, h1_h2] # Simple Meteor lowercase, stemmed h1p, h2p, refp = self.p.preprocess(tokline, stem=True, lowercase=True) h1score = self.simple_meteor.score(h1p, refp) h2score = self.simple_meteor.score(h2p, refp) h1_h2 = h1score - h2score features += [h1score, h2score, h1_h2] # Simple Meteor referencing sequence of postags, lowercase, stemmed h1p, h2p, refp = self.p.preprocess(tokline, stem=True, lowercase=True) h1pos, h2pos, refpos = self.p.preprocess(posline) h1score = self.simple_meteor.score( h1p, refp, postags=True, hpos=h1pos, refpos=refpos) h2score = self.simple_meteor.score( h2p, refp, postags=True, hpos=h2pos, refpos=refpos) h1_h2 = h1score - h2score features += [h1score, h2score, h1_h2] # trigram BLEU, lowercased, stemmed h1p, h2p, refp = self.p.preprocess(tokline, stem=True, lowercase=True) h1score = self.tri_bleu.score(h1p, refp) h2score = self.tri_bleu.score(h2p, refp) h1_h2 = h1score - h2score features += [h1score, h2score, h1_h2] # postag-smoothed 4-gram BLEU h1p, h2p, refp = self.p.preprocess(tokline, stem=False, lowercase=False) h1pos, h2pos, refpos = self.p.preprocess(posline) h1score = self.four_bleu.score( h1p, refp, postag=True, hpos=h1pos, refpos=refpos) h2score = self.four_bleu.score( h2p, refp, postag=True, hpos=h2pos, refpos=refpos) h1_h2 = h1score - h2score features += [h1score, h2score, h1_h2] # postag-smoothed 4-gram BLEU, lowercased h1p, h2p, refp = self.p.preprocess(tokline, stem=False, lowercase=True) h1pos, h2pos, refpos = self.p.preprocess(posline) h1score = self.four_bleu.score( h1p, refp, postag=True, hpos=h1pos, refpos=refpos) h2score = self.four_bleu.score( h2p, refp, postag=True, hpos=h2pos, refpos=refpos) h1_h2 = h1score - h2score features += [h1score, h2score, h1_h2] # postag-smoothed 4-gram BLEU, lowercased, stemmed h1p, h2p, refp = self.p.preprocess(tokline, stem=True, lowercase=True) h1pos, h2pos, refpos = self.p.preprocess(posline) h1score = self.four_bleu.score( h1p, refp, postag=True, hpos=h1pos, refpos=refpos) h2score = self.four_bleu.score( h2p, refp, postag=True, hpos=h2pos, refpos=refpos) h1_h2 = h1score - h2score features += [h1score, h2score, h1_h2] # postag-smoothed 4-gram BLEU, lowercased, stemmed, weighted w = [10,5,2,1] h1p, h2p, refp = self.p.preprocess(tokline, stem=True, lowercase=True) h1pos, h2pos, refpos = self.p.preprocess(posline) h1score = self.four_bleu.score( h1p, refp, postag=True, hpos=h1pos, refpos=refpos, wts=w) h2score = self.four_bleu.score( h2p, refp, postag=True, hpos=h2pos, refpos=refpos, wts=w) h1_h2 = h1score - h2score features += [h1score, h2score, h1_h2] return features def evaluate(self, h1score, h2score): """ Scores hypothesis sentences based on scores Prints output """ if h1score > h2score: print -1 elif h1score == h2score: print 0 else: print 1
