def test_almost_correct(self):
segment_a = self.tokenize(
"risk assessment has to be undertaken by those who are qualified and expert in that area - that is the scientists ."
)
segment_b = self.tokenize(
" risk assessment must be made of those who are qualified and expertise in the sector - these are the scientists ."
)
scorer = CharacterFScorer('n=6,beta=3')
scorer.set_reference(segment_a)
self.assertEqual('{0:.12f}'.format(scorer.score(segment_b)),
"0.652414427449")
def get(self, config_string):
"""
Returns a scorer matching the metric and parameters defined in @param
config string.
Example: ScorerProvider.get("BLEU n=4") returns a SmoothedBleuScorer
object that considers n-gram precision up to n=4.
If more than one metrics are provided (separated by `;`),
an interpolated scorer will be returned.
Example: ScorerProvider.get("INTERPOLATE w=0.5,0.5; SENTENCEBLEU n=4; METEOR meteor_language=fr, meteor_path=/foo/bar/meteor")
returns an InterpolatedScorer object that scores hypotheses
using 0.5 * bleu_score + 0.5 * meteor_score.
"""
# interpolation
if config_string.startswith("INTERPOLATE"):
return si.ScorerInterpolator(config_string)
try:
scorer, arguments = config_string.split(" ", 1)
except ValueError:
scorer = config_string
arguments = ''
if scorer == 'SENTENCEBLEU':
return SentenceBleuScorer(arguments)
elif scorer == 'METEOR':
return MeteorScorer(arguments)
elif scorer == 'BEER':
return BeerScorer(arguments)
elif scorer == 'CHRF':
return CharacterFScorer(arguments)
# add other scorers here
else:
raise NotImplementedError("No such scorer: %s" % scorer)
def test_one_character(self):
segment_a = self.tokenize("A")
segment_b = self.tokenize("A")
scorer = CharacterFScorer('n=6,beta=3')
scorer.set_reference(segment_a)
self.assertEqual(scorer.score(segment_b), 1.0)
def test_half_right(self):
segment_a = self.tokenize("AB")
segment_b = self.tokenize("AA")
scorer = CharacterFScorer('n=6,beta=3')
scorer.set_reference(segment_a)
self.assertEqual(scorer.score(segment_b), 0.25)
def test_completely_different_segments(self):
segment_a = self.tokenize("AAAAAA")
segment_b = self.tokenize("BBBB")
scorer = CharacterFScorer('n=3,beta=3')
scorer.set_reference(segment_a)
self.assertEqual(scorer.score(segment_b), 0.0)
def test_identical_segments(self):
segment = self.tokenize("Consistency is the last refuge of the unimaginative")
scorer = CharacterFScorer('n=6,beta=3')
scorer.set_reference(segment)
self.assertEqual(scorer.score(segment), 1.0)
def test_almost_correct(self):
segment_a = self.tokenize("risk assessment has to be undertaken by those who are qualified and expert in that area - that is the scientists .")
segment_b = self.tokenize(" risk assessment must be made of those who are qualified and expertise in the sector - these are the scientists .")
scorer = CharacterFScorer('n=6,beta=3')
scorer.set_reference(segment_a)
self.assertEqual('{0:.12f}'.format(scorer.score(segment_b)), "0.652414427449")
def test_one_character(self):
segment_a = self.tokenize("A")
segment_b = self.tokenize("A")
scorer = CharacterFScorer('n=6,beta=3')
scorer.set_reference(segment_a)
self.assertEqual(scorer.score(segment_b), 1.0)
def test_half_right(self):
segment_a = self.tokenize("AB")
segment_b = self.tokenize("AA")
scorer = CharacterFScorer('n=6,beta=3')
scorer.set_reference(segment_a)
self.assertEqual(scorer.score(segment_b), 0.25)
def test_completely_different_segments(self):
segment_a = self.tokenize("AAAAAA")
segment_b = self.tokenize("BBBB")
scorer = CharacterFScorer('n=3,beta=3')
scorer.set_reference(segment_a)
self.assertEqual(scorer.score(segment_b), 0.0)
def test_identical_segments(self):
segment = self.tokenize("Consistency is the last refuge of the unimaginative")
scorer = CharacterFScorer('n=6,beta=3')
scorer.set_reference(segment)
self.assertEqual(scorer.score(segment), 1.0)