def test_compute_future_costs(self): # arrange phrase_table = TestStackDecoder.create_fake_phrase_table() language_model = TestStackDecoder.create_fake_language_model() stack_decoder = StackDecoder(phrase_table, language_model) sentence = ('my', 'hovercraft', 'is', 'full', 'of', 'eels') # act future_scores = stack_decoder.compute_future_scores(sentence) # assert self.assertEqual( future_scores[1][2], ( phrase_table.translations_for(('hovercraft',))[0].log_prob + language_model.probability(('hovercraft',)) ), ) self.assertEqual( future_scores[0][2], ( phrase_table.translations_for(('my', 'hovercraft'))[0].log_prob + language_model.probability(('my', 'hovercraft')) ), )
def test_compute_future_costs(self): # arrange phrase_table = TestStackDecoder.create_fake_phrase_table() language_model = TestStackDecoder.create_fake_language_model() stack_decoder = StackDecoder(phrase_table, language_model) sentence = ("my", "hovercraft", "is", "full", "of", "eels") # act future_scores = stack_decoder.compute_future_scores(sentence) # assert self.assertEqual( future_scores[1][2], ( phrase_table.translations_for(("hovercraft",))[0].log_prob + language_model.probability(("hovercraft",)) ), ) self.assertEqual( future_scores[0][2], ( phrase_table.translations_for(("my", "hovercraft"))[0].log_prob + language_model.probability(("my", "hovercraft")) ), )
def test_distortion_score_of_first_expansion(self): # arrange stack_decoder = StackDecoder(None, None) stack_decoder.distortion_factor = 0.5 hypothesis = _Hypothesis() # act score = stack_decoder.distortion_score(hypothesis, (8, 10)) # assert # expansion from empty hypothesis always has zero distortion cost self.assertEqual(score, 0.0)
def test_distortion_score(self): # arrange stack_decoder = StackDecoder(None, None) stack_decoder.distortion_factor = 0.5 hypothesis = _Hypothesis() hypothesis.src_phrase_span = (3, 5) # act score = stack_decoder.distortion_score(hypothesis, (8, 10)) # assert expected_score = log(stack_decoder.distortion_factor) * (8 - 5) self.assertEqual(score, expected_score)
def test_compute_future_costs_for_phrases_not_in_phrase_table(self): # arrange phrase_table = TestStackDecoder.create_fake_phrase_table() language_model = TestStackDecoder.create_fake_language_model() stack_decoder = StackDecoder(phrase_table, language_model) sentence = ('my', 'hovercraft', 'is', 'full', 'of', 'eels') # act future_scores = stack_decoder.compute_future_scores(sentence) # assert self.assertEqual( future_scores[1][3], # 'hovercraft is' is not in phrase table future_scores[1][2] + future_scores[2][3]) # backoff
def test_compute_future_costs_for_phrases_not_in_phrase_table(self): # arrange phrase_table = TestStackDecoder.create_fake_phrase_table() language_model = TestStackDecoder.create_fake_language_model() stack_decoder = StackDecoder(phrase_table, language_model) sentence = ("my", "hovercraft", "is", "full", "of", "eels") # act future_scores = stack_decoder.compute_future_scores(sentence) # assert self.assertEqual( future_scores[1][3], future_scores[1][2] + future_scores[2][3] # 'hovercraft is' is not in phrase table ) # backoff
def test_future_score(self): # arrange: sentence with 8 words; words 2, 3, 4 already translated hypothesis = _Hypothesis() hypothesis.untranslated_spans = lambda _: [(0, 2), (5, 8)] # mock future_score_table = defaultdict(lambda: defaultdict(float)) future_score_table[0][2] = 0.4 future_score_table[5][8] = 0.5 stack_decoder = StackDecoder(None, None) # act future_score = stack_decoder.future_score(hypothesis, future_score_table, 8) # assert self.assertEqual(future_score, 0.4 + 0.5)
def test_find_all_src_phrases(self): # arrange phrase_table = TestStackDecoder.create_fake_phrase_table() stack_decoder = StackDecoder(phrase_table, None) sentence = ("my", "hovercraft", "is", "full", "of", "eels") # act src_phrase_spans = stack_decoder.find_all_src_phrases(sentence) # assert self.assertEqual(src_phrase_spans[0], [2]) # 'my hovercraft' self.assertEqual(src_phrase_spans[1], [2]) # 'hovercraft' self.assertEqual(src_phrase_spans[2], [3]) # 'is' self.assertEqual(src_phrase_spans[3], [5, 6]) # 'full of', 'full of eels' self.assertFalse(src_phrase_spans[4]) # no entry starting with 'of' self.assertEqual(src_phrase_spans[5], [6]) # 'eels'
def test_find_all_src_phrases(self): # arrange phrase_table = TestStackDecoder.create_fake_phrase_table() stack_decoder = StackDecoder(phrase_table, None) sentence = ('my', 'hovercraft', 'is', 'full', 'of', 'eels') # act src_phrase_spans = stack_decoder.find_all_src_phrases(sentence) # assert self.assertEqual(src_phrase_spans[0], [2]) # 'my hovercraft' self.assertEqual(src_phrase_spans[1], [2]) # 'hovercraft' self.assertEqual(src_phrase_spans[2], [3]) # 'is' self.assertEqual(src_phrase_spans[3], [5, 6]) # 'full of', 'full of eels' self.assertFalse(src_phrase_spans[4]) # no entry starting with 'of' self.assertEqual(src_phrase_spans[5], [6]) # 'eels'
def test_valid_phrases(self): # arrange hypothesis = _Hypothesis() # mock untranslated_spans method hypothesis.untranslated_spans = lambda _: [(0, 2), (3, 6)] all_phrases_from = [[1, 4], [2], [], [5], [5, 6, 7], [], [7]] # act phrase_spans = StackDecoder.valid_phrases(all_phrases_from, hypothesis) # assert self.assertEqual(phrase_spans, [(0, 1), (1, 2), (3, 5), (4, 5), (4, 6)])
def test_compute_future_costs(self): # arrange phrase_table = TestStackDecoder.create_fake_phrase_table() language_model = TestStackDecoder.create_fake_language_model() stack_decoder = StackDecoder(phrase_table, language_model) sentence = ("my", "hovercraft", "is", "full", "of", "eels") # act future_scores = stack_decoder.compute_future_scores(sentence) # assert self.assertEqual( future_scores[1][2], (phrase_table.translations_for(("hovercraft",))[0].log_prob + language_model.probability(("hovercraft",))), ) self.assertEqual( future_scores[0][2], ( phrase_table.translations_for(("my", "hovercraft"))[0].log_prob + language_model.probability(("my", "hovercraft")) ), )
parts = row[0].split('->') mk_phrase = parts[0] en_phrase = parts[1] prob = float(parts[2]) mk_phrase_parts = tuple(mk_phrase.lstrip().rstrip().split(' ')) en_phrase_parts = tuple(en_phrase.lstrip().rstrip().split(' ')) phrase_table.add(en_phrase_parts, mk_phrase_parts, prob) language_prob = defaultdict(lambda: -999.0) language_model = type( '', (object, ), { 'probability_change': lambda self, context, phrase: language_prob[phrase], 'probability': lambda self, phrase: language_prob[phrase] })() stack_decoder = StackDecoder(phrase_table, language_model) input_sentence = "he said that macedonian women were victims involved in prostitution during a recent sweep in bulgaria" with open("translation.txt", 'a', encoding='utf8') as file: print("Input sentence: " + input_sentence) file.write("Input sentence: " + input_sentence + "\n") input_words = input_sentence.split(' ') translated_sentence = [] temp_sentence = [] previous_temp = [] iteration_index = 0 for i in range(0, len(input_words)): temp_sentence.append(input_words[i]) translated_temp = stack_decoder.translate(temp_sentence) print(translated_temp) iteration_index += 1 file.write("Iteration " + str(iteration_index) + ": " +