def test_probability_language_detector_calls_required_method(self, mock): unknown_file = open('lab_3/unknown_Arthur_Conan_Doyle.txt', encoding='utf-8') german_file = open('lab_3/Thomas_Mann.txt', encoding='utf-8') english_file = open('lab_3/Frank_Baum.txt', encoding='utf-8') text_unk = tokenize_by_sentence(unknown_file.read()) text_ger = tokenize_by_sentence(german_file.read()) text_eng = tokenize_by_sentence(english_file.read()) english_file.close() german_file.close() unknown_file.close() letter_storage = LetterStorage() letter_storage.update(text_eng) letter_storage.update(text_ger) letter_storage.update(text_unk) eng_encoded = encode_corpus(letter_storage, text_eng) unk_encoded = encode_corpus(letter_storage, text_unk) ger_encoded = encode_corpus(letter_storage, text_ger) language_detector = ProbabilityLanguageDetector((3, 4, 5), 1000) language_detector.new_language(eng_encoded, 'english') language_detector.new_language(ger_encoded, 'german') ngram_unknown = NGramTrie(4) ngram_unknown.fill_n_grams(unk_encoded) language_detector.detect_language(ngram_unknown.n_grams) self.assertTrue(mock.called)
def test_probability_language_detector_detect_language_ideal(self): unknown_file = open('lab_3/unknown_Arthur_Conan_Doyle.txt', encoding='utf-8') german_file = open('lab_3/Thomas_Mann.txt', encoding='utf-8') english_file = open('lab_3/Frank_Baum.txt', encoding='utf-8') text_unk = tokenize_by_sentence(unknown_file.read()) text_ger = tokenize_by_sentence(german_file.read()) text_eng = tokenize_by_sentence(english_file.read()) english_file.close() german_file.close() unknown_file.close() letter_storage = LetterStorage() letter_storage.update(text_eng) letter_storage.update(text_ger) letter_storage.update(text_unk) eng_encoded = encode_corpus(letter_storage, text_eng) unk_encoded = encode_corpus(letter_storage, text_unk) ger_encoded = encode_corpus(letter_storage, text_ger) language_detector = ProbabilityLanguageDetector((3, 4, 5), 1000) language_detector.new_language(eng_encoded, 'english') language_detector.new_language(ger_encoded, 'german') actual = language_detector.detect_language(unk_encoded) self.assertTrue(actual['german'] > actual['english'])
def test_probability_language_detector_detect_incorrect(self): language_detector = ProbabilityLanguageDetector((3, 5), 100) bad_inputs = [[], {}, '', 1, None, True, (None,)] expected = {} for bad_input in bad_inputs: actual = language_detector.detect_language(bad_input) self.assertEqual(expected, actual)
text_unk = tokenize_by_sentence(unknown_file.read()) text_ger = tokenize_by_sentence(german_file.read()) text_eng = tokenize_by_sentence(english_file.read()) english_file.close() german_file.close() unknown_file.close() letter_storage = LetterStorage() letter_storage.update(text_eng) letter_storage.update(text_ger) letter_storage.update(text_unk) eng_encoded = encode_corpus(letter_storage, text_eng) unk_encoded = encode_corpus(letter_storage, text_unk) ger_encoded = encode_corpus(letter_storage, text_ger) language_detector = ProbabilityLanguageDetector((3, 4, 5), 1000) language_detector.new_language(eng_encoded, 'english') language_detector.new_language(ger_encoded, 'german') ngram_unknown = NGramTrie(4) ngram_unknown.fill_n_grams(unk_encoded) actual = language_detector.detect_language(ngram_unknown.n_grams) print(actual) RESULT = actual['english'] < actual['german'] # DO NOT REMOVE NEXT LINE - KEEP IT INTENTIONALLY LAST assert RESULT == 1, ''