def test_detect_language_calls_required_method(self, mock): letter_storage = LetterStorage() language_detector = LanguageDetector((3, ), 10) text_to_detect = (((1, 2, 3), ), ) file = open('lab_3/Frank_Baum.txt', 'r', encoding='utf-8') text = tokenize_by_sentence(file.read()) letter_storage.update(text) encoded_text = encode_corpus(letter_storage, text) file.close() language_detector.new_language(encoded_text, 'english') language_detector.detect_language(text_to_detect) self.assertTrue(mock.called)
def test_detect_language_uses_several_ngrams(self): letter_storage = LetterStorage() language_detector = LanguageDetector((2, 3), 100) file_first = open('lab_3/Frank_Baum.txt', 'r', encoding='utf-8') file_second = open('lab_3/Thomas_Mann.txt', 'r', encoding='utf-8') file_third = open('lab_3/unknown_Arthur_Conan_Doyle.txt', 'r', encoding='utf-8') text_english = tokenize_by_sentence(file_first.read()) text_german = tokenize_by_sentence(file_second.read()) text_unknown = tokenize_by_sentence(file_third.read()) letter_storage.update(text_english) letter_storage.update(text_german) letter_storage.update(text_unknown) encoded_english = encode_corpus(letter_storage, text_english) encoded_german = encode_corpus(letter_storage, text_german) encoded_unknown = encode_corpus(letter_storage, text_unknown) file_first.close() file_second.close() file_third.close() language_detector.new_language(encoded_english, 'english') language_detector.new_language(encoded_german, 'german') actual = language_detector.detect_language(encoded_unknown) self.assertTrue(actual['german'] > actual['english'])
def test_detect_language_empty_text_input(self): language_detector = LanguageDetector((3, ), 10) encoded_text = () expected = {} actual = language_detector.detect_language(encoded_text) self.assertEqual(expected, actual)
def test_detect_language_not_filled_ngram_storages(self): language_detector = LanguageDetector((2, 3), 100) patches_sentence = (((1, 2, 3), ), ) expected = {} actual = language_detector.detect_language(patches_sentence) self.assertEqual(expected, actual)
def test_detect_language_incorrect_text_input(self): language_detector = LanguageDetector((3, ), 10) expected = {} bad_inputs = [[], {}, set(), '', 123, -1, 0.7, None, True, (None, )] for bad_input in bad_inputs: actual = language_detector.detect_language(bad_input) self.assertEqual(expected, actual)
unknown_file.close() letter_storage = LetterStorage() letter_storage.update(text_eng) letter_storage.update(text_ger) letter_storage.update(text_unk) eng_encoded = encode_corpus(letter_storage, text_eng) unk_encoded = encode_corpus(letter_storage, text_unk) ger_encoded = encode_corpus(letter_storage, text_ger) language_detector = LanguageDetector((3, 4, 5), 1000) language_detector.new_language(eng_encoded, 'english') language_detector.new_language(ger_encoded, 'german') ngram_unknown = NGramTrie(4) ngram_unknown.fill_n_grams(unk_encoded) language_log_probability_dict = language_detector.detect_language(ngram_unknown.n_grams) if language_log_probability_dict['german'] >\ language_log_probability_dict['english']: RESULT = 'english' else: RESULT = 'german' print('this is a {} text.'.format(RESULT)) # DO NOT REMOVE NEXT LINE - KEEP IT INTENTIONALLY LAST assert RESULT == 'german', 'Not working'
# here goes your function calls letter_storage = LetterStorage() language_detector = LanguageDetector((3, ), 100) file_first = open('lab_3/Frank_Baum.txt', 'r', encoding='utf-8') file_second = open('lab_3/Thomas_Mann.txt', 'r', encoding='utf-8') file_third = open('lab_3/unknown_Arthur_Conan_Doyle.txt', 'r', encoding='utf-8') text_english = tokenize_by_sentence(file_first.read()) text_german = tokenize_by_sentence(file_second.read()) text_unknown = tokenize_by_sentence(file_third.read()) letter_storage.update(text_english) letter_storage.update(text_german) letter_storage.update(text_unknown) encoded_english = encode_corpus(letter_storage, text_english) encoded_german = encode_corpus(letter_storage, text_german) encoded_unknown = encode_corpus(letter_storage, text_unknown) file_first.close() file_second.close() file_third.close() language_detector.new_language(encoded_english, 'english') language_detector.new_language(encoded_german, 'german') actual = language_detector.detect_language(encoded_unknown) RESULT = actual['german'] > actual['english'] # DO NOT REMOVE NEXT LINE - KEEP IT INTENTIONALLY LAST assert RESULT == 1, ''