def __next__(self) -> List[str]: """Return list of next sentence's words to be added to the word2vec vector.""" if self.__copy_to is None: raise StopIteration sentence = self.__copy_to.get_line() if sentence is None: self.__copy_to.end() self.__copy_to = None raise StopIteration sentence = sentence.strip() self.__sentence_counter += 1 if self.__sentence_counter % 1000 == 0: log.info("Feeding sentence %d..." % self.__sentence_counter) if not len(sentence): return [] language = None if identification_would_be_reliable(sentence): language_code = language_code_for_text(sentence) language = LanguageFactory.language_for_code(language_code) if language is None: language = LanguageFactory.default_language() words = language.split_sentence_to_words(sentence) if not len(words): return [] return words
def __next__(self) -> List[str]: """Return list of next sentence's words to be added to the word2vec vector.""" sentence = self.__next_sentence() if sentence is None: raise StopIteration sentence = sentence.strip() if not len(sentence): return [] language = None if identification_would_be_reliable(sentence): language_code = language_code_for_text(sentence) language = LanguageFactory.language_for_code(language_code) if language is None: language = LanguageFactory.default_language() words = language.split_sentence_to_words(sentence) if not len(words): return [] return words
def __next__(self) -> List[str]: """Return list of next sentence's words to be added to the word2vec vector.""" sentence = self.__next_sentence() if sentence is None: raise StopIteration sentence = sentence.strip() if not len(sentence): return [] language = None if identification_would_be_reliable(sentence): language_code = language_code_for_text(sentence) language = LanguageFactory.language_for_code(language_code) if language is None: language = LanguageFactory.default_language() words = language.split_sentence_to_words(sentence) if not len(words): return [] return words
def _get_sentences_from_story_text(story_text: str, story_lang: str) -> List[str]: """Split story text to individual sentences.""" story_text = decode_object_from_bytes_if_needed(story_text) story_lang = decode_object_from_bytes_if_needed(story_lang) # Tokenize into sentences lang = LanguageFactory.language_for_code(story_lang) if not lang: lang = LanguageFactory.default_language() sentences = lang.split_text_to_sentences(story_text) return sentences
def test_default_language(self): assert isinstance(LanguageFactory.default_language(), EnglishLanguage)
def test_default_language(self): assert isinstance(LanguageFactory.default_language(), EnglishLanguage)