def test_get_sentences(self):
     test_text = "First sentence. Second sentence. Sentence with 2.0 number. Sentence with H. pylori. Good bye."
     expected = [
         "First sentence.",
         "Second sentence.",
         "Sentence with 2.0 number.",
         "Sentence with H. pylori.",
         "Good bye.",
     ]
     actual = get_sentences(test_text)
     self.assertListEqual(expected, actual)
def main(article_data_sources, writers, sentence_finder, data_sources_to_skip=0, sentences_to_skip=0):
    data_source_names = list(map(lambda x: str(x), article_data_sources))
    constants.logger.info("data sources: %s" % data_source_names)
    total_sentence_number = 0
    for i in range(data_sources_to_skip, len(article_data_sources)):
        article_data_source = article_data_sources[i]

        articles = article_data_source.get_articles()
        # todo: sort to be able to continue
        sentences_articles_tuples = ((sentence, article) for article in articles
                                           for sentence in get_sentences(article.text))

        constants.logger.info("start looping sentences with data source №%i %s" % (i + 1, str(article_data_source)))
        sentence_number = sentences_to_skip
        for _ in range(sentences_to_skip):
            next(sentences_articles_tuples)
        sentences_to_skip = 0

        for sentence_text, article in sentences_articles_tuples:
            try:
                sentence = sentence_finder.get_sentence(sentence_text, article)
            except Exception:
                constants.logger.info(format_exc())
                constants.logger.info("sentence with error: %s" % sentence_text)
                constants.logger.info("got error in sentence loop; continue")
                continue
            if not sentence:
                continue

            for writer in writers:
                writer.write(sentence)

            sentence_number += 1
            constants.logger.info("memory usage: %f" % memory_usage_psutil())
            constants.logger.info("sentence № %i, data source № %i\n%s" % (sentence_number, i+1, sentence_text))
            constants.logger.info("=" * 80)

        total_sentence_number += sentence_number
        constants.logger.info("finish looping sentences with %s\n" % str(article_data_source))
    constants.pattern_logger.info('total number sentences: %i' % total_sentence_number)