def test_get_topics(self):
"""Test getting topics from the database."""
self.assertEqual(0, database_reader.get_number_topics())
self.assertEqual(set(database_reader.get_topics()), set())
groups = classifier.group_articles(test_utils.SIMILAR_ARTICLES)
database_writer.write_groups(groups)
self.assertEqual(database_reader.get_topics()[0]["title"], groups[0].get_title())
def test_get_grouped_articles(self):
"""Test getting grouped articles from the database."""
self.assertEqual(database_reader.get_grouped_articles(), [])
groups = classifier.group_articles(test_utils.SIMILAR_ARTICLES)
database_writer.write_articles(test_utils.SIMILAR_ARTICLES)
database_writer.write_groups(groups)
self.assertEqual(database_reader.get_grouped_articles()[0], groups[0])
def test_get_urls(self):
"""Test getting urls from the database."""
self.assertEqual(set(database_reader.get_urls()), set())
groups = classifier.group_articles(test_utils.SIMILAR_ARTICLES)
database_writer.write_groups(groups)
self.assertEqual(
set(database_reader.get_urls()),
set(model.get_url() for model in test_utils.SIMILAR_ARTICLES))
def test_remove_grouping_from_database(self):
"""Test remove grouping from database."""
database_writer.write_groups([self.grouping])
self.assertTrue(self.grouping.in_database())
self.assertEqual(1, len(database_reader.get_urls()))
database_writer.remove_grouping_from_database(self.grouping)
self.assertFalse(self.grouping.in_database())
self.assertEqual(0, len(database_reader.get_urls()))
def test_write_topics_to_database_grouping_in_database(self):
"""Test remove grouping from database."""
self.grouping.set_in_database(True)
for article in self.grouping.get_articles():
article.set_in_database(True)
database_writer.write_groups([self.grouping])
self.assertTrue(self.grouping.in_database())
self.assertEqual(0, len(database_reader.get_urls()))
def test_write_read_similar(self):
"""Test writing then reading similar articles."""
self.assertEqual(0, database_reader.get_number_topics())
groups = classifier.group_articles(test_utils.SIMILAR_ARTICLES)
database_writer.write_groups(groups)
stories = database_reader.get_stories_for_topic(groups[0].get_uuid())
stories = set(a.get('link') for a in stories.get('articles'))
self.assertEqual(stories, set(model.get_url() for model in test_utils.SIMILAR_ARTICLES))
self.assertEqual(1, database_reader.get_number_topics())
def test_clean_database(self):
"""Test clean database."""
database_writer.write_groups([self.grouping])
self.assertEqual(1, len(database_reader.get_urls()))
database_writer.clean_database()
self.assertEqual(1, len(database_reader.get_urls()))
grouping = models.Grouping(
models.Article(url="google.com",
publishedAt="2016-10-11T23:41:34Z",
keywords=["a"]))
database_writer.write_groups([grouping])
self.assertEqual(2, len(database_reader.get_urls()))
database_writer.clean_database()
self.assertEqual(1, len(database_reader.get_urls()))
def update_database():
"""Update the database with all the headlines from get_top_headlines."""
database_writer.clean_database()
articles = get_top_headlines(add_category_information=True)
urls_in_database = database_reader.get_urls()
articles = [article for article in articles if article.get_url() not in urls_in_database]
database_writer.write_articles(articles, )
grouped = classifier.group_articles()
database_writer.write_groups(grouped)
database_writer.write_group_fits()
database_writer.write_overall_fits()
if database_reader.get_number_articles_without_overall_fit() > constants.ARTICLES_NEEDED_BEFORE_ALL_FIT_UPDATED:
print "Not enough new articles"
database_writer.update_topic_pictures()
def test_similar_urls_one_in_database(self):
"""Test that similar urls get grouped together when one is already in the database."""
groupings = classifier.group_articles([test_utils.SIMILAR_ARTICLES[0]])
database_writer.write_groups(groupings)
self.assertEqual(1, len(classifier.group_articles([test_utils.SIMILAR_ARTICLES[1]])))