def test_changelist_view_returns_metrics_when_corpora_exist_and_query_is_on_the_added_at_date( self): news_item = NewsItem() news_item.title = 'foo' news_item.save() corpus = Corpus() corpus.news_item = news_item corpus.positive = True corpus.added_at = '2018-12-02 21:00:00+00:00' corpus.save() superuser = self.create_superuser('superuser') request = self.mocked_authenticated_request( '/admin/rss/corpusmetric/?added_at__month=12&added_at__year=2018', superuser) response = self.admin.changelist_view(request) expected_metrics = [{'positive': True, 'total': 1}] expected_metrics_total = {'total': 1} self.assertEquals(expected_metrics, response.context_data['corpus_metrics']) self.assertEquals(expected_metrics_total, response.context_data['corpus_metrics_total'])
def test_changelist_view_returns_metrics_when_accurate_and_inaccurate_newsitems_exist_and_finally_one_of_each_class(self): news_item = NewsItem() news_item.title = 'foo' news_item.score = 1 news_item.added_at = '2018-12-03 21:00:00+00:00' news_item.save() corpus = Corpus() corpus.positive = False corpus.news_item = news_item corpus.save() news_item = NewsItem() news_item.title = 'bar' news_item.score = 1 news_item.added_at = '2018-12-03 22:00:00+00:00' news_item.save() superuser = self.create_superuser('superuser') request = self.mocked_authenticated_request('/admin/rss/newsitemmetric/', superuser) response = self.admin.changelist_view(request) self.assertEquals(2, response.context_data['news_items_count']) self.assertEquals(0, response.context_data['news_items_unclassified']) self.assertEquals(2, response.context_data['classification_initial']['positive']) self.assertEquals(0, response.context_data['classification_initial']['negative']) self.assertEquals(1, response.context_data['classification_supervised']['positive']) self.assertEquals(1, response.context_data['classification_supervised']['negative']) self.assertEquals(0, response.context_data['corpus_count']['positive']) self.assertEquals(1, response.context_data['corpus_count']['negative']) self.assertEquals([{'accuracy': 50, 'added_at': '2018-12-03'}], response.context_data['accuracy'])
def test_to_dict_returns_one_dictionary_when_cursor_has_one_result(self): source = Source() source.id = 1 source.name = 'foo_source' source.save() news_item = NewsItem() news_item.id = 1 news_item.title = 'foo' news_item.description = 'bar' news_item.url = 'https://www.google.com' news_item.added_at = '2018-11-23 01:00:00+00:00' news_item.source = source news_item.published = False news_item.score = 1 news_item.save() with connection.cursor() as cursor: cursor.execute('SELECT * FROM news_item') result = self.newsitem_metric.to_dict(cursor) self.assertEquals(1, len(result)) self.assertEquals([{ 'id': 1, 'title': 'foo', 'description': 'bar', 'url': 'https://www.google.com', 'added_at': datetime(2018, 11, 23, 1, 0), 'source_id': 1, 'published': False, 'score': 1 }], result)
class RssFeedTestCase(TestCase): rss_feed = None news_item = None def setUp(self): self.news_item = NewsItem() self.rss_feed = RssFeed() def test_items_returns_empty_list_when_newsitem_does_not_exist(self): items = self.rss_feed.items() self.assertEquals([], list(items)) def test_items_returns_newsitems_when_positive_newsitem_exists(self): self.news_item.score = 1 self.news_item.published = True self.news_item.save() items = self.rss_feed.items() self.assertEquals(1, len(list(items))) def test_items_returns_empty_list_when_no_positive_newsitem_exists(self): self.news_item.score = 0 self.news_item.published = False self.news_item.save() items = self.rss_feed.items() self.assertEquals([], list(items)) def test_item_title_returns_newsitem_title_when_set(self): self.news_item.title = 'foo_title' self.assertEquals('foo_title', self.rss_feed.item_title(self.news_item)) def test_item_title_returns_empty_string_when_newsitem_title_not_set(self): self.assertEquals('', self.rss_feed.item_title(self.news_item)) def test_item_description_returns_newsitem_description_when_set(self): self.news_item.description = 'foo_description' self.assertEquals('foo_description', self.rss_feed.item_description(self.news_item)) def test_item_description_returns_empty_string_when_newsitem_description_not_set( self): self.assertEquals('', self.rss_feed.item_description(self.news_item)) def test_item_link_returns_newsitem_url_when_set(self): self.news_item.url = 'https://www.google.com' self.assertEquals('https://www.google.com', self.rss_feed.item_link(self.news_item)) def test_item_link_returns_empty_string_when_newsitem_url_not_set(self): self.assertEquals('', self.rss_feed.item_link(self.news_item)) def test_item_pubdate_returns_none_when_newsitem_added_at_not_set(self): self.assertNotEquals(None, self.rss_feed.item_pubdate(self.news_item)) def test_item_pubdate_returns_newsitem_added_at_datetime_when_set(self): self.news_item.url = timezone.now() self.assertNotEquals(None, self.rss_feed.item_pubdate(self.news_item))
def test_get_classifier_returns_naive_bayes_classifier_with_corpora_when_corpora_exist( self): # mock the shuffle list method classify.shuffle = mock.MagicMock() # regenerate the command class being tested as we mocked an extra module above self.command = classify.Command() # create a dummy news item and corpus news_item = NewsItem() news_item.title = 'foo' news_item.save() corpus = Corpus() corpus.news_item = news_item corpus.positive = True corpus.save() # call the method being tested classifier = self.command.get_classifier() # shuffled the corpora classify.shuffle.assert_called_once_with([('foo', 'pos')]) # a classifier with corpora was created classify.NaiveBayesClassifier.assert_called_once_with([('foo', 'pos')]) # the method returned a non-empty result self.assertTrue(classifier != None)
def test_get_classifier_uses_unique_corpora(self): # mock the shuffle list method classify.shuffle = mock.MagicMock() # regenerate the command class being tested as we mocked an extra module above self.command = classify.Command() # create dummy news items and corpora with same title and classification for index in range(1, 3): news_item = NewsItem() news_item.title = 'foo' news_item.save() corpus = Corpus() corpus.news_item = news_item corpus.positive = True corpus.save() # test that two corpora exist corpora = Corpus.objects.all() self.assertEquals(2, len(corpora)) # call the method being tested classifier = self.command.get_classifier() # shuffled the corpora with only one of the identical corpora classify.shuffle.assert_called_once_with([('foo', 'pos')])
def test_get_accuracy_total_returns_100_percent_when_no_corpora(self): news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-24 01:00:00+00:00' news_item.save() accuracy = self.newsitem_metric.get_accuracy_total(self.date_range) self.assertEquals(100, accuracy)
def test_get_accuracy_returns_100_percent_when_no_corpora(self): news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-24 01:00:00+00:00' news_item.save() metrics = self.newsitem_metric.get_accuracy(self.date_range) self.assertEquals(1, len(metrics)) self.assertEquals(100, metrics[0]['accuracy']) self.assertEquals('2018-11-24', metrics[0]['added_at'])
def test_get_accuracy_total_returns_0_percent_when_only_not_accurate_newsitems_exist( self): news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-24 01:00:00+00:00' news_item.save() corpus = Corpus() corpus.news_item = news_item corpus.positive = False corpus.save() accuracy = self.newsitem_metric.get_accuracy_total(self.date_range) self.assertEquals(0.0, accuracy)
def test_get_accuracy_total_returns_none_when_no_newsitems_between_range( self): news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-23 01:00:00+00:00' news_item.save() news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-26 01:00:00+00:00' news_item.save() accuracy = self.newsitem_metric.get_accuracy_total(self.date_range) self.assertEquals(None, accuracy)
def test_get_accuracy_returns_empty_list_when_no_newsitems_between_range( self): news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-23 01:00:00+00:00' news_item.save() news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-26 01:00:00+00:00' news_item.save() metrics = self.newsitem_metric.get_accuracy(self.date_range) self.assertEquals([], metrics)
def test_get_accuracy_does_not_include_unclassified_newsitems(self): news_item = NewsItem() news_item.score = None news_item.added_at = '2018-11-24 01:00:00+00:00' news_item.save() news_item = NewsItem() news_item.score = None news_item.published = True news_item.added_at = '2018-11-24 02:00:00+00:00' news_item.save() metrics = self.newsitem_metric.get_accuracy(self.date_range) self.assertEquals([], metrics)
def test_corpus_create_positive_creates_positive_corpora_and_enqueues_job_to_retrain_classifier_when_newsitems_in_query_set( self): news_item = NewsItem() news_item.title = 'foo' news_item.published = True news_item.save() query_set = [news_item] self.newsitem.corpus_create_positive(None, None, query_set) self.newsitem.enqueue_corpus_creation.assert_called_once() corpora = Corpus.objects.all() self.assertEquals(1, len(corpora)) self.assertEquals(True, corpora[0].positive)
def test_get_accuracy_returns_0_percent_when_only_not_accurate_newsitems_exist( self): news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-24 01:00:00+00:00' news_item.save() corpus = Corpus() corpus.news_item = news_item corpus.positive = False corpus.save() metrics = self.newsitem_metric.get_accuracy(self.date_range) self.assertEquals(1, len(metrics)) self.assertEquals(0, metrics[0]['accuracy']) self.assertEquals('2018-11-24', metrics[0]['added_at'])
def crawl(self, source, channel): source.crawling() self.logger.info('Crawling \'%s\'...', source.name) try: feedparser.USER_AGENT = settings.RSS_CRAWL_USER_AGENT feed = feedparser.parse(source.url) except RuntimeError: self.logger.error('Could not crawl \'%s\'.', source.name) return for entry in feed['entries']: if 'published' in entry: pass elif 'updated' in entry: entry['published'] = entry['updated'] else: entry['published'] = timezone.now().isoformat() if NewsItem.exists(entry['title'], parse(entry['published']), source): continue description = entry['summary'] if 'summary' in entry else entry[ 'title'] news_item = NewsItem() news_item.title = entry['title'] news_item.description = description news_item.url = entry['link'] news_item.source = source news_item.score = None news_item.added_at = parse(entry['published']) news_item.save() body = serializers.serialize('json', [news_item]) channel.basic_publish(exchange='', routing_key=settings.QUEUE_NAME_CLASSIFY, body=body, properties=pika.BasicProperties( delivery_mode=2, headers={'x-is-self-train': False})) source.crawled() self.logger.info('Successfully crawled \'%s\'!', source.name)
def test_news_item_publish_and_corpus_create_positive_publishes_newsitems_and_creates_positive_corpora_when_newsitems_in_query_set( self): news_item = NewsItem() news_item.title = 'foo' news_item.published = False news_item.save() query_set = [news_item] self.newsitem.news_item_publish_and_corpus_create_positive( None, None, query_set) self.newsitem.enqueue_corpus_creation.assert_called_once() corpora = Corpus.objects.all() self.assertEquals(1, len(corpora)) self.assertEquals(True, corpora[0].positive) news_items = NewsItem.objects.all() self.assertEquals(1, len(news_items)) self.assertEquals(True, news_items[0].published)
def test_get_accuracy_total_uses_two_day_statistics_when_exist(self): news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-24 01:00:00+00:00' news_item.save() corpus = Corpus() corpus.news_item = news_item corpus.positive = False corpus.save() news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-25 01:00:00+00:00' news_item.save() accuracy = self.newsitem_metric.get_accuracy_total(self.date_range) self.assertEquals(50.0, accuracy)
def test_to_dict_returns_dictionaries_when_cursor_more_than_one_results( self): news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-23 01:00:00+00:00' news_item.save() news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-23 02:00:00+00:00' news_item.save() with connection.cursor() as cursor: cursor.execute('SELECT * FROM news_item') results = self.newsitem_metric.to_dict(cursor) self.assertEquals(2, len(results)) self.assertEquals(dict, type(results[0])) self.assertEquals(dict, type(results[1]))
def test_news_item_publish_and_corpus_create_negative_publishes_newsitems_and_creates_negative_corpora_when_newsitems_in_query_set( self): news_item = NewsItem() news_item.title = 'foo' news_item.publshed = True news_item.score = 1.00 news_item.save() query_set = [news_item] self.newsitem.news_item_unpublish_and_corpus_create_negative( None, None, query_set) self.newsitem.enqueue_corpus_creation.assert_called_once() news_items = NewsItem.objects.all() self.assertEquals(1, len(news_items)) self.assertFalse(news_items[0].published) corpus = Corpus.objects.filter(news_item=news_items[0]) self.assertNotEquals(None, corpus) self.assertFalse(corpus[0].positive)
def test_handle_publishes_when_newsitems_exist(self): news_item = NewsItem() news_item.id = 1 news_item.title = 'foo' news_item.save() # the method being tested self.command.handle() classify_requeue.pika.BlockingConnection.assert_called_once() classify_requeue.pika.ConnectionParameters.assert_called_once() self.connection.channel.assert_called_once() self.channel.queue_declare.assert_called_once_with( queue=settings.QUEUE_NAME_CLASSIFY, durable=True) self.channel.basic_publish.assert_called_once() self.logger.info.assert_any_call( 'Found %s news items that need to be classified.', 1) self.logger.info.assert_any_call('Successfully re-queued #%s "%s"!', 1, 'foo')
def test_changelist_view_returns_metrics_when_newsitems_exist_but_no_corpora_and_date_query_includes_newsitem(self): news_item = NewsItem() news_item.title = 'foo' news_item.score = 1 news_item.added_at = '2018-12-03 21:00:00+00:00' news_item.save() superuser = self.create_superuser('superuser') request = self.mocked_authenticated_request('/admin/rss/newsitemmetric/?added_at__month=12&added_at__year=2018', superuser) response = self.admin.changelist_view(request) self.assertEquals(1, response.context_data['news_items_count']) self.assertEquals(0, response.context_data['news_items_unclassified']) self.assertEquals(1, response.context_data['classification_initial']['positive']) self.assertEquals(0, response.context_data['classification_initial']['negative']) self.assertEquals(1, response.context_data['classification_supervised']['positive']) self.assertEquals(0, response.context_data['classification_supervised']['negative']) self.assertEquals(0, response.context_data['corpus_count']['positive']) self.assertEquals(0, response.context_data['corpus_count']['negative']) self.assertEquals([{'accuracy': 100.0, 'added_at': '2018-12-03'}], response.context_data['accuracy'])
def test_changelist_view_returns_metrics_when_unclassified_newsitems_exist_and_no_corpora(self): news_item = NewsItem() news_item.title = 'foo' news_item.score = None news_item.added_at = '2018-12-03 21:00:00+00:00' news_item.save() superuser = self.create_superuser('superuser') request = self.mocked_authenticated_request('/admin/rss/newsitemmetric/', superuser) response = self.admin.changelist_view(request) self.assertEquals(1, response.context_data['news_items_count']) self.assertEquals(1, response.context_data['news_items_unclassified']) self.assertEquals(0, response.context_data['classification_initial']['positive']) self.assertEquals(0, response.context_data['classification_initial']['negative']) self.assertEquals(0, response.context_data['classification_supervised']['positive']) self.assertEquals(0, response.context_data['classification_supervised']['negative']) self.assertEquals(0, response.context_data['corpus_count']['positive']) self.assertEquals(0, response.context_data['corpus_count']['negative']) self.assertEquals([], response.context_data['accuracy'])
def test_news_returns_http_response_with_template_and_positive_newsitems_when_positive_newsitems_exist(self): # create a positive news item news_item = NewsItem() news_item.title = 'foo' news_item.score = 1 news_item.published = True news_item.save() # make request to news view response = self.get_news() # returns an instance of HttpResponse self.assertTrue(type(response) is HttpResponse) # request didn't fail self.assertEquals(200, response.status_code) # response contains news items content = response.getvalue() self.assertTrue('news-item' in str(content))
def test_get_classifier_uses_corpora_clean_of_stopwords(self): # mock the shuffle list method classify.shuffle = mock.MagicMock() # regenerate the command class being tested as we mocked an extra module above self.command = classify.Command() # create dummy news item (having title that contains stopwords) and corpora news_item = NewsItem() news_item.title = 'when foo then bar' news_item.save() corpus = Corpus() corpus.news_item = news_item corpus.positive = True corpus.save() # call the method being tested classifier = self.command.get_classifier() # shuffled the corpora with only one of the identical corpora classify.shuffle.assert_called_once_with([('foo bar', 'pos')])
def test_changelist_view_returns_positive_and_negative_metrics_when_positive_and_negative_corpora_exist( self): news_item = NewsItem() news_item.title = 'foo' news_item.save() corpus = Corpus() corpus.news_item = news_item corpus.positive = True corpus.save() news_item = NewsItem() news_item.title = 'bar' news_item.save() corpus = Corpus() corpus.news_item = news_item corpus.positive = False corpus.save() superuser = self.create_superuser('superuser') request = self.mocked_authenticated_request('/admin/rss/corpusmetric/', superuser) response = self.admin.changelist_view(request) expected_metrics = [{ 'positive': True, 'total': 1 }, { 'positive': False, 'total': 1 }] expected_metrics_total = {'total': 2} self.assertEquals(expected_metrics, response.context_data['corpus_metrics']) self.assertEquals(expected_metrics_total, response.context_data['corpus_metrics_total'])
def test_about_returns_http_response_with_stats(self): # create a news item news_item = NewsItem() news_item.title = 'foo' news_item.score = 1 news_item.published = False news_item.save() # create an active source source = Source() source.name = 'foo' source.url = 'http://www.foo.com' source.homepage = 'http://www.foo.com' source.save() # create an inactive source source = Source() source.name = 'bar' source.url = 'http://www.bar.com' source.homepage = 'http://www.bar.com' source.active = False source.save() # make request to news view response = self.get_about() # returns an instance of HttpResponse self.assertTrue(type(response) is HttpResponse) # request didn't fail self.assertEquals(200, response.status_code) # response does not contain any news items content = response.getvalue() self.assertTrue('Sources crawled</strong>: 1' in str(content)) self.assertTrue('News classified</strong>: 1' in str(content)) self.assertTrue('Corpora created</strong>: 0' in str(content)) self.assertTrue('Classification accuracy</strong>: 100%' in str(content))
def test_get_accuracy_returns_two_day_statistics_when_newsitems_for_two_days_exist( self): news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-24 01:00:00+00:00' news_item.save() corpus = Corpus() corpus.news_item = news_item corpus.positive = False corpus.save() news_item = NewsItem() news_item.score = 1 news_item.added_at = '2018-11-25 01:00:00+00:00' news_item.save() metrics = self.newsitem_metric.get_accuracy(self.date_range) self.assertEquals(2, len(metrics)) self.assertEquals('2018-11-24', metrics[0]['added_at']) self.assertEquals(0, metrics[0]['accuracy']) self.assertEquals('2018-11-25', metrics[1]['added_at']) self.assertEquals(100, metrics[1]['accuracy'])