def test_complete_articles_data(req_mock): articles = [( None, { "id": "12345", "comment": "test comment", "excerpt": "test excerpt", "quote": "test quote", "time_shared": "1531937658", "image_src": "https://www.test.com/test.png", }, )] api.complete_articles_data(articles) # article should have new key 'pocket_id' equal to value from 'id' assert articles[0][1]["pocket_id"] == "12345" # unnecessary keys should be removed assert "id" not in articles[0][1] assert "comment" not in articles[0][1] assert "excerpt" not in articles[0][1] assert "quote" not in articles[0][1] # 'time_shared' should be converted to a datetime value assert articles[0][1]["time_shared"] == make_aware( datetime.datetime(2018, 7, 18, 11, 14, 18), utc) # should have attempted to GET request against image_src req_mock.get.assert_called_once_with("https://www.test.com/test.png")
def test_complete_articles_data(req_mock): articles = [ ( None, { 'id': '12345', 'comment': 'test comment', 'excerpt': 'test excerpt', 'quote': 'test quote', 'time_shared': '1531937658', 'image_src': 'https://www.test.com/test.png', } ) ] api.complete_articles_data(articles) # article should have new key 'pocket_id' equal to value from 'id' assert articles[0][1]['pocket_id'] == '12345' # unnecessary keys should be removed assert 'id' not in articles[0][1] assert 'comment' not in articles[0][1] assert 'excerpt' not in articles[0][1] assert 'quote' not in articles[0][1] # 'time_shared' should be converted to a datetime value assert articles[0][1]['time_shared'] == make_aware(datetime.datetime(2018, 7, 18, 11, 14, 18), utc) # should have attempted to GET request against image_src req_mock.get.assert_called_once_with('https://www.test.com/test.png')
def test_complete_articles_data(req_mock): articles = [(None, { 'id': '12345', 'comment': 'test comment', 'excerpt': 'test excerpt', 'quote': 'test quote', 'time_shared': '1531937658', 'image_src': 'https://www.test.com/test.png', })] api.complete_articles_data(articles) # article should have new key 'pocket_id' equal to value from 'id' assert articles[0][1]['pocket_id'] == '12345' # unnecessary keys should be removed assert 'id' not in articles[0][1] assert 'comment' not in articles[0][1] assert 'excerpt' not in articles[0][1] assert 'quote' not in articles[0][1] # 'time_shared' should be converted to a datetime value assert articles[0][1]['time_shared'] == make_aware( datetime.datetime(2018, 7, 18, 11, 14, 18), utc) # should have attempted to GET request against image_src req_mock.get.assert_called_once_with('https://www.test.com/test.png')
def update_articles(self, articles): update_count = 0 del_count = 0 article_ids = [] articles_to_update = [] for article in articles: # apparently it's possible for an article to be # in the feed multiple times ¯\_(ツ)_/¯ if article["id"] in article_ids: continue article_ids.append(article["id"]) try: # look for an object in the db with the article's id obj = self.get(pocket_id=article["id"]) except PocketArticle.DoesNotExist: # this article from Pocket will be added to the db articles_to_update.append((None, article)) except PocketArticle.MultipleObjectsReturned: # multiple articles with the same ID snuck in. Fix that here. self.filter(pocket_id=article["id"]).delete() articles_to_update.append((None, article)) else: # this existing 'obj' will be updated in the db with # (potentially) new info contained in 'article' articles_to_update.append((obj, article)) # converts 'id' key to 'pocket_id' and converts 'time_shared' from # unix timestamp to datetime complete_articles_data(articles_to_update) for obj, article in articles_to_update: try: if obj: if obj.time_shared != article["time_shared"]: for key, value in article.items(): setattr(obj, key, value) obj.save() update_count += 1 else: self.create(**article) update_count += 1 except DatabaseError: capture_exception() raise # clean up after changes if update_count: expired_articles = self.exclude(pocket_id__in=article_ids) del_count = expired_articles.count() expired_articles.delete() return update_count, del_count
def update_articles(self, articles): update_count = 0 del_count = 0 article_ids = [] articles_to_update = [] for article in articles: # apparently it's possible for an article to be # in the feed multiple times ¯\_(ツ)_/¯ if article['id'] in article_ids: continue article_ids.append(article['id']) try: # look for an object in the db with the article's id obj = self.get(pocket_id=article['id']) except PocketArticle.DoesNotExist: # this article from Pocket will be added to the db articles_to_update.append((None, article)) except PocketArticle.MultipleObjectsReturned: # multiple articles with the same ID snuck in. Fix that here. self.filter(pocket_id=article['id']).delete() articles_to_update.append((None, article)) else: # this existing 'obj' will be updated in the db with # (potentially) new info contained in 'article' articles_to_update.append((obj, article)) # converts 'id' key to 'pocket_id' and converts 'time_shared' from # unix timestamp to datetime complete_articles_data(articles_to_update) for obj, article in articles_to_update: try: if obj: if obj.time_shared != article['time_shared']: for key, value in article.iteritems(): setattr(obj, key, value) obj.save() update_count += 1 else: self.create(**article) update_count += 1 except DatabaseError: sentry_client.captureException() raise # clean up after changes if update_count: expired_articles = self.exclude(pocket_id__in=article_ids) del_count = expired_articles.count() expired_articles.delete() return update_count, del_count
def test_refresh_articles_with_dupes(): setup_responses() updated, deleted = PocketArticle.objects.refresh() assert updated == 3 assert deleted == 0 # add dupes articles = api.get_articles_data()["recommendations"] api.complete_articles_data((None, a) for a in articles) for article in articles: PocketArticle.objects.create(**article) assert PocketArticle.objects.count() == 7 PocketArticle.objects.refresh() assert PocketArticle.objects.count() == 3
def test_refresh_articles_with_dupes(): setup_responses() updated, deleted = PocketArticle.objects.refresh() assert updated == 3 assert deleted == 0 # add dupes articles = api.get_articles_data()['recommendations'] api.complete_articles_data((None, a) for a in articles) for article in articles: PocketArticle.objects.create(**article) assert PocketArticle.objects.count() == 7 PocketArticle.objects.refresh() assert PocketArticle.objects.count() == 3
def update_articles(self, articles): update_count = 0 del_count = 0 article_ids = [] articles_to_update = [] for article in articles: article_ids.append(article['id']) try: # look for an object in the db with the article's id obj = self.get(pocket_id=article['id']) except PocketArticle.DoesNotExist: # this article from Pocket will be added to the db articles_to_update.append((None, article)) else: # this existing 'obj' will be updated in the db with # (potentially) new info contained in 'article' articles_to_update.append((obj, article)) # converts 'id' key to 'pocket_id' and converts 'time_shared' from # unix timestamp to datetime complete_articles_data(articles_to_update) for obj, article in articles_to_update: try: if obj: if obj.time_shared != article['time_shared']: for key, value in article.iteritems(): setattr(obj, key, value) obj.save() update_count += 1 else: self.create(**article) update_count += 1 except DatabaseError: sentry_client.captureException() raise # clean up after changes if update_count: expired_articles = self.exclude(pocket_id__in=article_ids) del_count = expired_articles.count() expired_articles.delete() return update_count, del_count