def test_complete_articles_data(req_mock):
articles = [(
None,
{
"id": "12345",
"comment": "test comment",
"excerpt": "test excerpt",
"quote": "test quote",
"time_shared": "1531937658",
"image_src": "https://www.test.com/test.png",
},
)]
api.complete_articles_data(articles)
# article should have new key 'pocket_id' equal to value from 'id'
assert articles[0][1]["pocket_id"] == "12345"
# unnecessary keys should be removed
assert "id" not in articles[0][1]
assert "comment" not in articles[0][1]
assert "excerpt" not in articles[0][1]
assert "quote" not in articles[0][1]
# 'time_shared' should be converted to a datetime value
assert articles[0][1]["time_shared"] == make_aware(
datetime.datetime(2018, 7, 18, 11, 14, 18), utc)
# should have attempted to GET request against image_src
req_mock.get.assert_called_once_with("https://www.test.com/test.png")
def test_complete_articles_data(req_mock):
articles = [
(
None,
{
'id': '12345',
'comment': 'test comment',
'excerpt': 'test excerpt',
'quote': 'test quote',
'time_shared': '1531937658',
'image_src': 'https://www.test.com/test.png',
}
)
]
api.complete_articles_data(articles)
# article should have new key 'pocket_id' equal to value from 'id'
assert articles[0][1]['pocket_id'] == '12345'
# unnecessary keys should be removed
assert 'id' not in articles[0][1]
assert 'comment' not in articles[0][1]
assert 'excerpt' not in articles[0][1]
assert 'quote' not in articles[0][1]
# 'time_shared' should be converted to a datetime value
assert articles[0][1]['time_shared'] == make_aware(datetime.datetime(2018, 7, 18, 11, 14, 18), utc)
# should have attempted to GET request against image_src
req_mock.get.assert_called_once_with('https://www.test.com/test.png')
def test_complete_articles_data(req_mock):
articles = [(None, {
'id': '12345',
'comment': 'test comment',
'excerpt': 'test excerpt',
'quote': 'test quote',
'time_shared': '1531937658',
'image_src': 'https://www.test.com/test.png',
})]
api.complete_articles_data(articles)
# article should have new key 'pocket_id' equal to value from 'id'
assert articles[0][1]['pocket_id'] == '12345'
# unnecessary keys should be removed
assert 'id' not in articles[0][1]
assert 'comment' not in articles[0][1]
assert 'excerpt' not in articles[0][1]
assert 'quote' not in articles[0][1]
# 'time_shared' should be converted to a datetime value
assert articles[0][1]['time_shared'] == make_aware(
datetime.datetime(2018, 7, 18, 11, 14, 18), utc)
# should have attempted to GET request against image_src
req_mock.get.assert_called_once_with('https://www.test.com/test.png')
def update_articles(self, articles):
update_count = 0
del_count = 0
article_ids = []
articles_to_update = []
for article in articles:
# apparently it's possible for an article to be
# in the feed multiple times ¯\_(ツ)_/¯
if article["id"] in article_ids:
continue
article_ids.append(article["id"])
try:
# look for an object in the db with the article's id
obj = self.get(pocket_id=article["id"])
except PocketArticle.DoesNotExist:
# this article from Pocket will be added to the db
articles_to_update.append((None, article))
except PocketArticle.MultipleObjectsReturned:
# multiple articles with the same ID snuck in. Fix that here.
self.filter(pocket_id=article["id"]).delete()
articles_to_update.append((None, article))
else:
# this existing 'obj' will be updated in the db with
# (potentially) new info contained in 'article'
articles_to_update.append((obj, article))
# converts 'id' key to 'pocket_id' and converts 'time_shared' from
# unix timestamp to datetime
complete_articles_data(articles_to_update)
for obj, article in articles_to_update:
try:
if obj:
if obj.time_shared != article["time_shared"]:
for key, value in article.items():
setattr(obj, key, value)
obj.save()
update_count += 1
else:
self.create(**article)
update_count += 1
except DatabaseError:
capture_exception()
raise
# clean up after changes
if update_count:
expired_articles = self.exclude(pocket_id__in=article_ids)
del_count = expired_articles.count()
expired_articles.delete()
return update_count, del_count
def update_articles(self, articles):
update_count = 0
del_count = 0
article_ids = []
articles_to_update = []
for article in articles:
# apparently it's possible for an article to be
# in the feed multiple times ¯\_(ツ)_/¯
if article['id'] in article_ids:
continue
article_ids.append(article['id'])
try:
# look for an object in the db with the article's id
obj = self.get(pocket_id=article['id'])
except PocketArticle.DoesNotExist:
# this article from Pocket will be added to the db
articles_to_update.append((None, article))
except PocketArticle.MultipleObjectsReturned:
# multiple articles with the same ID snuck in. Fix that here.
self.filter(pocket_id=article['id']).delete()
articles_to_update.append((None, article))
else:
# this existing 'obj' will be updated in the db with
# (potentially) new info contained in 'article'
articles_to_update.append((obj, article))
# converts 'id' key to 'pocket_id' and converts 'time_shared' from
# unix timestamp to datetime
complete_articles_data(articles_to_update)
for obj, article in articles_to_update:
try:
if obj:
if obj.time_shared != article['time_shared']:
for key, value in article.iteritems():
setattr(obj, key, value)
obj.save()
update_count += 1
else:
self.create(**article)
update_count += 1
except DatabaseError:
sentry_client.captureException()
raise
# clean up after changes
if update_count:
expired_articles = self.exclude(pocket_id__in=article_ids)
del_count = expired_articles.count()
expired_articles.delete()
return update_count, del_count
def test_refresh_articles_with_dupes():
setup_responses()
updated, deleted = PocketArticle.objects.refresh()
assert updated == 3
assert deleted == 0
# add dupes
articles = api.get_articles_data()["recommendations"]
api.complete_articles_data((None, a) for a in articles)
for article in articles:
PocketArticle.objects.create(**article)
assert PocketArticle.objects.count() == 7
PocketArticle.objects.refresh()
assert PocketArticle.objects.count() == 3
def test_refresh_articles_with_dupes():
setup_responses()
updated, deleted = PocketArticle.objects.refresh()
assert updated == 3
assert deleted == 0
# add dupes
articles = api.get_articles_data()['recommendations']
api.complete_articles_data((None, a) for a in articles)
for article in articles:
PocketArticle.objects.create(**article)
assert PocketArticle.objects.count() == 7
PocketArticle.objects.refresh()
assert PocketArticle.objects.count() == 3
def update_articles(self, articles):
update_count = 0
del_count = 0
article_ids = []
articles_to_update = []
for article in articles:
article_ids.append(article['id'])
try:
# look for an object in the db with the article's id
obj = self.get(pocket_id=article['id'])
except PocketArticle.DoesNotExist:
# this article from Pocket will be added to the db
articles_to_update.append((None, article))
else:
# this existing 'obj' will be updated in the db with
# (potentially) new info contained in 'article'
articles_to_update.append((obj, article))
# converts 'id' key to 'pocket_id' and converts 'time_shared' from
# unix timestamp to datetime
complete_articles_data(articles_to_update)
for obj, article in articles_to_update:
try:
if obj:
if obj.time_shared != article['time_shared']:
for key, value in article.iteritems():
setattr(obj, key, value)
obj.save()
update_count += 1
else:
self.create(**article)
update_count += 1
except DatabaseError:
sentry_client.captureException()
raise
# clean up after changes
if update_count:
expired_articles = self.exclude(pocket_id__in=article_ids)
del_count = expired_articles.count()
expired_articles.delete()
return update_count, del_count
