def test_connection_error(mocker, caplog):
mock_requests = mocker.patch(
"podcasts.utils.session.get",
side_effect=requests.exceptions.ConnectionError)
caplog.set_level(logging.ERROR, logger="podcasts.utils")
with pytest.raises(requests.exceptions.ConnectionError):
utils.refresh_feed("https://any.feed/is/fine/here")
mock_requests.assert_called_once()
def update(
self,
feed_info=None,
create_episodes=True,
insert_cover=True,
update_all=False,
only_first_page=False,
):
if not feed_info:
feed_info = refresh_feed(self.feed_url)
defaults = feed_info.data
defaults["feed_url"] = feed_info.url
defaults["fetched"] = timezone.now()
logger.info("Fetched %(podcast)s at %(timestamp)s" % {
"podcast": defaults["title"],
"timestamp": defaults["fetched"]
})
episodes = defaults.pop("episodes", None)
image = defaults.pop("image", None)
for key, value in defaults.items():
setattr(self, key, value)
if image is not None and insert_cover:
logger.info("Inserting cover image")
self.insert_cover(image)
if create_episodes:
logger.info("Inserting episodes from first page")
all_created = self.create_episodes(info_or_episodes=episodes)
# Continue while
# * There is a next page
# * Not only the first page should be processed
# * All episodes are new OR existing episodes should be updated
while (feed_info.next_page and not only_first_page
and (False not in all_created or update_all)):
logger.info("Fetching next page %s ..." % feed_info.next_page)
feed_info = refresh_feed(feed_info.next_page)
episodes = feed_info.data.pop("episodes", None)
all_created = self.create_episodes(info_or_episodes=episodes)
if False in all_created:
logger.info("Found existing episodes")
if not feed_info.next_page:
logger.info("No next page found")
logger.info("All done")
self.save()
action.send(self, verb="was fetched", timestamp=defaults["fetched"])
if self.updated:
action.send(self, verb="was updated", timestamp=self.updated)
def update(self,
feed_info=None,
create_episodes=True,
insert_cover=True,
update_all=False,
only_first_page=False):
if not feed_info:
feed_info = refresh_feed(self.feed_url)
defaults = feed_info.data
defaults['feed_url'] = feed_info.url
defaults['fetched'] = timezone.now()
logger.info('Fetched %(podcast)s at %(timestamp)s' % {
'podcast': defaults['title'],
'timestamp': defaults['fetched']
})
episodes = defaults.pop('episodes', None)
image = defaults.pop('image', None)
for key, value in defaults.items():
setattr(self, key, value)
if image is not None and insert_cover:
logger.info('Inserting cover image')
self.insert_cover(image)
if create_episodes:
logger.info('Inserting episodes from first page')
all_created = self.create_episodes(info_or_episodes=episodes)
# Continue while
# * There is a next page
# * Not only the first page should be processed
# * All episodes are new OR existing episodes should be updated
while (feed_info.next_page and not only_first_page
and (False not in all_created or update_all)):
logger.info('Fetching next page %s ...' % feed_info.next_page)
feed_info = refresh_feed(feed_info.next_page)
episodes = feed_info.data.pop('episodes', None)
all_created = self.create_episodes(info_or_episodes=episodes)
if False in all_created:
logger.info('Found existing episodes')
if not feed_info.next_page:
logger.info('No next page found')
logger.info('All done')
self.save()
action.send(self, verb='was fetched', timestamp=defaults['fetched'])
if self.updated:
action.send(self, verb='was updated', timestamp=self.updated)
def get_or_create_from_feed_url(self,
feed_url,
feed_info=None,
subscriber=None,
**kwargs):
self._for_write = True
if not feed_info:
feed_info = refresh_feed(feed_url)
if not feed_info:
logger.error("Feed %(feed)s seems dead" % {"feed": feed_url})
return None, False
created = False
try:
podcast = self.get(feed_url=feed_info.url)
logger.info("Found existing %(podcast)s" %
{"podcast": podcast.title})
except self.model.DoesNotExist:
# Args now include feed_info to prevent a second refresh happening down the line
podcast = self.create_from_feed_url(feed_info.url,
feed_info=feed_info,
**kwargs)
created = True
if subscriber:
podcast.subscribers.add(subscriber)
return podcast, created
def test_long_subtitle_feed(caplog):
"""Test if an overly long subtitle is properly truncated"""
caplog.set_level(logging.WARNING, logger="podcasts.utils")
feed_info = utils.refresh_feed(TEST_FEED_SUBTITLE_TOO_LONG)
assert len(feed_info.data["subtitle"]) == 255
assert feed_info.data["subtitle"].endswith(" ...")
assert "Subtitle too long, will be truncated" in caplog.text
def test_paged_feed(caplog):
"""Test proper handling of a paged feed."""
caplog.set_level(logging.INFO, logger="podcasts.utils")
feed_info = utils.refresh_feed(TEST_FEED_NEXT_PAGE)
assert feed_info.next_page is not None
assert "Feed has next page" in caplog.text
def create_from_feed_url(self, feed_url, **kwargs):
feed_info = kwargs.pop('feed_info', None)
if not feed_info:
feed_info = refresh_feed(feed_url)
if feed_info is None:
return
podcast = Podcast(feed_url=feed_info.url)
return podcast.update(feed_info=feed_info, **kwargs)
def get_or_create_from_feed_url(self, feed_url, **kwargs):
self._for_write = True
feed_info = kwargs.pop('feed_info', None)
if not feed_info:
feed_info = refresh_feed(feed_url)
try:
return self.get(feed_url=feed_info.url), False
except self.model.DoesNotExist:
# Args now include feed_info to prevent a second refresh happening down the line
return self.create_from_feed_url(feed_info.url, feed_info=feed_info, **kwargs), True
def create_from_feed_url(self, feed_url, **kwargs):
feed_info = kwargs.pop('feed_info', None)
if not feed_info:
feed_info = refresh_feed(feed_url)
if feed_info is None:
logger.error('Feed %(feed)s seems dead' % {'feed': feed_url})
return
logger.info('Creating %(podcast)s' %
{'podcast': feed_info.data['title']})
podcast = Podcast(feed_url=feed_info.url)
podcast.update(feed_info=feed_info, **kwargs)
return podcast
def create_from_feed_url(self, feed_url, **kwargs):
feed_info = kwargs.pop("feed_info", None)
if not feed_info:
feed_info = refresh_feed(feed_url)
if feed_info is None:
logger.error("Feed %(feed)s seems dead" % {"feed": feed_url})
return
logger.info("Creating %(podcast)s" %
{"podcast": feed_info.data["title"]})
podcast = Podcast(feed_url=feed_info.url)
podcast.update(feed_info=feed_info, create_episodes=False, **kwargs)
podcast.queue_full_update()
return podcast
def create_episodes(self, info_or_episodes=None):
if info_or_episodes is None:
feed_info_obj = refresh_feed(self.feed_url)
episode_info = feed_info_obj.data['episodes']
elif isinstance(info_or_episodes, dict):
episode_info = info_or_episodes['episodes']
elif isinstance(info_or_episodes, feed_info):
episode_info = info_or_episodes.data['episodes']
else:
episode_info = info_or_episodes
all_created = []
for ep in episode_info:
ep['podcast'] = self
episode, created = Episode.objects.update_or_create(guid=ep['guid'], defaults=ep)
all_created.append(created)
return all_created
def get_or_create_from_feed_url(self, feed_url, **kwargs):
self._for_write = True
feed_info = kwargs.pop('feed_info', None)
if not feed_info:
feed_info = refresh_feed(feed_url)
if not feed_info:
logger.error('Feed %(feed)s seems dead' % {'feed': feed_url})
return None, False
try:
podcast = self.get(feed_url=feed_info.url)
logger.info('Found existing %(podcast)s' %
{'podcast': podcast.title})
return podcast, False
except self.model.DoesNotExist:
# Args now include feed_info to prevent a second refresh happening down the line
return self.create_from_feed_url(feed_info.url,
feed_info=feed_info,
**kwargs), True
def create_episodes(self, info_or_episodes=None):
if info_or_episodes is None:
feed_info_obj = refresh_feed(self.feed_url)
episode_info = feed_info_obj.data["episodes"]
elif isinstance(info_or_episodes, dict):
episode_info = info_or_episodes["episodes"]
elif isinstance(info_or_episodes, feed_info):
episode_info = info_or_episodes.data["episodes"]
else:
episode_info = info_or_episodes
all_created = []
for ep in episode_info:
ep["podcast"] = self
image = ep.pop("image", None)
episode, created = Episode.objects.update_or_create(
guid=ep["guid"], defaults=ep)
episode.insert_cover(image)
all_created.append(created)
return all_created
def queue_full_update(self):
from podcasts.tasks import refresh_feed
refresh_feed(self.id)
logger.info("Queued refresh task")
def test_invalid_feed(feed, expected, message, caplog):
"""Querying an invalid feed should always fail softly, returning None."""
caplog.set_level(logging.ERROR, logger="podcasts.utils")
with pytest.raises(Exception):
utils.refresh_feed(TEST_FEED_MALFORMED[feed])
def test_valid_feed():
feed_info = utils.refresh_feed(TEST_FEED)
assert feed_info is not None
assert feed_info.data["title"] == "Killing Time"