def get_or_fetch(cls, url, **kwargs):
"""Get a podcast by its feed url. If it's not in the db, fetch it."""
p = cls.get_by_url(url)
if not p:
logging.debug("Not in the database, need to fetch.")
code = error_store.get_error(url)
if code:
logging.debug("Refusing to fetch, because we've encountered an error recently: %s" % code)
return None
# Doing this import inside a function to avoid circular import.
from webapp.podcasts import crawler
results = list(crawler.fetch(url).results[0].collect())
logging.debug("Fetched, got %s." % results)
p = cls.get_by_url(url, **kwargs)
return p
def get_or_fetch(cls, url, **kwargs):
"""Get a podcast by its feed url. If it's not in the db, fetch it."""
p = cls.get_by_url(url)
if not p:
logging.debug("Not in the database, need to fetch.")
code = error_store.get_error(url)
if code:
logging.debug(
"Refusing to fetch, because we've encountered an error recently: %s"
% code)
return None
# Doing this import inside a function to avoid circular import.
from webapp.podcasts import crawler
results = list(crawler.fetch(url).results[0].collect())
logging.debug("Fetched, got %s." % results)
p = cls.get_by_url(url, **kwargs)
return p
def _fetch_podcast_data(url):
utils.validate_url(url, allow_hash=False)
try:
error_code = error_store.get_error(url)
if error_code:
logging.info("Won't fetch %s, since we got a %s response recently." % (url, error_code))
return
headers = {"User-Agent": current_app.config["USER_AGENT"]}
logging.info("fetching %s" % url)
resp, body = http.request(url, headers=headers)
if resp.status != 200:
logging.info("Received an error (%s) when trying to fetch %s" % (url, resp.status))
_handle_error_response(resp)
return
parsed = feedparser.parse(body)
except Exception as e:
logging.exception("Exception raised trying to fetch %s" % url)
return
return _handle_feed(parsed, resp)
def _fetch_podcast_data(url):
utils.validate_url(url, allow_hash=False)
try:
error_code = error_store.get_error(url)
if error_code:
logging.info(
"Won't fetch %s, since we got a %s response recently." %
(url, error_code))
return
headers = {"User-Agent": current_app.config["USER_AGENT"]}
logging.info("fetching %s" % url)
resp, body = http.request(url, headers=headers)
if resp.status != 200:
logging.info("Received an error (%s) when trying to fetch %s" %
(url, resp.status))
_handle_error_response(resp)
return
parsed = feedparser.parse(body)
except Exception as e:
logging.exception("Exception raised trying to fetch %s" % url)
return
return _handle_feed(parsed, resp)
