def get_or_fetch(cls, url, **kwargs): """Get a podcast by its feed url. If it's not in the db, fetch it.""" p = cls.get_by_url(url) if not p: logging.debug("Not in the database, need to fetch.") code = error_store.get_error(url) if code: logging.debug("Refusing to fetch, because we've encountered an error recently: %s" % code) return None # Doing this import inside a function to avoid circular import. from webapp.podcasts import crawler results = list(crawler.fetch(url).results[0].collect()) logging.debug("Fetched, got %s." % results) p = cls.get_by_url(url, **kwargs) return p
def get_or_fetch(cls, url, **kwargs): """Get a podcast by its feed url. If it's not in the db, fetch it.""" p = cls.get_by_url(url) if not p: logging.debug("Not in the database, need to fetch.") code = error_store.get_error(url) if code: logging.debug( "Refusing to fetch, because we've encountered an error recently: %s" % code) return None # Doing this import inside a function to avoid circular import. from webapp.podcasts import crawler results = list(crawler.fetch(url).results[0].collect()) logging.debug("Fetched, got %s." % results) p = cls.get_by_url(url, **kwargs) return p
def _fetch_podcast_data(url): utils.validate_url(url, allow_hash=False) try: error_code = error_store.get_error(url) if error_code: logging.info("Won't fetch %s, since we got a %s response recently." % (url, error_code)) return headers = {"User-Agent": current_app.config["USER_AGENT"]} logging.info("fetching %s" % url) resp, body = http.request(url, headers=headers) if resp.status != 200: logging.info("Received an error (%s) when trying to fetch %s" % (url, resp.status)) _handle_error_response(resp) return parsed = feedparser.parse(body) except Exception as e: logging.exception("Exception raised trying to fetch %s" % url) return return _handle_feed(parsed, resp)
def _fetch_podcast_data(url): utils.validate_url(url, allow_hash=False) try: error_code = error_store.get_error(url) if error_code: logging.info( "Won't fetch %s, since we got a %s response recently." % (url, error_code)) return headers = {"User-Agent": current_app.config["USER_AGENT"]} logging.info("fetching %s" % url) resp, body = http.request(url, headers=headers) if resp.status != 200: logging.info("Received an error (%s) when trying to fetch %s" % (url, resp.status)) _handle_error_response(resp) return parsed = feedparser.parse(body) except Exception as e: logging.exception("Exception raised trying to fetch %s" % url) return return _handle_feed(parsed, resp)