feed_code = 550
feed = self.refresh_feed(feed.pk)
if (
(self.options["force"])
or (random.random() > 0.9)
or (
fetched_feed
and feed.feed_link
and feed.has_page
and (ret_feed == FEED_OK or (ret_feed == FEED_SAME and feed.stories_last_month > 10))
)
):
logging.debug(u" ---> [%-30s] ~FYFetching page: %s" % (feed.title[:30], feed.feed_link))
page_importer = PageImporter(feed)
try:
page_data = page_importer.fetch_page()
page_duration = time.time() - start_duration
except TimeoutError, e:
logging.debug(" ---> [%-30s] ~FRPage fetch timed out..." % (feed.title[:30]))
page_data = None
feed.save_page_history(555, "Timeout", "")
except Exception, e:
logging.debug("[%d] ! -------------------------" % (feed_id,))
tb = traceback.format_exc()
logging.error(tb)
logging.debug("[%d] ! -------------------------" % (feed_id,))
feed.save_page_history(550, "Page Error", tb)
fetched_feed = None
page_data = None
feed_code = 550
if not feed: continue
feed = self.refresh_feed(feed.pk)
if not feed: continue
if ((self.options['force']) or
(random.random() > .9) or
(fetched_feed and
feed.feed_link and
feed.has_page and
(ret_feed == FEED_OK or
(ret_feed == FEED_SAME and feed.stories_last_month > 10)))):
logging.debug(u' ---> [%-30s] ~FYFetching page: %s' % (feed.title[:30], feed.feed_link))
page_importer = PageImporter(feed)
try:
page_data = page_importer.fetch_page()
page_duration = time.time() - start_duration
except TimeoutError, e:
logging.debug(' ---> [%-30s] ~FRPage fetch timed out...' % (feed.title[:30]))
page_data = None
feed.save_page_history(555, 'Timeout', '')
except Exception, e:
logging.debug('[%d] ! -------------------------' % (feed_id,))
tb = traceback.format_exc()
logging.error(tb)
logging.debug('[%d] ! -------------------------' % (feed_id,))
feed.save_page_history(550, "Page Error", tb)
fetched_feed = None
page_data = None
tb = traceback.format_exc()
logging.error(tb)
logging.debug('[%d] ! -------------------------' % (feed_id, ))
ret_feed = FEED_ERREXC
feed.save_feed_history(500, "Error", tb)
fetched_feed = None
feed = self.refresh_feed(feed_id)
if ((self.options['force']) or
(fetched_feed and feed.feed_link and
(ret_feed == FEED_OK or
(ret_feed == FEED_SAME and feed.stories_last_month > 10)))):
logging.debug(u' ---> [%-30s] Fetching page: %s' %
(unicode(feed)[:30], feed.feed_link))
page_importer = PageImporter(feed.feed_link, feed)
try:
page_importer.fetch_page()
except TimeoutError, e:
logging.debug(' ---> [%-30s] Page fetch timed out...' %
(unicode(feed)[:30]))
feed.save_page_history(555, 'Timeout', '')
except Exception, e:
logging.debug('[%d] ! -------------------------' %
(feed_id, ))
tb = traceback.format_exc()
logging.error(tb)
logging.debug('[%d] ! -------------------------' %
(feed_id, ))
ret_feed = FEED_ERREXC
feed.save_page_history(550, "Page Error", tb)
logging.error(tb)
logging.debug('[%d] ! -------------------------' % (feed_id,))
ret_feed = FEED_ERREXC
feed.save_feed_history(500, "Error", tb)
fetched_feed = None
mail_feed_error_to_admin(feed, e)
feed = self.refresh_feed(feed_id)
if ((self.options['force']) or
(fetched_feed and
feed.feed_link and
(ret_feed == FEED_OK or
(ret_feed == FEED_SAME and feed.stories_last_month > 10)))):
logging.debug(u' ---> [%-30s] Fetching page: %s' % (unicode(feed)[:30], feed.feed_link))
page_importer = PageImporter(feed.feed_link, feed)
try:
page_importer.fetch_page()
except TimeoutError, e:
logging.debug(' ---> [%-30s] Page fetch timed out...' % (unicode(feed)[:30]))
feed.save_page_history(555, 'Timeout', '')
except Exception, e:
logging.debug('[%d] ! -------------------------' % (feed_id,))
tb = traceback.format_exc()
logging.error(tb)
logging.debug('[%d] ! -------------------------' % (feed_id,))
feed.save_page_history(550, "Page Error", tb)
fetched_feed = None
mail_feed_error_to_admin(feed, e)
logging.debug(u' ---> [%-30s] Fetching icon: %s' % (unicode(feed)[:30], feed.feed_link))
