def update_feed(self): logger.debug(u"updating %s" % self.netloc) # Check freshness for value in [self.feed.last_checked_on, self.feed.last_updated_on]: if not value: continue # No datetime.timedelta since we need to # deal with large seconds values delta = datetime_as_epoch(self.instant) - datetime_as_epoch(value) if delta < config.fetcher.min_interval: logger.debug(u"%s is below minimun fetch interval, skipped" % self.netloc) return try: response = fetch_url(self.feed.self_link, timeout=config.fetcher.timeout, etag=self.feed.etag, modified_since=self.feed.last_updated_on) except RequestException: # Record any network error as 'Service Unavailable' self.feed.last_status = HTTPServiceUnavailable.code self.feed.error_count += 1 logger.warn(u"a network error occured while fetching %s, skipped" % self.netloc) return self.feed.last_checked_on = self.instant # Check if we got a redirect first if response.history: status = response.history[0].status_code else: status = response.status_code try: handler = getattr(self, 'handle_%d' % status, None) if handler: handler(response) else: self.feed.last_status = status logger.warn(u"%s replied with status %d, aborted" % (self.netloc, status)) return self._parse_feed(response.text) self._fetch_icon() except (HTTPError, HTTPNotModified, DuplicatedFeedError): return # Bail out finally: if self.feed.error_count > config.fetcher.max_errors: self._synthesize_entry( 'Feed has accomulated too many errors (last was %s).' % filters.status_title(status)) logger.warn(u"%s has accomulated too many errors, disabled" % self.netloc) self.feed.is_enabled = False self.feed.save()
def feed_add(self, request): message = '' groups = get_groups(self.user) if request.method == 'GET': return self.respond_with_template('_feed_add_wizard_1.html', locals()) # Handle postback self_link = request.POST['self_link'].strip() if not is_valid_url(self_link): message = u'ERROR Error, please specify a valid web address' return self.respond_with_template('_feed_add_wizard_1.html', locals()) status = fetcher.check_url(self_link) if status not in fetcher.POSITIVE_STATUS_CODES: message = u'ERROR Error, feed host returned: %s' % filters.status_title(status) return self.respond_with_template('_feed_add_wizard_1.html', locals()) group_id = int(request.POST.get('group', 0)) if group_id: group = Group.get(Group.id == group_id) else: group = Group.get(Group.title == Group.DEFAULT_GROUP) feed = Feed() feed.self_link = self_link feed = fetcher.add_feed(feed, fetch_icon=True, add_entries=True) subscription = fetcher.add_subscription(feed, self.user, group) if subscription: self.alert_message = u'SUCCESS Feed has been added to <i>%s</i> group' % group.title else: self.alert_message = u'INFO Feed is already in <i>%s</i> group' % group.title return self.respond_with_script('_modal_done.js', {'location': '%s/?feed=%d' % (request.application_url, feed.id)})
def check_feed_health(self): if config.fetcher.max_errors and self.feed.error_count > config.fetcher.max_errors: self._synthesize_entry( 'Feed has accumulated too many errors (last was %s).' % filters.status_title(self.feed.last_status)) logger.warn(u"%s has accomulated too many errors, disabled" % self.netloc) self.feed.is_enabled = False return
def post_fetch(status, error=False): if status: feed.last_status = status if error: feed.error_count = feed.error_count + 1 error_threshold = config.getint('fetcher', 'error_threshold') if error_threshold and (feed.error_count > error_threshold): feed.is_enabled = False feed.last_status = status # Save status code for posterity logger.warn("%s has too many errors, disabled" % netloc) synthesize_entry('Feed has accomulated too many errors (last was %s).' % status_title(status)) feed.save()
def post_fetch(status, error=False): if status: feed.last_status = status if error: feed.error_count = feed.error_count + 1 error_threshold = config.getint('fetcher', 'error_threshold') if error_threshold and (feed.error_count > error_threshold): feed.is_enabled = False feed.last_status = status # Save status code for posterity logger.warn(u"%s has too many errors, disabled" % netloc) synthesize_entry( 'Feed has accomulated too many errors (last was %s).' % status_title(status)) feed.save()
def feed_add(self, request): form_message = '' groups = get_groups(self.user) # URL could be passed via a GET (bookmarklet) or POST self_link = request.params.get('self_link', '').strip() if request.method == 'GET': return self.respond_with_template('_feed_add_wizard_1.html', locals()) if not is_valid_url(self_link): form_message = u'ERROR Error, specify a valid web address' return self.respond_with_template('_feed_add_wizard_1.html', locals()) response = fetcher.fetch_url(self_link) if response: if response.status_code not in fetcher.POSITIVE_STATUS_CODES: form_message = u'ERROR Error, feed host returned: %s' % filters.status_title( response.status_code) return self.respond_with_template('_feed_add_wizard_1.html', locals()) else: form_message = u'ERROR Error, a network error occured' return self.respond_with_template('_feed_add_wizard_1.html', locals()) group_id = int(request.POST.get('group', 0)) if group_id: group = Group.get(Group.id == group_id) else: group = Group.get(Group.title == Group.DEFAULT_GROUP) fetcher.load_plugins() trigger_event('fetch_started') feed = Feed() feed.self_link = self_link feed = fetcher.add_feed(feed, fetch_icon=True, add_entries=True) trigger_event('fetch_done', [feed]) subscription = fetcher.add_subscription(feed, self.user, group) if subscription: self.alert_message = u'SUCCESS Feed has been added to <i>%s</i> group' % group.title else: self.alert_message = u'INFO Feed is already in <i>%s</i> group' % group.title return self.respond_with_script('_modal_done.js', {'location': '%s/?feed=%d' % (request.application_url, feed.id)})
def update_feed(self): logger.debug(u"updating %s" % self.netloc) # Check freshness for value in [self.feed.last_checked_on, self.feed.last_updated_on]: if not value: continue # No datetime.timedelta since we need to # deal with large seconds values delta = datetime_as_epoch(self.instant) - datetime_as_epoch(value) if delta < config.fetcher.min_interval: logger.debug(u"%s is below minimun fetch interval, skipped" % self.netloc) return try: response = fetch_url(self.feed.self_link, timeout=config.fetcher.timeout, etag=self.feed.etag, modified_since=self.feed.last_updated_on) except RequestException: # Record any network error as 'Service Unavailable' self.feed.last_status = HTTPServiceUnavailable.code self.feed.error_count += 1 logger.warn(u"a network error occured while fetching %s, skipped" % self.netloc) return self.feed.last_checked_on = self.instant # Check if we got a redirect first if response.history: status = response.history[0].status_code else: status = response.status_code try: handler = getattr(self, 'handle_%d' % status, None) if handler: handler(response) else: self.feed.last_status = status logger.warn(u"%s replied with status %d, aborted" % (self.netloc, status)) return self._parse_feed(response.text) self._fetch_icon() except (HTTPError, HTTPNotModified, DuplicatedFeedError): return # Bail out finally: if self.feed.error_count > config.fetcher.max_errors: self._synthesize_entry('Feed has accomulated too many errors (last was %s).' % filters.status_title(status)) logger.warn(u"%s has accomulated too many errors, disabled" % self.netloc) self.feed.is_enabled = False self.feed.save()
def check_feed_health(self): if config.fetcher.max_errors and self.feed.error_count > config.fetcher.max_errors: self._synthesize_entry('Feed has accumulated too many errors (last was %s).' % filters.status_title(self.feed.last_status)) logger.warn(u"%s has accomulated too many errors, disabled" % self.netloc) self.feed.is_enabled = False return