Esempio n. 1
0
    def update_feed(self):

        logger.debug(u"updating %s" % self.netloc)

        # Check freshness
        for value in [self.feed.last_checked_on, self.feed.last_updated_on]:
            if not value:
                continue

            # No datetime.timedelta since we need to
            #   deal with large seconds values
            delta = datetime_as_epoch(self.instant) - datetime_as_epoch(value)
            if delta < config.fetcher.min_interval:
                logger.debug(u"%s is below minimun fetch interval, skipped" %
                             self.netloc)
                return

        try:
            response = fetch_url(self.feed.self_link,
                                 timeout=config.fetcher.timeout,
                                 etag=self.feed.etag,
                                 modified_since=self.feed.last_updated_on)
        except RequestException:
            # Record any network error as 'Service Unavailable'
            self.feed.last_status = HTTPServiceUnavailable.code
            self.feed.error_count += 1
            logger.warn(u"a network error occured while fetching %s, skipped" %
                        self.netloc)
            return

        self.feed.last_checked_on = self.instant

        # Check if we got a redirect first
        if response.history:
            status = response.history[0].status_code
        else:
            status = response.status_code

        try:
            handler = getattr(self, 'handle_%d' % status, None)
            if handler:
                handler(response)
            else:
                self.feed.last_status = status
                logger.warn(u"%s replied with status %d, aborted" %
                            (self.netloc, status))
                return
            self._parse_feed(response.text)
            self._fetch_icon()
        except (HTTPError, HTTPNotModified, DuplicatedFeedError):
            return  # Bail out
        finally:
            if self.feed.error_count > config.fetcher.max_errors:
                self._synthesize_entry(
                    'Feed has accomulated too many errors (last was %s).' %
                    filters.status_title(status))
                logger.warn(u"%s has accomulated too many errors, disabled" %
                            self.netloc)
                self.feed.is_enabled = False
            self.feed.save()
Esempio n. 2
0
    def feed_add(self, request):        
        message = ''
        groups = get_groups(self.user)
        
        if request.method == 'GET':
            return self.respond_with_template('_feed_add_wizard_1.html', locals())

        # Handle postback
        self_link = request.POST['self_link'].strip()
        if not is_valid_url(self_link):
            message = u'ERROR Error, please specify a valid web address'
            return self.respond_with_template('_feed_add_wizard_1.html', locals())
        status = fetcher.check_url(self_link)
        if status not in fetcher.POSITIVE_STATUS_CODES:
            message = u'ERROR Error, feed host returned: %s' % filters.status_title(status)
            return self.respond_with_template('_feed_add_wizard_1.html', locals())

        group_id = int(request.POST.get('group', 0))
        if group_id:
            group = Group.get(Group.id == group_id) 
        else:
            group = Group.get(Group.title == Group.DEFAULT_GROUP)    

        feed = Feed()
        feed.self_link = self_link
        feed = fetcher.add_feed(feed, fetch_icon=True, add_entries=True)        
        subscription = fetcher.add_subscription(feed, self.user, group)
        if subscription:
            self.alert_message = u'SUCCESS Feed has been added to <i>%s</i> group' % group.title
        else:
            self.alert_message = u'INFO Feed is already in <i>%s</i> group' % group.title
        return self.respond_with_script('_modal_done.js', {'location': '%s/?feed=%d' % (request.application_url, feed.id)}) 
Esempio n. 3
0
 def check_feed_health(self):
     if config.fetcher.max_errors and self.feed.error_count > config.fetcher.max_errors:
         self._synthesize_entry(
             'Feed has accumulated too many errors (last was %s).' %
             filters.status_title(self.feed.last_status))
         logger.warn(u"%s has accomulated too many errors, disabled" %
                     self.netloc)
         self.feed.is_enabled = False
     return
Esempio n. 4
0
 def post_fetch(status, error=False):
     if status:
         feed.last_status = status
     if error:
         feed.error_count = feed.error_count + 1        
     error_threshold = config.getint('fetcher', 'error_threshold')
     if error_threshold and (feed.error_count > error_threshold):
         feed.is_enabled = False
         feed.last_status = status # Save status code for posterity           
         logger.warn("%s has too many errors, disabled" % netloc)        
         synthesize_entry('Feed has accomulated too many errors (last was %s).' % status_title(status))
     feed.save()
Esempio n. 5
0
 def post_fetch(status, error=False):
     if status:
         feed.last_status = status
     if error:
         feed.error_count = feed.error_count + 1
     error_threshold = config.getint('fetcher', 'error_threshold')
     if error_threshold and (feed.error_count > error_threshold):
         feed.is_enabled = False
         feed.last_status = status  # Save status code for posterity
         logger.warn(u"%s has too many errors, disabled" % netloc)
         synthesize_entry(
             'Feed has accomulated too many errors (last was %s).' %
             status_title(status))
     feed.save()
Esempio n. 6
0
    def feed_add(self, request):
        form_message = ''
        groups = get_groups(self.user)

        # URL could be passed via a GET (bookmarklet) or POST
        self_link = request.params.get('self_link', '').strip()

        if request.method == 'GET':
            return self.respond_with_template('_feed_add_wizard_1.html', locals())

        if not is_valid_url(self_link):
            form_message = u'ERROR Error, specify a valid web address'
            return self.respond_with_template('_feed_add_wizard_1.html', locals())
        response = fetcher.fetch_url(self_link)
        if response:
            if response.status_code not in fetcher.POSITIVE_STATUS_CODES:
                form_message = u'ERROR Error, feed host returned: %s' % filters.status_title(
                    response.status_code)
                return self.respond_with_template('_feed_add_wizard_1.html', locals())
        else:
            form_message = u'ERROR Error, a network error occured'
            return self.respond_with_template('_feed_add_wizard_1.html', locals())

        group_id = int(request.POST.get('group', 0))
        if group_id:
            group = Group.get(Group.id == group_id)
        else:
            group = Group.get(Group.title == Group.DEFAULT_GROUP)

        fetcher.load_plugins()
        trigger_event('fetch_started')
        feed = Feed()
        feed.self_link = self_link
        feed = fetcher.add_feed(feed, fetch_icon=True, add_entries=True)
        trigger_event('fetch_done', [feed])
        subscription = fetcher.add_subscription(feed, self.user, group)
        if subscription:
            self.alert_message = u'SUCCESS Feed has been added to <i>%s</i> group' % group.title
        else:
            self.alert_message = u'INFO Feed is already in <i>%s</i> group' % group.title
        return self.respond_with_script('_modal_done.js', {'location': '%s/?feed=%d' % (request.application_url, feed.id)})
Esempio n. 7
0
 def update_feed(self):
             
     logger.debug(u"updating %s" % self.netloc)
            
     # Check freshness
     for value in [self.feed.last_checked_on, self.feed.last_updated_on]:
         if not value:
             continue
         
         # No datetime.timedelta since we need to
         #   deal with large seconds values
         delta = datetime_as_epoch(self.instant) - datetime_as_epoch(value)    
         if delta < config.fetcher.min_interval:
             logger.debug(u"%s is below minimun fetch interval, skipped" % self.netloc)
             return                                      
     
     try:
         response = fetch_url(self.feed.self_link, 
             timeout=config.fetcher.timeout, 
             etag=self.feed.etag, 
             modified_since=self.feed.last_updated_on)
     except RequestException: 
         # Record any network error as 'Service Unavailable'
         self.feed.last_status   =  HTTPServiceUnavailable.code
         self.feed.error_count   += 1   
         logger.warn(u"a network error occured while fetching %s, skipped" % self.netloc)
         return
 
     self.feed.last_checked_on = self.instant
 
     # Check if we got a redirect first
     if response.history:
         status = response.history[0].status_code
     else:
         status = response.status_code
     
     try:
         handler = getattr(self, 'handle_%d' % status, None)
         if handler:
             handler(response)
         else: 
             self.feed.last_status = status
             logger.warn(u"%s replied with status %d, aborted" % (self.netloc, status))
             return
         self._parse_feed(response.text)
         self._fetch_icon()
     except (HTTPError, HTTPNotModified, DuplicatedFeedError):
         return # Bail out
     finally:
         if self.feed.error_count > config.fetcher.max_errors:
             self._synthesize_entry('Feed has accomulated too many errors (last was %s).' % filters.status_title(status))
             logger.warn(u"%s has accomulated too many errors, disabled" % self.netloc)
             self.feed.is_enabled = False
         self.feed.save()
Esempio n. 8
0
 def check_feed_health(self):
     if config.fetcher.max_errors and self.feed.error_count > config.fetcher.max_errors:
         self._synthesize_entry('Feed has accumulated too many errors (last was %s).' % filters.status_title(self.feed.last_status))
         logger.warn(u"%s has accomulated too many errors, disabled" % self.netloc)
         self.feed.is_enabled = False        
     return