def poll(self, timestamp, filters): logging.info('Polling feed "%s"' % self.url) result = [] self.last_poll = timestamp username = util.decode_password(self.username) password = util.decode_password(self.password) d = util.parse(self.url, username, password, self.etag, self.modified) self.etag = util.get(d, 'etag', None) self.modified = util.get(d, 'modified', None) feed = util.get(d, 'feed', None) if feed: self.title = self.title or util.get(feed, 'title', '') self.link = self.link or util.get(feed, 'link', self.url) entries = util.get(d, 'entries', []) for entry in reversed(entries): id = create_id(entry) if id in self.id_set: continue self.item_count += 1 self.id_list.append(id) self.id_set.add(id) item = Item(self, id) item.timestamp = calendar.timegm(util.get(entry, 'date_parsed', time.gmtime())) item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH) item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH) item.link = util.get(entry, 'link', '') item.author = util.format(util.get(entry, 'author', '')) # TODO: max length if all(filter.filter(item) for filter in filters): result.append(item) self.clean_cache(settings.FEED_CACHE_SIZE) return result
def poll(self, timestamp, filters): logging.info('Polling feed "%s"' % self.url) result = [] self.last_poll = timestamp username = util.decode_password(self.username) password = util.decode_password(self.password) d = util.parse(self.url, username, password, self.etag, self.modified) self.etag = util.get(d, 'etag', None) self.modified = util.get(d, 'modified', None) feed = util.get(d, 'feed', None) if feed: self.title = self.title or util.get(feed, 'title', '') self.link = self.link or util.get(feed, 'link', self.url) entries = util.get(d, 'entries', []) for entry in reversed(entries): id = create_id(entry) if id in self.id_set: continue self.item_count += 1 self.id_list.append(id) self.id_set.add(id) item = Item(self, id) item.timestamp = calendar.timegm( util.get(entry, 'date_parsed', time.gmtime())) item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH) item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH) item.link = util.get(entry, 'link', '') item.author = util.format(util.get(entry, 'author', '')) # TODO: max length if all(filter.filter(item) for filter in filters): result.append(item) self.clean_cache(settings.FEED_CACHE_SIZE) return result
def poll(self, timestamp, filters): try: logging.info('Polling feed "%s"' % self.url) result = [] self.last_poll = timestamp username = util.decode_password(self.username) password = util.decode_password(self.password) d = util.parse(self.url, username, password, self.etag, self.modified) self.etag = util.get(d, 'etag', None) self.modified = util.get(d, 'modified', None) feed = util.get(d, 'feed', None) if feed: self.title = self.title or util.get(feed, 'title', '') self.link = self.link or util.get(feed, 'link', self.url) entries = util.get(d, 'entries', []) for entry in reversed(entries): found = False id = create_id(entry) for i in self.id_set: # be cheap, compare the links - they should be unique. # Can't compare objects anymore because the time is present which will always be different if id[1] == i[1]: found = True break if found: continue # The entry is not new, move on to the next one. self.id_set.add(id) self.item_count += 1 item = Item(self, id) item.timestamp = calendar.timegm(util.get(entry, 'date_parsed', time.gmtime())) item.title = util.format(util.get(entry, 'title', ''), settings.POPUP_TITLE_LENGTH) item.description = util.format(util.get(entry, 'description', ''), settings.POPUP_BODY_LENGTH) item.link = util.get(entry, 'link', '') item.author = util.format(util.get(entry, 'author', '')) # TODO: max length if all(filter.filter(item) for filter in filters): result.append(item) # determine if there are any entries in self.id_set that are older than CACHE_AGE_LIMIT # if there are, remove them from the list (they aged out of the cache) idsToRemove = set() now = datetime.fromtimestamp(time.time()) for tempId in self.id_set: diff = now - datetime.fromtimestamp(tempId[3]) if abs(diff.days) > settings.CACHE_AGE_LIMIT: idsToRemove.add(tempId) # can't modify seld.id_set right here. for i in idsToRemove: logging.info('Removing %s because its too old' % i[1]) self.id_set.remove(i) return result except Exception, e: logging.info('Error durring poll: %s' % e) raise