if remember_time: if isinstance(remember_time, basestring): remember_time = parse_timedelta(remember_time) expires = datetime.now() + remember_time if not entry.get('title') or not entry.get('original_url'): log.debug('Can\'t remember rejection for entry without title or url.') return message = 'Remembering rejection of `%s`' % entry['title'] if remember_time: message += ' for %i minutes' % (remember_time.seconds / 60) log.info(message) (remember_feed_id,) = feed.session.query(RememberFeed.id).filter(RememberFeed.name == feed.name).first() feed.session.add(RememberEntry(title=entry['title'], url=entry['original_url'], feed_id=remember_feed_id, rejected_by=feed.current_plugin, reason=kwargs.get('reason'), expires=expires)) # The test stops passing when this is taken out for some reason... feed.session.flush() @event('manager.db_cleanup') def db_cleanup(session): # Remove entries older than 30 days result = session.query(RememberEntry).filter(RememberEntry.added < datetime.now() - timedelta(days=30)).delete() if result: log.verbose('Removed %d entries from remember rejected table.' % result) register_plugin(FilterRememberRejected, 'remember_rejected', builtin=True, api_ver=2) register_feed_phase('prefilter', after='input') register_parser_option('--forget-rejected', action='store_true', dest='forget_rejected', help='Forget all previous rejections so entries can be processed again.')
return False @priority(255) def url_rewrite(self, feed, entry): """Rewrites given entry url. Raises UrlRewritingError if failed.""" tries = 0 while self.url_rewritable(feed, entry): tries += 1 if tries > 300: raise UrlRewritingError('URL rewriting was left in infinite loop while rewriting url for %s, some rewriter is returning always True' % entry) for urlrewriter in get_plugins_by_group('urlrewriter'): name = urlrewriter.name try: if urlrewriter.instance.url_rewritable(feed, entry): log.debug('Url rewriting %s' % entry['url']) urlrewriter.instance.url_rewrite(feed, entry) log.info('Entry \'%s\' URL rewritten to %s (with %s)' % (entry['title'], entry['url'], name)) except UrlRewritingError, r: # increase failcount #count = self.shared_cache.storedefault(entry['url'], 1) #count += 1 raise UrlRewritingError('URL rewriting %s failed: %s' % (name, str(r.value))) except PluginError, e: raise UrlRewritingError('URL rewriting %s failed: %s' % (name, str(e.value))) except Exception, e: log.exception(e) raise UrlRewritingError('%s: Internal error with url %s' % (name, entry['url'])) register_plugin(PluginUrlRewriting, 'urlrewriting', builtin=True) register_feed_phase('urlrewrite', before='download')