def register_plugin(): plugin.register(PluginUrlRewriting, 'urlrewriting', builtin=True, api_ver=2) plugin.register(DisableUrlRewriter, 'disable_urlrewriters', api_ver=2) plugin.register_task_phase('urlrewrite', before='download')
return False @priority(255) def url_rewrite(self, task, entry): """Rewrites given entry url. Raises UrlRewritingError if failed.""" tries = 0 while self.url_rewritable(task, entry): tries += 1 if tries > 300: raise UrlRewritingError('URL rewriting was left in infinite loop while rewriting url for %s, some rewriter is returning always True' % entry) for urlrewriter in get_plugins_by_group('urlrewriter'): name = urlrewriter.name try: if urlrewriter.instance.url_rewritable(task, entry): log.debug('Url rewriting %s' % entry['url']) urlrewriter.instance.url_rewrite(task, entry) log.info('Entry \'%s\' URL rewritten to %s (with %s)' % (entry['title'], entry['url'], name)) except UrlRewritingError as r: # increase failcount #count = self.shared_cache.storedefault(entry['url'], 1) #count += 1 raise UrlRewritingError('URL rewriting %s failed: %s' % (name, r.value)) except PluginError as e: raise UrlRewritingError('URL rewriting %s failed: %s' % (name, e.value)) except Exception as e: log.exception(e) raise UrlRewritingError('%s: Internal error with url %s' % (name, entry['url'])) register_plugin(PluginUrlRewriting, 'urlrewriting', builtin=True) register_task_phase('urlrewrite', before='download')
root.accept('text') return root def on_task_start(self, task, config): urlrewrite = get_plugin_by_name('urlrewriting')['instance'] for disable in config: try: get_plugin_by_name(disable) except DependencyError: log.critical('Unknown url-rewriter %s' % disable) continue log.debug('Disabling url rewriter %s' % disable) urlrewrite.disabled_rewriters.append(disable) def on_task_exit(self, task, config): urlrewrite = get_plugin_by_name('urlrewriting')['instance'] for disable in config: log.debug('Enabling url rewriter %s' % disable) try: urlrewrite.disabled_rewriters.remove(disable) except ValueError: log.debug('%s does not exists' % disable) on_task_abort = on_task_exit register_plugin(PluginUrlRewriting, 'urlrewriting', builtin=True, api_ver=2) register_plugin(DisableUrlRewriter, 'disable_urlrewriters', api_ver=2) register_task_phase('urlrewrite', before='download')
"""Rewrites given entry url. Raises UrlRewritingError if failed.""" tries = 0 while self.url_rewritable(task, entry): tries += 1 if tries > 300: raise UrlRewritingError( "URL rewriting was left in infinite loop while rewriting url for %s, some rewriter is returning always True" % entry ) for urlrewriter in get_plugins_by_group("urlrewriter"): name = urlrewriter.name try: if urlrewriter.instance.url_rewritable(task, entry): log.debug("Url rewriting %s" % entry["url"]) urlrewriter.instance.url_rewrite(task, entry) log.info("Entry '%s' URL rewritten to %s (with %s)" % (entry["title"], entry["url"], name)) except UrlRewritingError, r: # increase failcount # count = self.shared_cache.storedefault(entry['url'], 1) # count += 1 raise UrlRewritingError("URL rewriting %s failed: %s" % (name, str(r.value))) except PluginError, e: raise UrlRewritingError("URL rewriting %s failed: %s" % (name, str(e.value))) except Exception, e: log.exception(e) raise UrlRewritingError("%s: Internal error with url %s" % (name, entry["url"])) register_plugin(PluginUrlRewriting, "urlrewriting", builtin=True) register_task_phase("urlrewrite", before="download")