def register_plugin():
plugin.register(PluginUrlRewriting,
'urlrewriting',
builtin=True,
api_ver=2)
plugin.register(DisableUrlRewriter, 'disable_urlrewriters', api_ver=2)
plugin.register_task_phase('urlrewrite', before='download')
return False
@priority(255)
def url_rewrite(self, task, entry):
"""Rewrites given entry url. Raises UrlRewritingError if failed."""
tries = 0
while self.url_rewritable(task, entry):
tries += 1
if tries > 300:
raise UrlRewritingError('URL rewriting was left in infinite loop while rewriting url for %s, some rewriter is returning always True' % entry)
for urlrewriter in get_plugins_by_group('urlrewriter'):
name = urlrewriter.name
try:
if urlrewriter.instance.url_rewritable(task, entry):
log.debug('Url rewriting %s' % entry['url'])
urlrewriter.instance.url_rewrite(task, entry)
log.info('Entry \'%s\' URL rewritten to %s (with %s)' % (entry['title'], entry['url'], name))
except UrlRewritingError as r:
# increase failcount
#count = self.shared_cache.storedefault(entry['url'], 1)
#count += 1
raise UrlRewritingError('URL rewriting %s failed: %s' % (name, r.value))
except PluginError as e:
raise UrlRewritingError('URL rewriting %s failed: %s' % (name, e.value))
except Exception as e:
log.exception(e)
raise UrlRewritingError('%s: Internal error with url %s' % (name, entry['url']))
register_plugin(PluginUrlRewriting, 'urlrewriting', builtin=True)
register_task_phase('urlrewrite', before='download')
root.accept('text')
return root
def on_task_start(self, task, config):
urlrewrite = get_plugin_by_name('urlrewriting')['instance']
for disable in config:
try:
get_plugin_by_name(disable)
except DependencyError:
log.critical('Unknown url-rewriter %s' % disable)
continue
log.debug('Disabling url rewriter %s' % disable)
urlrewrite.disabled_rewriters.append(disable)
def on_task_exit(self, task, config):
urlrewrite = get_plugin_by_name('urlrewriting')['instance']
for disable in config:
log.debug('Enabling url rewriter %s' % disable)
try:
urlrewrite.disabled_rewriters.remove(disable)
except ValueError:
log.debug('%s does not exists' % disable)
on_task_abort = on_task_exit
register_plugin(PluginUrlRewriting, 'urlrewriting', builtin=True, api_ver=2)
register_plugin(DisableUrlRewriter, 'disable_urlrewriters', api_ver=2)
register_task_phase('urlrewrite', before='download')
def register_plugin():
plugin.register(PluginUrlRewriting, 'urlrewriting', builtin=True, api_ver=2)
plugin.register(DisableUrlRewriter, 'disable_urlrewriters', api_ver=2)
plugin.register_task_phase('urlrewrite', before='download')
"""Rewrites given entry url. Raises UrlRewritingError if failed."""
tries = 0
while self.url_rewritable(task, entry):
tries += 1
if tries > 300:
raise UrlRewritingError(
"URL rewriting was left in infinite loop while rewriting url for %s, some rewriter is returning always True"
% entry
)
for urlrewriter in get_plugins_by_group("urlrewriter"):
name = urlrewriter.name
try:
if urlrewriter.instance.url_rewritable(task, entry):
log.debug("Url rewriting %s" % entry["url"])
urlrewriter.instance.url_rewrite(task, entry)
log.info("Entry '%s' URL rewritten to %s (with %s)" % (entry["title"], entry["url"], name))
except UrlRewritingError, r:
# increase failcount
# count = self.shared_cache.storedefault(entry['url'], 1)
# count += 1
raise UrlRewritingError("URL rewriting %s failed: %s" % (name, str(r.value)))
except PluginError, e:
raise UrlRewritingError("URL rewriting %s failed: %s" % (name, str(e.value)))
except Exception, e:
log.exception(e)
raise UrlRewritingError("%s: Internal error with url %s" % (name, entry["url"]))
register_plugin(PluginUrlRewriting, "urlrewriting", builtin=True)
register_task_phase("urlrewrite", before="download")