def url_rewrite(self, task, entry):
"""Rewrites given entry url. Raises UrlRewritingError if failed."""
tries = 0
while self.url_rewritable(task, entry):
tries += 1
if tries > 20:
raise UrlRewritingError('URL rewriting was left in infinite loop while rewriting url for %s, '
'some rewriter is returning always True' % entry)
for urlrewriter in plugin.get_plugins_by_group('urlrewriter'):
name = urlrewriter.name
if name in self.disabled_rewriters:
log.trace('Skipping rewriter %s since it\'s disabled' % name)
continue
try:
if urlrewriter.instance.url_rewritable(task, entry):
log.debug('Url rewriting %s' % entry['url'])
urlrewriter.instance.url_rewrite(task, entry)
log.info('Entry \'%s\' URL rewritten to %s (with %s)' % (entry['title'], entry['url'], name))
except UrlRewritingError as r:
# increase failcount
#count = self.shared_cache.storedefault(entry['url'], 1)
#count += 1
raise UrlRewritingError('URL rewriting %s failed: %s' % (name, r.value))
except plugin.PluginError as e:
raise UrlRewritingError('URL rewriting %s failed: %s' % (name, e.value))
except Exception as e:
log.exception(e)
raise UrlRewritingError('%s: Internal error with url %s' % (name, entry['url']))
def url_rewrite(self, task, entry):
"""Rewrites given entry url. Raises UrlRewritingError if failed."""
tries = 0
while self.url_rewritable(task, entry):
tries += 1
if tries > 300:
raise UrlRewritingError(
"URL rewriting was left in infinite loop while rewriting url for %s, some rewriter is returning always True"
% entry
)
for urlrewriter in get_plugins_by_group("urlrewriter"):
name = urlrewriter.name
try:
if urlrewriter.instance.url_rewritable(task, entry):
log.debug("Url rewriting %s" % entry["url"])
urlrewriter.instance.url_rewrite(task, entry)
log.info("Entry '%s' URL rewritten to %s (with %s)" % (entry["title"], entry["url"], name))
except UrlRewritingError, r:
# increase failcount
# count = self.shared_cache.storedefault(entry['url'], 1)
# count += 1
raise UrlRewritingError("URL rewriting %s failed: %s" % (name, str(r.value)))
except PluginError, e:
raise UrlRewritingError("URL rewriting %s failed: %s" % (name, str(e.value)))
except Exception, e:
log.exception(e)
raise UrlRewritingError("%s: Internal error with url %s" % (name, entry["url"]))
def url_rewrite(self, task, entry):
"""Rewrites given entry url. Raises UrlRewritingError if failed."""
tries = 0
while self.url_rewritable(task, entry):
tries += 1
if tries > 20:
raise UrlRewritingError('URL rewriting was left in infinite loop while rewriting url for %s, '
'some rewriter is returning always True' % entry)
for urlrewriter in get_plugins_by_group('urlrewriter'):
name = urlrewriter.name
if name in self.disabled_rewriters:
log.trace('Skipping rewriter %s since it\'s disabled' % name)
continue
try:
if urlrewriter.instance.url_rewritable(task, entry):
log.debug('Url rewriting %s' % entry['url'])
urlrewriter.instance.url_rewrite(task, entry)
log.info('Entry \'%s\' URL rewritten to %s (with %s)' % (entry['title'], entry['url'], name))
except UrlRewritingError as r:
# increase failcount
#count = self.shared_cache.storedefault(entry['url'], 1)
#count += 1
raise UrlRewritingError('URL rewriting %s failed: %s' % (name, r.value))
except PluginError as e:
raise UrlRewritingError('URL rewriting %s failed: %s' % (name, e.value))
except Exception as e:
log.exception(e)
raise UrlRewritingError('%s: Internal error with url %s' % (name, entry['url']))
def url_rewritable(self, task, entry):
"""Return True if entry is urlrewritable by registered rewriter."""
for urlrewriter in get_plugins_by_group('urlrewriter'):
log.trace('checking urlrewriter %s' % urlrewriter.name)
if urlrewriter.instance.url_rewritable(self, entry):
return True
return False
def url_rewritable(self, task, entry):
"""Return True if entry is urlrewritable by registered rewriter."""
for urlrewriter in get_plugins_by_group('urlrewriter'):
log.trace('checking urlrewriter %s' % urlrewriter.name)
if urlrewriter.instance.url_rewritable(self, entry):
return True
return False
def on_process_start(self, task):
if task.manager.options.search_plugins:
task.manager.disable_tasks()
header = '-- Supported search plugins: '
header = header + '-' * (79 - len(header))
print header
for plugin in get_plugins_by_group('search'):
print ' %s' % plugin.name
print '-' * 79
def url_rewritable(self, task, entry):
"""Return True if entry is urlrewritable by registered rewriter."""
for urlrewriter in get_plugins_by_group('urlrewriter'):
if urlrewriter.name in self.disabled_rewriters:
log.trace('Skipping rewriter %s since it\'s disabled' % urlrewriter.name)
continue
log.trace('checking urlrewriter %s' % urlrewriter.name)
if urlrewriter.instance.url_rewritable(self, entry):
return True
return False
def url_rewritable(self, task, entry):
"""Return True if entry is urlrewritable by registered rewriter."""
for urlrewriter in plugin.get_plugins_by_group('urlrewriter'):
if urlrewriter.name in self.disabled_rewriters:
log.trace('Skipping rewriter %s since it\'s disabled' % urlrewriter.name)
continue
log.trace('checking urlrewriter %s' % urlrewriter.name)
if urlrewriter.instance.url_rewritable(self, entry):
return True
return False
def estimate(self, entry):
"""
Estimate release schedule for Entry
:param entry:
:return: estimated date of released for the entry, None if it can't figure it out
"""
log.info(entry['title'])
estimators = get_plugins_by_group('estimate_release')
for estimator in estimators:
estimate = estimator.instance.estimate(entry)
# return first successful estimation
if estimate is not None:
return estimate
def estimate(self, entry):
"""
Estimate release schedule for Entry
:param entry:
:return: estimated date of released for the entry, None if it can't figure it out
"""
log.debug(entry['title'])
estimators = [e.instance.estimate for e in get_plugins_by_group('estimate_release')]
for estimator in sorted(estimators, key=lambda e: getattr(e, 'priority', DEFAULT_PRIORITY), reverse=True):
estimate = estimator(entry)
# return first successful estimation
if estimate is not None:
return estimate
def validator(self):
from flexget import validator
discover = validator.factory("dict")
inputs = discover.accept("list", key="what", required=True)
add_plugin_validators(inputs, phase="input", excluded=["discover"])
searches = discover.accept("list", key="from", required=True)
no_config = searches.accept("choice")
for plugin in get_plugins_by_group("search"):
if hasattr(plugin.instance, "validator"):
searches.accept("dict").accept(plugin.instance.validator(), key=plugin.name)
else:
no_config.accept(plugin.name)
discover.accept("integer", key="limit")
discover.accept("choice", key="type").accept_choices(["normal", "exact", "movies"])
return discover
def validator(self):
from flexget import validator
search = validator.factory('list')
names = []
for plugin in get_plugins_by_group('search'):
# If the plugin has a validator, get it's validator and make it a
# child of the search plugins
if not hasattr(plugin.instance, 'validator'):
# Create choice validator for plugins without validators later
names.append(plugin.name)
else:
plugin_validator = plugin.instance.validator()
if isinstance(plugin_validator, validator.Validator):
search.accept('dict').accept(plugin_validator, key=plugin.name)
else:
log.error("plugin %s has a validator method, but does not "
"return a validator instance when called with "
"search plugin." % plugin.name)
search.accept('choice').accept_choices(names)
return search
def validator(self):
from flexget import validator
discover = validator.factory('dict')
inputs = discover.accept('list', key='what', required=True).accept('dict')
for plugin in get_plugins_by_phase('input'):
if hasattr(plugin.instance, 'validator'):
inputs.accept(plugin.instance.validator, key=plugin.name)
searches = discover.accept('list', key='from', required=True)
no_config = searches.accept('choice')
for plugin in get_plugins_by_group('search'):
if hasattr(plugin.instance, 'validator'):
searches.accept('dict').accept(plugin.instance.validator, key=plugin.name)
else:
no_config.accept(plugin.name)
discover.accept('integer', key='limit')
discover.accept('choice', key='type').accept_choices(['any', 'normal', 'exact', 'movies'])
return discover
def on_feed_urlrewrite(self, feed):
# no searches in unit test mode
if feed.manager.unit_test:
return
plugins = {}
for plugin in get_plugins_by_group('search'):
plugins[plugin.name] = plugin.instance
# search accepted and imaginary entries
for entry in feed.accepted + [e for e in feed.entries if e.get('imaginary')]:
found = False
# loop through configured searches
search_plugins = feed.config.get('search', [])
for name in search_plugins:
config = None
if isinstance(name, dict):
# assume the name is the first/only key in the dict.
name, config = name.items()[0]
log.verbose('Searching `%s` from %s' % (entry['title'], name))
try:
results = plugins[name].search(entry['title'], StringComparator(cutoff=0.9), config)
if results:
url = results[0]['url']
log.debug('Found url: %s' % url)
entry['url'] = url
found = True
break
except (PluginError, PluginWarning), pw:
log.verbose('Failed: %s' % pw.value)
continue
# Search failed
if not found:
# If I don't have a URL, doesn't matter if I'm immortal...
entry['immortal'] = False
feed.reject(entry, 'search failed')
else:
# Populate quality
# TODO: why does search plugin invoke quality here?
get_plugin_by_name('metainfo_quality').instance.get_quality(entry)
def on_task_urlrewrite(self, task, config):
# no searches in unit test mode
if task.manager.unit_test:
return
plugins = {}
for plugin in get_plugins_by_group('search'):
plugins[plugin.name] = plugin.instance
# search accepted
for entry in task.accepted:
# loop through configured searches
for name in config:
search_config = None
if isinstance(name, dict):
# assume the name is the first/only key in the dict.
name, search_config = name.items()[0]
log.verbose('Searching `%s` from %s' % (entry['title'], name))
try:
results = plugins[name].search(entry, search_config)
matcher = SequenceMatcher(a=entry['title'])
for result in results:
matcher.set_seq2(result['title'])
if matcher.ratio() > 0.9:
log.debug('Found url: %s', result['url'])
entry['url'] = result['url']
break
else:
continue
break
except (PluginError, PluginWarning) as pw:
log.verbose('Failed: %s' % pw.value)
continue
# Search failed
else:
# If I don't have a URL, doesn't matter if I'm immortal...
entry['immortal'] = False
entry.reject('search failed')
def on_task_urlrewrite(self, task, config):
# no searches in unit test mode
if task.manager.unit_test:
return
plugins = {}
for plugin in get_plugins_by_group('search'):
plugins[plugin.name] = plugin.instance
# search accepted
for entry in task.accepted:
found = False
# loop through configured searches
for name in config:
search_config = None
if isinstance(name, dict):
# assume the name is the first/only key in the dict.
name, search_config = name.items()[0]
log.verbose('Searching `%s` from %s' % (entry['title'], name))
try:
results = plugins[name].search(
entry['title'], StringComparator(cutoff=0.9),
search_config)
if results:
url = results[0]['url']
log.debug('Found url: %s' % url)
entry['url'] = url
found = True
break
except (PluginError, PluginWarning), pw:
log.verbose('Failed: %s' % pw.value)
continue
# Search failed
if not found:
# If I don't have a URL, doesn't matter if I'm immortal...
entry['immortal'] = False
task.reject(entry, 'search failed')
def validator(self):
from flexget import validator
discover = validator.factory('dict')
inputs = discover.accept('list', key='what',
required=True).accept('dict')
for plugin in get_plugins_by_phase('input'):
if hasattr(plugin.instance, 'validator'):
inputs.accept(plugin.instance.validator, key=plugin.name)
searches = discover.accept('list', key='from', required=True)
no_config = searches.accept('choice')
for plugin in get_plugins_by_group('search'):
if hasattr(plugin.instance, 'validator'):
searches.accept('dict').accept(plugin.instance.validator,
key=plugin.name)
else:
no_config.accept(plugin.name)
discover.accept('integer', key='limit')
discover.accept('choice', key='type').accept_choices(
['any', 'normal', 'exact', 'movies'])
return discover
def on_feed_urlrewrite(self, feed, config):
# no searches in unit test mode
if feed.manager.unit_test:
return
plugins = {}
for plugin in get_plugins_by_group('search'):
plugins[plugin.name] = plugin.instance
# search accepted
for entry in feed.accepted:
found = False
# loop through configured searches
for name in config:
search_config = None
if isinstance(name, dict):
# assume the name is the first/only key in the dict.
name, search_config = name.items()[0]
log.verbose('Searching `%s` from %s' % (entry['title'], name))
try:
results = plugins[name].search(entry['title'], StringComparator(cutoff=0.9), search_config)
if results:
url = results[0]['url']
log.debug('Found url: %s' % url)
entry['url'] = url
found = True
break
except (PluginError, PluginWarning), pw:
log.verbose('Failed: %s' % pw.value)
continue
# Search failed
if not found:
# If I don't have a URL, doesn't matter if I'm immortal...
entry['immortal'] = False
feed.reject(entry, 'search failed')