def createIndex(self):
plugins = []
active = []
deactive = []
plugin_manager = get_plugin_manager()
for pluginname in plugin_manager.hookPlugins:
try:
# hookClass = getattr(plugin, plugin.__name__)
if self.core.config.getPlugin(pluginname, "activated"):
pluginClass = plugin_manager.loadClass("hooks", pluginname)
if not pluginClass: continue
plugin = pluginClass(self.core, self)
plugins.append(plugin)
self.pluginMap[pluginClass.__name__] = plugin
if plugin.isActivated():
active.append(pluginClass.__name__)
else:
deactive.append(pluginname)
except:
self.log.warning(
_("Failed activating %(name)s") % {"name": pluginname})
if self.core.debug:
traceback.print_exc()
self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active)))
self.log.info(
_("Deactivate plugins: %s") % ", ".join(sorted(deactive)))
self.plugins = plugins
def activate(self):
plugin_manager = get_plugin_manager()
for p in ("UnRar", "SevenZip", "UnZip", "UnTar"):
try:
module = plugin_manager.loadModule("internal", p)
klass = getattr(module, p)
if klass.find():
self.extractors.append(klass)
if klass.REPAIR:
self.repair = self.config.get('repair')
except OSError as e:
if e.errno == 2:
self.log_warning(_("No %s installed") % p)
else:
self.log_warning(_("Could not activate: %s") % p, e)
except Exception as e:
self.log_warning(_("Could not activate: %s") % p, e)
if self.extractors:
self.log_debug(*["Found %s %s" % (Extractor.__name__,
Extractor.VERSION) for Extractor in self.extractors])
self.extract_queued() #: Resume unfinished extractions
else:
self.log_info(_("No Extract plugins activated"))
def decryptContainer(self, plugin, url):
data = []
# only works on container plugins
self.m.log.debug("Pre decrypting %s with %s" % (url, plugin))
# dummy pyfile
pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1,
-1)
pyfile.initPlugin()
# little plugin lifecycle
try:
pyfile.plugin.setup()
pyfile.plugin.loadToDisk()
pyfile.plugin.decrypt(pyfile)
pyfile.plugin.deleteTmp()
for pack in pyfile.plugin.packages:
pyfile.plugin.urls.extend(pack[1])
data = get_plugin_manager().parseUrls(pyfile.plugin.urls)
self.m.log.debug("Got %d links." % len(data))
except Exception as e:
self.m.log.debug("Pre decrypting error: %s" % str(e))
finally:
pyfile.release()
return data
def deactivate(self):
"""
Remove override for all plugins.
"""
self.log_info(_("Reverting back to default hosters"))
try:
get_hook_manager().removeEvent("plugin_updated",
self.plugins_updated)
except ValueError:
pass
self.periodical.stop()
self.fail_count = 0
if self.supported:
self.log_debug("Unload: %s" % ", ".join(self.supported))
for plugin in self.supported:
self.unload_plugin(plugin)
#: Reset pattern
hdict = get_plugin_manager().plugins[self.plugintype][self.classname]
hdict['pattern'] = getattr(self.pluginclass, "__pattern__",
r'^unmatchable$')
hdict['re'] = re.compile(hdict['pattern'])
def autoreload_plugins(self):
"""
Reload and reindex all modified plugins
"""
reloads = []
modules = list(filter(
lambda m: m and (m.__name__.startswith("module.plugins.") or
m.__name__.startswith("userplugins.")) and
m.__name__.count(".") >= 2, sys.modules.values()
))
plugin_manager = get_plugin_manager()
for m in modules:
root, plugin_type, plugin_name = m.__name__.rsplit(".", 2)
plugin_id = (plugin_type, plugin_name)
if plugin_type in plugin_manager.plugins:
f = m.__file__.replace(".pyc", ".py")
if not os.path.isfile(f):
continue
mtime = os.path.getmtime(f)
if plugin_id not in self.mtimes:
self.mtimes[plugin_id] = mtime
elif self.mtimes[plugin_id] < mtime:
reloads.append(plugin_id)
self.mtimes[plugin_id] = mtime
return bool(plugin_manager.reloadPlugins(reloads))
def update_plugins(self):
server_data = self.server_response()
if not server_data or server_data[0] != self.pyload.api.getServerVersion():
return 0
updated = self._update_plugins(server_data)
if updated:
self.log_info(_("*** Plugins updated ***"))
if get_plugin_manager().reloadPlugins(updated):
exitcode = 1
else:
self.log_warning(_("You have to restart pyLoad to use the updated plugins"))
self.info['plugins'] = True
exitcode = 2
paused = get_thread_manager().pause
self.pyload.api.pauseServer()
self.manager.dispatchEvent("plugin_updated", updated)
if not paused:
self.pyload.api.unpauseServer()
else:
self.log_info(_("All plugins are up to date!"))
exitcode = 0
#: Exit codes:
#: 0 = No plugin updated
#: 1 = Plugins updated
#: 2 = Plugins updated, but restart required
return exitcode
def is_hoster_link(self, link):
#declare all links as hoster links so the filter will work on all links
if self.config.get('filter_all'):
return True
for item in get_plugin_manager().hosterPlugins.items():
if item[1]['re'].match(link):
return True
return False
def unload_plugin(self, plugin):
#: Reset module
hdict = get_plugin_manager().plugins[self.plugintype][plugin]
if "module" in hdict:
hdict.pop('module', None)
if "new_module" in hdict:
hdict.pop('new_module', None)
hdict.pop('new_name', None)
def initPlugin(self):
""" inits plugin instance """
if not self.plugin:
plugin_manager = get_plugin_manager()
self.pluginmodule = plugin_manager.getPlugin(self.pluginname)
self.pluginclass = getattr(
self.pluginmodule,
plugin_manager.getPluginName(self.pluginname))
self.plugin = self.pluginclass(self)
def getAccountPlugin(self, plugin):
"""get account instance for plugin or None if anonymous"""
if plugin in self.accounts:
if plugin not in self.plugins:
self.plugins[plugin] = get_plugin_manager().loadClass(
"accounts", plugin)(self, self.accounts[plugin])
return self.plugins[plugin]
else:
return None
def checkAndAddPackages(self, links, dest=Destination.Queue):
"""Checks online status, retrieves names, and will add packages.\
Because of this packages are not added immediatly, only for internal use.
:param links: list of urls
:param dest: `Destination`
:return: None
"""
data = get_plugin_manager().parseUrls(links)
get_thread_manager().createResultThread(data, True)
def init(self):
self.__pattern__ = get_plugin_manager().crypterPlugins[self.classname][
'pattern'] # @TODO: Recheck in 0.4.10
self.PLUGIN_DOMAIN = re.match(self.__pattern__,
self.pyfile.url).group("DOMAIN").lower()
self.PLUGIN_NAME = "".join(
part.capitalize()
for part in re.split(r'\.|\d+|-', self.PLUGIN_DOMAIN)
if part != '.')
def get_pattern(self, type, plugin):
if self.config.get('use_%s_list' % type):
plugin_list = self.config.get('%s_list' % type)
plugin_list = plugin_list.replace(' ', '').replace('\\', '')
plugin_list = plugin_list.replace('|', ',').replace(';', ',')
plugin_list = plugin_list.lower().split(',')
plugin_set = set(plugin_list)
if self.config.get('use_builtin_list'):
builtin_list = getattr(self, "BUILTIN_%sS" % type.upper())
plugin_set.update(builtin_list)
plugin_set.difference_update(('', ))
if not plugin_set:
self.log_info(_("No %s to handle") % type)
return
match_list = '|'.join(sorted(plugin_set)).replace('.', '\.')
pattern = self._regexmap[type][1] % match_list
self.log_info(
_("Handle %d %s%s: %s") %
(len(plugin_set), type, "" if len(plugin_set) == 1 else "s",
match_list.replace('\.', '.').replace('|', ', ')))
else:
plugin_list = []
isXFS = lambda klass: any(
k.__name__.startswith("XFS") for k in inspect.getmro(klass))
plugin_manager = get_plugin_manager()
for p in plugin_manager.plugins[type].values():
try:
klass = plugin_manager.loadClass(type, p['name'])
except AttributeError as e:
self.log_debug(e, trace=True)
continue
if hasattr(klass, "PLUGIN_DOMAIN"
) and klass.PLUGIN_DOMAIN and isXFS(klass):
plugin_list.append(klass.PLUGIN_DOMAIN)
if plugin_list:
unmatch_list = '|'.join(sorted(plugin_list)).replace('.', '\.')
pattern = self._regexmap[type][0] % unmatch_list
else:
pattern = self._regexmap[type][0]
self.log_info(_("Auto-discover new %ss") % type)
return pattern
def _load(self, type, plugin):
dict = get_plugin_manager().plugins[type][plugin]
pattern = self.get_pattern(type, plugin)
if not pattern:
return
dict['pattern'] = pattern
dict['re'] = re.compile(pattern)
self.log_debug("Pattern for %ss: %s" % (type, pattern))
def addLinks(self, urls, package):
"""adds links"""
get_hook_manager().dispatchEvent("linksAdded", urls, package)
data = get_plugin_manager().parseUrls(urls)
self.db.addLinks(data, package)
get_thread_manager().createInfoThread(data, package)
#@TODO change from reloadAll event to package update event
get_pull_manager().addEvent(ReloadAllEvent("collector"))
def getDecryptJob(self):
"""return job for decrypting"""
if "decrypt" in self.jobCache:
return None
plugin_manager = get_plugin_manager()
plugins = plugin_manager.crypterPlugins.keys() + plugin_manager.containerPlugins.keys()
plugins = str(tuple(plugins))
jobs = self.db.getPluginJob(plugins)
if jobs:
return self.getFile(jobs[0])
else:
self.jobCache["decrypt"] = "empty"
return None
def checkURLs(self, urls):
""" Gets urls and returns pluginname mapped to list of matches urls.
:param urls:
:return: {plugin: urls}
"""
data = get_plugin_manager().parseUrls(urls)
plugins = {}
for url, plugin in data:
if plugin in plugins:
plugins[plugin].append(url)
else:
plugins[plugin] = [url]
return plugins
def links_added(self, links, pid):
plugin_manager = get_plugin_manager()
hosterdict = plugin_manager.hosterPlugins
linkdict = self.pyload.api.checkURLs(links)
premiumplugins = {
account.type
for account in self.pyload.api.getAccounts(False)
if account.valid and account.premium
}
multihosters = {
hoster
for hoster in plugin_manager.hosterPlugins
if 'new_name' in hosterdict[hoster]
and hosterdict[hoster]['new_name'] in premiumplugins
}
excluded = list(
map(
lambda domain: "".join(part.capitalize() for part in re.split(
r'(\.|\d+)', domain) if part != '.'),
self.config.get('excluded').replace(' ', '').replace(
',', '|').replace(';', '|').split('|'),
))
included = list(
map(
lambda domain: "".join(part.capitalize() for part in re.split(
r'(\.|\d+)', domain) if part != '.'),
self.config.get('included').replace(' ', '').replace(
',', '|').replace(';', '|').split('|'),
))
hosterlist = (premiumplugins
| multihosters).union(excluded).difference(included)
#: Found at least one hoster with account or multihoster
if not any(True
for pluginname in linkdict if pluginname in hosterlist):
return
for pluginname in set(linkdict.keys()) - hosterlist:
self.log_info(_("Remove links of plugin: %s") % pluginname)
for link in linkdict[pluginname]:
self.log_debug("Remove link: %s" % link)
links.remove(link)
def activateHook(self, plugin):
#check if already loaded
for inst in self.plugins:
if inst.__name__ == plugin:
return
pluginClass = get_plugin_manager().loadClass("hooks", plugin)
if not pluginClass: return
self.log.debug("Plugin loaded: %s" % plugin)
plugin = pluginClass(self.core, self)
self.plugins.append(plugin)
self.pluginMap[pluginClass.__name__] = plugin
# call core Ready
thread.start_new_thread(plugin.coreReady, tuple())
def init_plugin(self):
plugin_manager = get_plugin_manager()
plugin, self.plugintype = plugin_manager.findPlugin(self.classname)
if plugin:
self.pluginmodule = plugin_manager.loadModule(
self.plugintype, self.classname)
self.pluginclass = plugin_manager.loadClass(
self.plugintype, self.classname)
get_hook_manager().addEvent("plugin_updated", self.plugins_updated)
interval = self.config.get('mh_interval', 12) * 60 * 60
self.periodical.start(interval, threaded=True, delay=2)
else:
self.log_warning(
_("Multi-hoster feature will be deactivated due missing plugin reference"
))
def whitelist(self, links, filters):
plugindict = dict(get_plugin_manager().parseUrls(links))
linkcount = len(links)
links[:] = [
link
for link in links
if (
any(link.find(_filter) != -1 for _filter in filters) or
not self.is_hoster_link(link) and
plugindict[link] != "BasePlugin"
)
]
linkcount -= len(links)
if linkcount > 0:
linkstring = '' if self.config.get('filter_all') else 'hoster '
linkstring += 'link' if linkcount == 1 else 'links'
self.log_warning(
_('Whitelist filter removed %s %s not containing (%s)') %
(linkcount, linkstring, ', '.join(filters)))
def checkOnlineStatus(self, urls):
""" initiates online status check
:param urls:
:return: initial set of data as `OnlineCheck` instance containing the result id
"""
data = get_plugin_manager().parseUrls(urls)
rid = get_thread_manager().createResultThread(data, False)
tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0)))
for url, pluginname in data]
data = parseNames(tmp)
result = {}
for k, v in six.iteritems(data):
for url, status in v:
status.packagename = k
result[url] = status
return OnlineCheck(rid, result)
def decrypt_image(self,
img,
input_type='jpg',
output_type='textual',
ocr=False,
timeout=120):
"""
Loads a captcha and decrypts it with ocr, plugin, user input
:param img: image raw data
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
:param input_type: Type of the Image
:param output_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
:param ocr: if True, builtin ocr is used. if string, the OCR plugin name is used
:return: result of decrypting
"""
result = None
time_ref = ("%.2f" % time.time())[-6:].replace(".", "")
with open(
os.path.join(
"tmp", "captcha_image_%s_%s.%s" %
(self.pyfile.plugin.__name__, time_ref, input_type)),
"wb") as img_f:
img_f.write(img)
if ocr:
self.log_info(_("Using OCR to decrypt captcha..."))
if isinstance(ocr, six.string_types):
_OCR = get_plugin_manager().loadClass(
"captcha", ocr) #: Rename `captcha` to `ocr` in 0.4.10
result = _OCR(self.pyfile).recognize(img_f.name)
else:
result = self.recognize(img_f.name)
if not result:
self.log_warning(_("No OCR result"))
if not result:
captcha_manager = get_captcha_manager()
try:
self.task = captcha_manager.newTask(img, input_type,
img_f.name, output_type)
captcha_manager.handleCaptcha(self.task)
# @TODO: Move to `CaptchaManager` in 0.4.10
self.task.setWaiting(max(timeout, 50))
while self.task.isWaiting():
self.pyfile.plugin.check_status()
time.sleep(1)
finally:
captcha_manager.removeTask(self.task)
result = self.task.result
if self.task.error:
if not self.task.handler and not self.pyload.isClientConnected(
):
self.log_warning(
_("No Client connected for captcha decrypting"))
self.fail(_("No Client connected for captcha decrypting"))
else:
self.pyfile.plugin.retry_captcha(msg=self.task.error)
elif self.task.result:
self.log_info(_("Captcha result: `%s`") % (result, ))
else:
self.pyfile.plugin.retry_captcha(
msg=_("No captcha result obtained in appropriate timing"))
if not self.pyload.debug:
self.remove(img_f.name, trash=False)
return result
def init(self):
self.PLUGIN_NAME = get_plugin_manager().crypterPlugins.get(
self.classname)['name']
def _override(self):
prev_supported = self.supported
new_supported = []
excluded = []
self.supported = []
plugin_manager = get_plugin_manager()
if self.plugintype == "hoster":
plugin_map = {
name.lower(): name
for name in plugin_manager.hosterPlugins.keys()
}
account_list = [
account.type.lower()
for account in self.pyload.api.getAccounts(False)
if account.valid and account.premium
]
else:
plugin_map = {}
account_list = [
name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1]
for name in plugin_manager.crypterPlugins.keys()
]
for plugin in self.get_plugins():
name = remove_chars(plugin, "-.")
if name in account_list:
excluded.append(plugin)
else:
if name in plugin_map:
self.supported.append(plugin_map[name])
else:
new_supported.append(plugin)
removed = [
plugin for plugin in prev_supported if plugin not in self.supported
]
if removed:
self.log_debug("Unload: %s" % ", ".join(removed))
for plugin in removed:
self.unload_plugin(plugin)
if not self.supported and not new_supported:
self.log_error(_("No %s loaded") % self.plugintype)
return
#: Inject plugin plugin
self.log_debug("Overwritten %ss: %s" %
(self.plugintype, ", ".join(sorted(self.supported))))
for plugin in self.supported:
hdict = plugin_manager.plugins[self.plugintype][plugin]
hdict['new_module'] = self.pluginmodule
hdict['new_name'] = self.classname
if excluded:
self.log_info(
_("%ss not overwritten: %s") %
(self.plugintype.capitalize(), ", ".join(sorted(excluded))))
if new_supported:
plugins = sorted(new_supported)
self.log_debug("New %ss: %s" %
(self.plugintype, ", ".join(plugins)))
#: Create new regexp
pattern = r'.*(?P<DOMAIN>%s).*' % "|".join(
x.replace('.', '\.') for x in plugins)
if (hasattr(self.pluginclass, "__pattern__") and isinstance(
self.pluginclass.__pattern__, six.string_types)
and "://" in self.pluginclass.__pattern__):
pattern = r'%s|%s' % (self.pluginclass.__pattern__, pattern)
self.log_debug("Pattern: %s" % pattern)
hdict = plugin_manager.plugins[self.plugintype][self.classname]
hdict['pattern'] = pattern
hdict['re'] = re.compile(pattern)
def initAccountPlugins(self):
"""init names"""
for name in get_plugin_manager().getAccountPlugins():
self.accounts[name] = {}
def _unload(self, type, plugin):
dict = get_plugin_manager().plugins[type][plugin]
dict['pattern'] = r'^unmatchable$'
dict['re'] = re.compile(dict['pattern'])
def setup_base(self):
klass = get_plugin_manager().loadClass("hoster", self.classname)
self.get_info = klass.get_info
SimpleHoster.setup_base(self)
def decryptCaptcha(self,
url,
get={},
post={},
cookies=False,
forceUser=False,
imgtype='jpg',
result_type='textual'):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
:param forceUser: if True, ocr is not used
:param imgtype: Type of the Image
:param result_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
:return: result of decrypting
"""
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".", "")
temp_file = open(
join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)),
"wb")
temp_file.write(img)
temp_file.close()
plugin_manager = get_plugin_manager()
has_plugin = self.__name__ in plugin_manager.captchaPlugins
if self.core.captcha:
Ocr = plugin_manager.loadClass("captcha", self.__name__)
else:
Ocr = None
if Ocr and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
if self.pyfile.abort:
raise Abort
ocr = Ocr()
result = ocr.get_captcha(temp_file.name)
else:
captcha_manager = get_captcha_manager()
task = captcha_manager.newTask(img, imgtype, temp_file.name,
result_type)
self.cTask = task
captcha_manager.handleCaptcha(task)
while task.isWaiting():
if self.pyfile.abort:
captcha_manager.removeTask(task)
raise Abort
sleep(1)
captcha_manager.removeTask(task)
if task.error and has_plugin: #ignore default error message since the user could use OCR
self.fail(
_("Pil and tesseract not installed and no Client connected for captcha decrypting"
))
elif task.error:
self.fail(task.error)
elif not task.result:
self.fail(
_("No captcha result obtained in appropiate time by any of the plugins."
))
result = task.result
self.log.debug("Received captcha result: %s" % str(result))
if not self.core.debug:
try:
remove(temp_file.name)
except:
pass
return result
def _update_plugins(self, server_data):
"""
Check for plugin updates
"""
updated = []
updatelist, blacklist = self.parse_updates(server_data)
url = server_data[1]
req = get_request_factory().getRequest(self.classname)
if blacklist:
#@NOTE: Protect UpdateManager from self-removing
if IS_WINDOWS:
#@NOTE: Windows filesystem is case insensitive, make sure we do not delete legitimate plugins
whitelisted_plugins = [
(plugin['type'], plugin['name'].upper()) for plugin in updatelist]
blacklisted_plugins = [(plugin['type'], plugin['name']) for plugin in blacklist
if not (plugin['name'] == self.classname and plugin['type'] == self.__type__)
and (plugin['type'], plugin['name'].upper()) not in whitelisted_plugins]
else:
blacklisted_plugins = [(plugin['type'], plugin['name']) for plugin in blacklist
if not (plugin['name'] == self.classname and plugin['type'] == self.__type__)]
c = 1
l = len(blacklisted_plugins)
for idx, plugin in enumerate(updatelist):
if c > l:
break
plugin_name = plugin['name']
plugin_type = plugin['type']
for t, n in blacklisted_plugins:
if n != plugin_name or t != plugin_type:
continue
updatelist.pop(idx)
c += 1
break
for t, n in self.remove_plugins(blacklisted_plugins):
self.log_info(_("Removed blacklisted plugin: %(type)s %(name)s") %
{'type': t.upper(),
'name': n,})
plugin_manager = get_plugin_manager()
for plugin in updatelist:
plugin_name = plugin['name']
plugin_type = plugin['type']
plugin_version = plugin['version']
# @TODO: Remove rstrip in 0.4.10
plugins = getattr(plugin_manager, "%sPlugins" % plugin_type.rstrip('s'))
oldver = float(plugins[plugin_name]['v']) if plugin_name in plugins else None
try:
newver = float(plugin_version)
except ValueError:
self.log_error(_("Error updating plugin: %s %s") % (plugin_type.rstrip('s').upper(), plugin_name),
_("Bad version number on the server"))
continue
if not oldver:
msg = "New plugin: %(type)s %(name)s (v%(newver).2f)"
elif newver > oldver:
msg = "New version of plugin: %(type)s %(name)s (v%(oldver).2f -> v%(newver).2f)"
else:
continue
self.log_info(_(msg) % {'type': plugin_type.rstrip('s').upper(), # @TODO: Remove rstrip in 0.4.10
'name': plugin_name,
'oldver': oldver,
'newver': newver})
try:
content = self.load(url % plugin + ".py", decode=False, req=req)
if req.code == 404:
raise Exception(_("URL not found"))
m = self._VERSION.search(content)
if m and m.group(2) == plugin_version:
with open(fsjoin("userplugins", plugin_type, plugin_name + ".py"), "wb") as f:
f.write(encode(content))
updated.append((plugin_type, plugin_name))
else:
raise Exception(_("Version mismatch"))
except Exception as e:
self.log_error(_("Error updating plugin: %s %s") %
(plugin_type.rstrip('s').upper(), plugin_name),
e) # @TODO: Remove rstrip in 0.4.10
return updated
