def createIndex(self): plugins = [] active = [] deactive = [] plugin_manager = get_plugin_manager() for pluginname in plugin_manager.hookPlugins: try: # hookClass = getattr(plugin, plugin.__name__) if self.core.config.getPlugin(pluginname, "activated"): pluginClass = plugin_manager.loadClass("hooks", pluginname) if not pluginClass: continue plugin = pluginClass(self.core, self) plugins.append(plugin) self.pluginMap[pluginClass.__name__] = plugin if plugin.isActivated(): active.append(pluginClass.__name__) else: deactive.append(pluginname) except: self.log.warning( _("Failed activating %(name)s") % {"name": pluginname}) if self.core.debug: traceback.print_exc() self.log.info(_("Activated plugins: %s") % ", ".join(sorted(active))) self.log.info( _("Deactivate plugins: %s") % ", ".join(sorted(deactive))) self.plugins = plugins
def activate(self): plugin_manager = get_plugin_manager() for p in ("UnRar", "SevenZip", "UnZip", "UnTar"): try: module = plugin_manager.loadModule("internal", p) klass = getattr(module, p) if klass.find(): self.extractors.append(klass) if klass.REPAIR: self.repair = self.config.get('repair') except OSError as e: if e.errno == 2: self.log_warning(_("No %s installed") % p) else: self.log_warning(_("Could not activate: %s") % p, e) except Exception as e: self.log_warning(_("Could not activate: %s") % p, e) if self.extractors: self.log_debug(*["Found %s %s" % (Extractor.__name__, Extractor.VERSION) for Extractor in self.extractors]) self.extract_queued() #: Resume unfinished extractions else: self.log_info(_("No Extract plugins activated"))
def decryptContainer(self, plugin, url): data = [] # only works on container plugins self.m.log.debug("Pre decrypting %s with %s" % (url, plugin)) # dummy pyfile pyfile = PyFile(self.m.core.files, -1, url, url, 0, 0, "", plugin, -1, -1) pyfile.initPlugin() # little plugin lifecycle try: pyfile.plugin.setup() pyfile.plugin.loadToDisk() pyfile.plugin.decrypt(pyfile) pyfile.plugin.deleteTmp() for pack in pyfile.plugin.packages: pyfile.plugin.urls.extend(pack[1]) data = get_plugin_manager().parseUrls(pyfile.plugin.urls) self.m.log.debug("Got %d links." % len(data)) except Exception as e: self.m.log.debug("Pre decrypting error: %s" % str(e)) finally: pyfile.release() return data
def deactivate(self): """ Remove override for all plugins. """ self.log_info(_("Reverting back to default hosters")) try: get_hook_manager().removeEvent("plugin_updated", self.plugins_updated) except ValueError: pass self.periodical.stop() self.fail_count = 0 if self.supported: self.log_debug("Unload: %s" % ", ".join(self.supported)) for plugin in self.supported: self.unload_plugin(plugin) #: Reset pattern hdict = get_plugin_manager().plugins[self.plugintype][self.classname] hdict['pattern'] = getattr(self.pluginclass, "__pattern__", r'^unmatchable$') hdict['re'] = re.compile(hdict['pattern'])
def autoreload_plugins(self): """ Reload and reindex all modified plugins """ reloads = [] modules = list(filter( lambda m: m and (m.__name__.startswith("module.plugins.") or m.__name__.startswith("userplugins.")) and m.__name__.count(".") >= 2, sys.modules.values() )) plugin_manager = get_plugin_manager() for m in modules: root, plugin_type, plugin_name = m.__name__.rsplit(".", 2) plugin_id = (plugin_type, plugin_name) if plugin_type in plugin_manager.plugins: f = m.__file__.replace(".pyc", ".py") if not os.path.isfile(f): continue mtime = os.path.getmtime(f) if plugin_id not in self.mtimes: self.mtimes[plugin_id] = mtime elif self.mtimes[plugin_id] < mtime: reloads.append(plugin_id) self.mtimes[plugin_id] = mtime return bool(plugin_manager.reloadPlugins(reloads))
def update_plugins(self): server_data = self.server_response() if not server_data or server_data[0] != self.pyload.api.getServerVersion(): return 0 updated = self._update_plugins(server_data) if updated: self.log_info(_("*** Plugins updated ***")) if get_plugin_manager().reloadPlugins(updated): exitcode = 1 else: self.log_warning(_("You have to restart pyLoad to use the updated plugins")) self.info['plugins'] = True exitcode = 2 paused = get_thread_manager().pause self.pyload.api.pauseServer() self.manager.dispatchEvent("plugin_updated", updated) if not paused: self.pyload.api.unpauseServer() else: self.log_info(_("All plugins are up to date!")) exitcode = 0 #: Exit codes: #: 0 = No plugin updated #: 1 = Plugins updated #: 2 = Plugins updated, but restart required return exitcode
def is_hoster_link(self, link): #declare all links as hoster links so the filter will work on all links if self.config.get('filter_all'): return True for item in get_plugin_manager().hosterPlugins.items(): if item[1]['re'].match(link): return True return False
def unload_plugin(self, plugin): #: Reset module hdict = get_plugin_manager().plugins[self.plugintype][plugin] if "module" in hdict: hdict.pop('module', None) if "new_module" in hdict: hdict.pop('new_module', None) hdict.pop('new_name', None)
def initPlugin(self): """ inits plugin instance """ if not self.plugin: plugin_manager = get_plugin_manager() self.pluginmodule = plugin_manager.getPlugin(self.pluginname) self.pluginclass = getattr( self.pluginmodule, plugin_manager.getPluginName(self.pluginname)) self.plugin = self.pluginclass(self)
def getAccountPlugin(self, plugin): """get account instance for plugin or None if anonymous""" if plugin in self.accounts: if plugin not in self.plugins: self.plugins[plugin] = get_plugin_manager().loadClass( "accounts", plugin)(self, self.accounts[plugin]) return self.plugins[plugin] else: return None
def checkAndAddPackages(self, links, dest=Destination.Queue): """Checks online status, retrieves names, and will add packages.\ Because of this packages are not added immediatly, only for internal use. :param links: list of urls :param dest: `Destination` :return: None """ data = get_plugin_manager().parseUrls(links) get_thread_manager().createResultThread(data, True)
def init(self): self.__pattern__ = get_plugin_manager().crypterPlugins[self.classname][ 'pattern'] # @TODO: Recheck in 0.4.10 self.PLUGIN_DOMAIN = re.match(self.__pattern__, self.pyfile.url).group("DOMAIN").lower() self.PLUGIN_NAME = "".join( part.capitalize() for part in re.split(r'\.|\d+|-', self.PLUGIN_DOMAIN) if part != '.')
def get_pattern(self, type, plugin): if self.config.get('use_%s_list' % type): plugin_list = self.config.get('%s_list' % type) plugin_list = plugin_list.replace(' ', '').replace('\\', '') plugin_list = plugin_list.replace('|', ',').replace(';', ',') plugin_list = plugin_list.lower().split(',') plugin_set = set(plugin_list) if self.config.get('use_builtin_list'): builtin_list = getattr(self, "BUILTIN_%sS" % type.upper()) plugin_set.update(builtin_list) plugin_set.difference_update(('', )) if not plugin_set: self.log_info(_("No %s to handle") % type) return match_list = '|'.join(sorted(plugin_set)).replace('.', '\.') pattern = self._regexmap[type][1] % match_list self.log_info( _("Handle %d %s%s: %s") % (len(plugin_set), type, "" if len(plugin_set) == 1 else "s", match_list.replace('\.', '.').replace('|', ', '))) else: plugin_list = [] isXFS = lambda klass: any( k.__name__.startswith("XFS") for k in inspect.getmro(klass)) plugin_manager = get_plugin_manager() for p in plugin_manager.plugins[type].values(): try: klass = plugin_manager.loadClass(type, p['name']) except AttributeError as e: self.log_debug(e, trace=True) continue if hasattr(klass, "PLUGIN_DOMAIN" ) and klass.PLUGIN_DOMAIN and isXFS(klass): plugin_list.append(klass.PLUGIN_DOMAIN) if plugin_list: unmatch_list = '|'.join(sorted(plugin_list)).replace('.', '\.') pattern = self._regexmap[type][0] % unmatch_list else: pattern = self._regexmap[type][0] self.log_info(_("Auto-discover new %ss") % type) return pattern
def _load(self, type, plugin): dict = get_plugin_manager().plugins[type][plugin] pattern = self.get_pattern(type, plugin) if not pattern: return dict['pattern'] = pattern dict['re'] = re.compile(pattern) self.log_debug("Pattern for %ss: %s" % (type, pattern))
def addLinks(self, urls, package): """adds links""" get_hook_manager().dispatchEvent("linksAdded", urls, package) data = get_plugin_manager().parseUrls(urls) self.db.addLinks(data, package) get_thread_manager().createInfoThread(data, package) #@TODO change from reloadAll event to package update event get_pull_manager().addEvent(ReloadAllEvent("collector"))
def getDecryptJob(self): """return job for decrypting""" if "decrypt" in self.jobCache: return None plugin_manager = get_plugin_manager() plugins = plugin_manager.crypterPlugins.keys() + plugin_manager.containerPlugins.keys() plugins = str(tuple(plugins)) jobs = self.db.getPluginJob(plugins) if jobs: return self.getFile(jobs[0]) else: self.jobCache["decrypt"] = "empty" return None
def checkURLs(self, urls): """ Gets urls and returns pluginname mapped to list of matches urls. :param urls: :return: {plugin: urls} """ data = get_plugin_manager().parseUrls(urls) plugins = {} for url, plugin in data: if plugin in plugins: plugins[plugin].append(url) else: plugins[plugin] = [url] return plugins
def links_added(self, links, pid): plugin_manager = get_plugin_manager() hosterdict = plugin_manager.hosterPlugins linkdict = self.pyload.api.checkURLs(links) premiumplugins = { account.type for account in self.pyload.api.getAccounts(False) if account.valid and account.premium } multihosters = { hoster for hoster in plugin_manager.hosterPlugins if 'new_name' in hosterdict[hoster] and hosterdict[hoster]['new_name'] in premiumplugins } excluded = list( map( lambda domain: "".join(part.capitalize() for part in re.split( r'(\.|\d+)', domain) if part != '.'), self.config.get('excluded').replace(' ', '').replace( ',', '|').replace(';', '|').split('|'), )) included = list( map( lambda domain: "".join(part.capitalize() for part in re.split( r'(\.|\d+)', domain) if part != '.'), self.config.get('included').replace(' ', '').replace( ',', '|').replace(';', '|').split('|'), )) hosterlist = (premiumplugins | multihosters).union(excluded).difference(included) #: Found at least one hoster with account or multihoster if not any(True for pluginname in linkdict if pluginname in hosterlist): return for pluginname in set(linkdict.keys()) - hosterlist: self.log_info(_("Remove links of plugin: %s") % pluginname) for link in linkdict[pluginname]: self.log_debug("Remove link: %s" % link) links.remove(link)
def activateHook(self, plugin): #check if already loaded for inst in self.plugins: if inst.__name__ == plugin: return pluginClass = get_plugin_manager().loadClass("hooks", plugin) if not pluginClass: return self.log.debug("Plugin loaded: %s" % plugin) plugin = pluginClass(self.core, self) self.plugins.append(plugin) self.pluginMap[pluginClass.__name__] = plugin # call core Ready thread.start_new_thread(plugin.coreReady, tuple())
def init_plugin(self): plugin_manager = get_plugin_manager() plugin, self.plugintype = plugin_manager.findPlugin(self.classname) if plugin: self.pluginmodule = plugin_manager.loadModule( self.plugintype, self.classname) self.pluginclass = plugin_manager.loadClass( self.plugintype, self.classname) get_hook_manager().addEvent("plugin_updated", self.plugins_updated) interval = self.config.get('mh_interval', 12) * 60 * 60 self.periodical.start(interval, threaded=True, delay=2) else: self.log_warning( _("Multi-hoster feature will be deactivated due missing plugin reference" ))
def whitelist(self, links, filters): plugindict = dict(get_plugin_manager().parseUrls(links)) linkcount = len(links) links[:] = [ link for link in links if ( any(link.find(_filter) != -1 for _filter in filters) or not self.is_hoster_link(link) and plugindict[link] != "BasePlugin" ) ] linkcount -= len(links) if linkcount > 0: linkstring = '' if self.config.get('filter_all') else 'hoster ' linkstring += 'link' if linkcount == 1 else 'links' self.log_warning( _('Whitelist filter removed %s %s not containing (%s)') % (linkcount, linkstring, ', '.join(filters)))
def checkOnlineStatus(self, urls): """ initiates online status check :param urls: :return: initial set of data as `OnlineCheck` instance containing the result id """ data = get_plugin_manager().parseUrls(urls) rid = get_thread_manager().createResultThread(data, False) tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] data = parseNames(tmp) result = {} for k, v in six.iteritems(data): for url, status in v: status.packagename = k result[url] = status return OnlineCheck(rid, result)
def decrypt_image(self, img, input_type='jpg', output_type='textual', ocr=False, timeout=120): """ Loads a captcha and decrypts it with ocr, plugin, user input :param img: image raw data :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled :param input_type: Type of the Image :param output_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha :param ocr: if True, builtin ocr is used. if string, the OCR plugin name is used :return: result of decrypting """ result = None time_ref = ("%.2f" % time.time())[-6:].replace(".", "") with open( os.path.join( "tmp", "captcha_image_%s_%s.%s" % (self.pyfile.plugin.__name__, time_ref, input_type)), "wb") as img_f: img_f.write(img) if ocr: self.log_info(_("Using OCR to decrypt captcha...")) if isinstance(ocr, six.string_types): _OCR = get_plugin_manager().loadClass( "captcha", ocr) #: Rename `captcha` to `ocr` in 0.4.10 result = _OCR(self.pyfile).recognize(img_f.name) else: result = self.recognize(img_f.name) if not result: self.log_warning(_("No OCR result")) if not result: captcha_manager = get_captcha_manager() try: self.task = captcha_manager.newTask(img, input_type, img_f.name, output_type) captcha_manager.handleCaptcha(self.task) # @TODO: Move to `CaptchaManager` in 0.4.10 self.task.setWaiting(max(timeout, 50)) while self.task.isWaiting(): self.pyfile.plugin.check_status() time.sleep(1) finally: captcha_manager.removeTask(self.task) result = self.task.result if self.task.error: if not self.task.handler and not self.pyload.isClientConnected( ): self.log_warning( _("No Client connected for captcha decrypting")) self.fail(_("No Client connected for captcha decrypting")) else: self.pyfile.plugin.retry_captcha(msg=self.task.error) elif self.task.result: self.log_info(_("Captcha result: `%s`") % (result, )) else: self.pyfile.plugin.retry_captcha( msg=_("No captcha result obtained in appropriate timing")) if not self.pyload.debug: self.remove(img_f.name, trash=False) return result
def init(self): self.PLUGIN_NAME = get_plugin_manager().crypterPlugins.get( self.classname)['name']
def _override(self): prev_supported = self.supported new_supported = [] excluded = [] self.supported = [] plugin_manager = get_plugin_manager() if self.plugintype == "hoster": plugin_map = { name.lower(): name for name in plugin_manager.hosterPlugins.keys() } account_list = [ account.type.lower() for account in self.pyload.api.getAccounts(False) if account.valid and account.premium ] else: plugin_map = {} account_list = [ name[::-1].replace("Folder"[::-1], "", 1).lower()[::-1] for name in plugin_manager.crypterPlugins.keys() ] for plugin in self.get_plugins(): name = remove_chars(plugin, "-.") if name in account_list: excluded.append(plugin) else: if name in plugin_map: self.supported.append(plugin_map[name]) else: new_supported.append(plugin) removed = [ plugin for plugin in prev_supported if plugin not in self.supported ] if removed: self.log_debug("Unload: %s" % ", ".join(removed)) for plugin in removed: self.unload_plugin(plugin) if not self.supported and not new_supported: self.log_error(_("No %s loaded") % self.plugintype) return #: Inject plugin plugin self.log_debug("Overwritten %ss: %s" % (self.plugintype, ", ".join(sorted(self.supported)))) for plugin in self.supported: hdict = plugin_manager.plugins[self.plugintype][plugin] hdict['new_module'] = self.pluginmodule hdict['new_name'] = self.classname if excluded: self.log_info( _("%ss not overwritten: %s") % (self.plugintype.capitalize(), ", ".join(sorted(excluded)))) if new_supported: plugins = sorted(new_supported) self.log_debug("New %ss: %s" % (self.plugintype, ", ".join(plugins))) #: Create new regexp pattern = r'.*(?P<DOMAIN>%s).*' % "|".join( x.replace('.', '\.') for x in plugins) if (hasattr(self.pluginclass, "__pattern__") and isinstance( self.pluginclass.__pattern__, six.string_types) and "://" in self.pluginclass.__pattern__): pattern = r'%s|%s' % (self.pluginclass.__pattern__, pattern) self.log_debug("Pattern: %s" % pattern) hdict = plugin_manager.plugins[self.plugintype][self.classname] hdict['pattern'] = pattern hdict['re'] = re.compile(pattern)
def initAccountPlugins(self): """init names""" for name in get_plugin_manager().getAccountPlugins(): self.accounts[name] = {}
def _unload(self, type, plugin): dict = get_plugin_manager().plugins[type][plugin] dict['pattern'] = r'^unmatchable$' dict['re'] = re.compile(dict['pattern'])
def setup_base(self): klass = get_plugin_manager().loadClass("hoster", self.classname) self.get_info = klass.get_info SimpleHoster.setup_base(self)
def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input :param url: url of captcha image :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled :param forceUser: if True, ocr is not used :param imgtype: Type of the Image :param result_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha :return: result of decrypting """ img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time())[-6:].replace(".", "") temp_file = open( join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") temp_file.write(img) temp_file.close() plugin_manager = get_plugin_manager() has_plugin = self.__name__ in plugin_manager.captchaPlugins if self.core.captcha: Ocr = plugin_manager.loadClass("captcha", self.__name__) else: Ocr = None if Ocr and not forceUser: sleep(randint(3000, 5000) / 1000.0) if self.pyfile.abort: raise Abort ocr = Ocr() result = ocr.get_captcha(temp_file.name) else: captcha_manager = get_captcha_manager() task = captcha_manager.newTask(img, imgtype, temp_file.name, result_type) self.cTask = task captcha_manager.handleCaptcha(task) while task.isWaiting(): if self.pyfile.abort: captcha_manager.removeTask(task) raise Abort sleep(1) captcha_manager.removeTask(task) if task.error and has_plugin: #ignore default error message since the user could use OCR self.fail( _("Pil and tesseract not installed and no Client connected for captcha decrypting" )) elif task.error: self.fail(task.error) elif not task.result: self.fail( _("No captcha result obtained in appropiate time by any of the plugins." )) result = task.result self.log.debug("Received captcha result: %s" % str(result)) if not self.core.debug: try: remove(temp_file.name) except: pass return result
def _update_plugins(self, server_data): """ Check for plugin updates """ updated = [] updatelist, blacklist = self.parse_updates(server_data) url = server_data[1] req = get_request_factory().getRequest(self.classname) if blacklist: #@NOTE: Protect UpdateManager from self-removing if IS_WINDOWS: #@NOTE: Windows filesystem is case insensitive, make sure we do not delete legitimate plugins whitelisted_plugins = [ (plugin['type'], plugin['name'].upper()) for plugin in updatelist] blacklisted_plugins = [(plugin['type'], plugin['name']) for plugin in blacklist if not (plugin['name'] == self.classname and plugin['type'] == self.__type__) and (plugin['type'], plugin['name'].upper()) not in whitelisted_plugins] else: blacklisted_plugins = [(plugin['type'], plugin['name']) for plugin in blacklist if not (plugin['name'] == self.classname and plugin['type'] == self.__type__)] c = 1 l = len(blacklisted_plugins) for idx, plugin in enumerate(updatelist): if c > l: break plugin_name = plugin['name'] plugin_type = plugin['type'] for t, n in blacklisted_plugins: if n != plugin_name or t != plugin_type: continue updatelist.pop(idx) c += 1 break for t, n in self.remove_plugins(blacklisted_plugins): self.log_info(_("Removed blacklisted plugin: %(type)s %(name)s") % {'type': t.upper(), 'name': n,}) plugin_manager = get_plugin_manager() for plugin in updatelist: plugin_name = plugin['name'] plugin_type = plugin['type'] plugin_version = plugin['version'] # @TODO: Remove rstrip in 0.4.10 plugins = getattr(plugin_manager, "%sPlugins" % plugin_type.rstrip('s')) oldver = float(plugins[plugin_name]['v']) if plugin_name in plugins else None try: newver = float(plugin_version) except ValueError: self.log_error(_("Error updating plugin: %s %s") % (plugin_type.rstrip('s').upper(), plugin_name), _("Bad version number on the server")) continue if not oldver: msg = "New plugin: %(type)s %(name)s (v%(newver).2f)" elif newver > oldver: msg = "New version of plugin: %(type)s %(name)s (v%(oldver).2f -> v%(newver).2f)" else: continue self.log_info(_(msg) % {'type': plugin_type.rstrip('s').upper(), # @TODO: Remove rstrip in 0.4.10 'name': plugin_name, 'oldver': oldver, 'newver': newver}) try: content = self.load(url % plugin + ".py", decode=False, req=req) if req.code == 404: raise Exception(_("URL not found")) m = self._VERSION.search(content) if m and m.group(2) == plugin_version: with open(fsjoin("userplugins", plugin_type, plugin_name + ".py"), "wb") as f: f.write(encode(content)) updated.append((plugin_type, plugin_name)) else: raise Exception(_("Version mismatch")) except Exception as e: self.log_error(_("Error updating plugin: %s %s") % (plugin_type.rstrip('s').upper(), plugin_name), e) # @TODO: Remove rstrip in 0.4.10 return updated