def run(self): self.running = True import webinterface global webinterface if self.https: if not exists(self.cert) or not exists(self.key): log.warning(_("SSL certificates not found.")) self.https = False prefer = None # These cases covers all settings if self.server == "threaded": prefer = "threaded" elif self.server == "fastcgi": prefer = "flup" elif self.server == "fallback": prefer = "wsgiref" server = self.select_server(prefer) try: self.start_server(server) except Exception, e: log.error(_("Failed starting webserver: " + e.message)) self.error = e if core: core.print_exc()
def run(self): self.running = True import webinterface global webinterface if self.https: if not exists(self.cert) or not exists(self.key): log.warning(_("SSL certificates not found.")) self.https = False if self.server == "fastcgi": try: import flup except: log.warning(_("Can't use %(server)s, python-flup is not installed!") % { "server": self.server}) self.server = "builtin" elif self.server == "lightweight": try: import bjoern except Exception, e: log.error(_("Error importing lightweight server: %s") % e) log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) log.warning(_("Copy the boern.so to module/lib folder or use setup.py install")) log.warning(_("Of course you need to be familiar with linux and know how to compile software")) self.server = "builtin"
def run(self): self.running = True import webinterface global webinterface if self.https: if not exists(self.cert) or not exists(self.key): log.warning(_("SSL certificates not found.")) self.https = False if self.server == "fastcgi": try: import flup except: log.warning(_("Can't use %(server)s, python-flup is not installed!") % {"server": self.server}) self.server = "builtin" elif self.server == "lightweight": try: import bjoern except Exception, e: log.error(_("Error importing lightweight server: %s") % e) log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern")) log.warning(_("Copy the boern.so to module/lib folder or use setup.py install")) log.warning(_("Of course you need to be familiar with linux and know how to compile software")) self.server = "builtin"
def checkForSameFiles(self, starting=False): """ checks if same file was/is downloaded within same package :param starting: indicates that the current download is going to start :raises SkipDownload: """ pack = self.pyfile.package() for pyfile in self.core.files.cachedFiles(): if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder: if pyfile.status in (0, 12): #finished or downloading raise SkipDownload(pyfile.pluginname) elif pyfile.status in ( 5, 7) and starting: #a download is waiting/starting and was apparently started before raise SkipDownload(pyfile.pluginname) download_folder = self.config['general']['download_folder'] location = save_join(download_folder, pack.folder, self.pyfile.name) if starting and self.core.config['download']['skip_existing'] and exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: raise SkipDownload("File exists.") pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: if exists(location): raise SkipDownload(pyfile[0]) self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
def setUp(self): PluginTester.setUp(self) for f in self.files: if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f)) # folder for reports report = join("tmp", self.__class__.__name__) if exists(report): for f in listdir(report): remove(join(report, f))
def conf_path(self, trans=False): if trans: gettext.setpaths( [join(os.sep, "usr", "share", "pyload", "locale"), None]) translation = gettext.translation( "setup", join(self.path, "locale"), languages=[self.config["general"]["language"], "en"], fallback=True) translation.install(True) print _( "Setting new configpath, current configuration will not be transferred!" ) path = self.ask(_("Configpath"), abspath("")) try: path = join(pypath, path) if not exists(path): makedirs(path) f = open(join(pypath, "module", "config", "configdir"), "wb") f.write(path) f.close() print _( "Configpath changed, setup will now close, please restart to go on." ) print _("Press Enter to exit.") raw_input() exit() except Exception, e: print _("Setting config path failed: %s") % str(e)
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder :param url: :param get: :param post: :param ref: :param cookies: :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ self.checkForSameFiles() self.checkAbort() self.pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception, e: self.log.warning(_("Setting User and Group failed: %s") % str(e))
def startExtracting(self, plugin, fid, passwords, thread): pyfile = self.core.files.getFile(fid) if not pyfile: return [] pyfile.setCustomStatus(_("extracting")) thread.addActive(pyfile) # keep this file until everything is done try: progress = lambda x: pyfile.setProgress(x) success = False if not plugin.checkArchive(): plugin.extract(progress) success = True else: self.logInfo(basename(plugin.file), _("Password protected")) self.logDebug("Passwords: %s" % str(passwords)) pwlist = copy(self.getPasswords()) #remove already supplied pws from list (only local) for pw in passwords: if pw in pwlist: pwlist.remove(pw) for pw in passwords + pwlist: try: self.logDebug("Try password: %s" % pw) if plugin.checkPassword(pw): plugin.extract(progress, pw) self.addPassword(pw) success = True break except WrongPassword: self.logDebug("Password was wrong") if not success: self.logError(basename(plugin.file), _("Wrong password")) return [] if self.core.debug: self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles())) if self.getConfig("deletearchive"): files = plugin.getDeleteFiles() self.logInfo(_("Deleting %s files") % len(files)) for f in files: if exists(f): remove(f) else: self.logDebug("%s does not exists" % f) self.logInfo(basename(plugin.file), _("Extracting finished")) self.manager.dispatchEvent("unrarFinished", plugin.out, plugin.file) return plugin.getExtractedFiles() except ArchiveError, e: self.logError(basename(plugin.file), _("Archive Error"), str(e))
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it :param url: url as string :param get: GET as dict :param post: POST as dict, list or string :param ref: Set HTTP_REFERER header :param cookies: use saved cookies :param just_header: if True only the header will be retrieved and returned as dict :param decode: Whether to decode the output according to http header, should be True in most cases :return: Loaded content """ if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") self.checkAbort() res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.core.debug: from inspect import currentframe frame = currentframe() if not exists(join("tmp", self.__name__)): makedirs(join("tmp", self.__name__)) f = open( join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) , "wb") del frame # delete the frame or it wont be cleaned try: tmp = res.encode("utf8") except: tmp = res f.write(tmp) f.close() if just_header: #parse header header = {"code": self.req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") key = key.lower().strip() value = value.strip() if key in header: if type(header[key]) == list: header[key].append(value) else: header[key] = [header[key], value] else: header[key] = value res = header return res
def test_plugin(self, name, url, flag): # Print to stdout to see whats going on print "%s: %s, %s" % (name, url.encode("utf8"), flag) log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag) # url and plugin should be only important thing pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0) pyfile.initPlugin() self.thread.pyfile = pyfile self.thread.plugin = pyfile.plugin try: a = time() pyfile.plugin.preprocessing(self.thread) log(DEBUG, "downloading took %ds" % (time() - a)) log(DEBUG, "size %d kb" % (pyfile.size / 1024)) if flag == "offline": raise Exception("No offline Exception raised.") if pyfile.name not in self.files: raise Exception("Filename %s not recognized." % pyfile.name) if not exists(save_join(DL_DIR, pyfile.name)): raise Exception("File %s does not exists." % pyfile.name) hash = md5() f = open(save_join(DL_DIR, pyfile.name), "rb") while True: buf = f.read(4096) if not buf: break hash.update(buf) f.close() if hash.hexdigest() != self.files[pyfile.name]: log(DEBUG, "Hash is %s" % hash.hexdigest()) size = stat(f.name).st_size if size < 1024 * 1024 * 10: # 10MB # Copy for debug report log(DEBUG, "Downloaded file copied to report") move(f.name, join("tmp", plugin, f.name)) raise Exception("Hash does not match.") except Exception, e: if isinstance(e, Fail) and flag == "fail": pass elif isinstance(e, Fail) and flag == "offline" and e.message == "offline": pass else: raise
def test_plugin(self, name, url, flag): # Print to stdout to see whats going on print "%s: %s, %s" % (name, url.encode("utf8"), flag) log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag) # url and plugin should be only important thing pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0) pyfile.initPlugin() self.thread.pyfile = pyfile self.thread.plugin = pyfile.plugin try: a = time() pyfile.plugin.preprocessing(self.thread) log(DEBUG, "downloading took %ds" % (time() - a)) log(DEBUG, "size %d kb" % (pyfile.size / 1024)) if flag == "offline": raise Exception("No offline Exception raised.") if pyfile.name not in self.files: raise Exception("Filename %s not recognized." % pyfile.name) if not exists(save_join(DL_DIR, pyfile.name)): raise Exception("File %s does not exists." % pyfile.name) hash = md5() f = open(save_join(DL_DIR, pyfile.name), "rb") while True: buf = f.read(4096) if not buf: break hash.update(buf) f.close() if hash.hexdigest() != self.files[pyfile.name]: log(DEBUG, "Hash is %s" % hash.hexdigest()) size = stat(f.name).st_size if size < 1024 * 1024 * 10: # 10MB # Copy for debug report log(DEBUG, "Downloaded file copied to report") move(f.name, join("tmp", plugin, f.name)) raise Exception("Hash does not match.") except Exception, e: if isinstance(e, Fail) and flag == "fail": pass elif isinstance( e, Fail) and flag == "offline" and e.message == "offline": pass else: raise
def reloadPasswords(self): pwfile = self.getConfig("passwordfile") if not exists(pwfile): open(pwfile, "wb").close() passwords = [] f = open(pwfile, "rb") for pw in f.read().splitlines(): passwords.append(pw) f.close() self.passwords = passwords
def _checkVersion(self): """ get db version""" if not exists(self.VERSION_FILE): f = open(self.VERSION_FILE, "wb") f.write(str(DB_VERSION)) f.close() return f = open(self.VERSION_FILE, "rb") v = int(f.read().strip()) f.close() return v
def checkForSameFiles(self, starting=False): """ checks if same file was/is downloaded within same package :param starting: indicates that the current download is going to start :raises SkipDownload: """ pack = self.pyfile.package() for pyfile in self.core.files.cachedFiles(): if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package( ).folder == pack.folder: if pyfile.status in (0, 12): #finished or downloading raise SkipDownload(pyfile.pluginname) elif pyfile.status in ( 5, 7 ) and starting: #a download is waiting/starting and was apparently started before raise SkipDownload(pyfile.pluginname) download_folder = self.config['general']['download_folder'] location = save_join(download_folder, pack.folder, self.pyfile.name) if starting and self.core.config['download'][ 'skip_existing'] and exists(location): size = os.stat(location).st_size if size >= self.pyfile.size: raise SkipDownload("File exists.") pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name) if pyfile: if exists(location): raise SkipDownload(pyfile[0]) self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
def getLocalContent(self, urls): """Load files from disk and separate to file content and url list :param urls: :return: list of (filename, content), remote urls """ content = [] # do nothing if no decryptFile method if hasattr(self.__class__, "decryptFile"): remote = [] for url in urls: path = None if url.startswith("http"): # skip urls directly pass elif url.startswith(self.CONTENT_PREFIX): path = url elif exists(url): path = url elif exists(self.core.path(url)): path = self.core.path(url) if path: try: if path.startswith(self.CONTENT_PREFIX): content.append( ("", path[len(self.CONTENT_PREFIX)])) else: f = open(fs_encode(path), "rb") content.append((f.name, f.read())) f.close() except IOError, e: self.logError("IOError", e) else: remote.append(url) #swap filtered url list urls = remote
def getLocalContent(self, urls): """Load files from disk and separate to file content and url list :param urls: :return: list of (filename, content), remote urls """ content = [] # do nothing if no decryptFile method if hasattr(self.__class__, "decryptFile"): remote = [] for url in urls: path = None if url.startswith("http"): # skip urls directly pass elif url.startswith(self.CONTENT_PREFIX): path = url elif exists(url): path = url elif exists(self.core.path(url)): path = self.core.path(url) if path: try: if path.startswith(self.CONTENT_PREFIX): content.append(("", path[len(self.CONTENT_PREFIX)])) else: f = open(fs_encode(path), "rb") content.append((f.name, f.read())) f.close() except IOError, e: self.logError("IOError", e) else: remote.append(url) #swap filtered url list urls = remote
def setPermissions(self, files): for f in files: if not exists(f): continue try: if self.core.config["permission"]["change_file"]: if isfile(f): chmod(f, int(self.core.config["permission"]["file"], 8)) elif isdir(f): chmod(f, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(f, uid, gid) except Exception, e: self.log.warning(_("Setting User and Group failed"), e)
def initPluginType(self, folder, path): if not exists(path): try: makedirs(path) except : self.logDebug("Script folder %s not created" % folder) return for f in listdir(path): if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"): continue if not access(join(path,f), X_OK): self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f)) self.scripts[folder].append(join(path, f))
def initPluginType(self, folder, path): if not exists(path): try: makedirs(path) except: self.logDebug("Script folder %s not created" % folder) return for f in listdir(path): if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"): continue if not access(join(path, f), X_OK): self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f)) self.scripts[folder].append(join(path, f))
def writeDebugReport(self, name, pyfile=None, plugin=None): """ writes a debug report to disk """ dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S")) if pyfile: dump = self.getFileDump(pyfile) else: dump = self.getPluginDump(plugin) try: import zipfile zip = zipfile.ZipFile(dump_name, "w") if exists(join("tmp", name)): for f in listdir(join("tmp", name)): try: # avoid encoding errors zip.write(join("tmp", name, f), save_join(name, f)) except: pass info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime()) info.external_attr = 0644 << 16L # change permissions zip.writestr(info, dump) info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime()) info.external_attr = 0644 << 16L zip.writestr(info, self.getSystemDump()) zip.close() if not stat(dump_name).st_size: raise Exception("Empty Zipfile") except Exception, e: self.log.debug("Error creating zip file: %s" % e) dump_name = dump_name.replace(".zip", ".txt") f = open(dump_name, "wb") f.write(dump) f.close()
def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0): """ checks the content of the last downloaded file, re match is saved to `lastCheck` :param rules: dict with names and rules to match (compiled regexp or strings) :param api_size: expected file size :param max_size: if the file is larger then it wont be checked :param delete: delete if matched :param read_size: amount of bytes to read from files larger then max_size :return: dictionary key of the first rule that matched """ lastDownload = fs_encode(self.lastDownload) if not exists(lastDownload): return None size = stat(lastDownload) size = size.st_size if api_size and api_size <= size: return None elif size > max_size and not read_size: return None self.log.debug("Download Check triggered") f = open(lastDownload, "rb") content = f.read(read_size if read_size else -1) f.close() #produces encoding errors, better log to other file in the future? #self.log.debug("Content: %s" % content) for name, rule in rules.iteritems(): if type(rule) in (str, unicode): if rule in content: if delete: remove(lastDownload) return name elif hasattr(rule, "search"): m = rule.search(content) if m: if delete: remove(lastDownload) self.lastCheck = m return name
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder :param url: :param get: :param post: :param ref: :param cookies: :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ self.checkForSameFiles() self.checkAbort() self.pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception, e: self.log.warning( _("Setting User and Group failed: %s") % str(e))
def conf_path(self, trans=False): if trans: gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) translation = gettext.translation("setup", join(self.path, "locale"), languages=[self.config["general"]["language"], "en"], fallback=True) translation.install(True) print _("Setting new configpath, current configuration will not be transferred!") path = self.ask(_("Configpath"), abspath("")) try: path = join(pypath, path) if not exists(path): makedirs(path) f = open(join(pypath, "module", "config", "configdir"), "wb") f.write(path) f.close() print _("Configpath changed, setup will now close, please restart to go on.") print _("Press Enter to exit.") raw_input() exit() except Exception, e: print _("Setting config path failed: %s") % str(e)
def extract(self, ids, thread=None): # reload from txt file self.reloadPasswords() # dl folder dl = self.config['general']['download_folder'] extracted = [] #iterate packages -> plugins -> targets for pid in ids: p = self.core.files.getPackage(pid) self.logInfo(_("Check package %s") % p.name) if not p: continue # determine output folder out = save_join(dl, p.folder, "") # force trailing slash if self.getConfig("destination") and self.getConfig("destination").lower() != "none": out = save_join(dl, p.folder, self.getConfig("destination"), "") #relative to package folder if destination is relative, otherwise absolute path overwrites them if self.getConfig("subfolder"): out = join(out, fs_encode(p.folder)) if not exists(out): makedirs(out) files_ids = [(save_join(dl, p.folder, x["name"]), x["id"]) for x in p.getChildren().itervalues()] matched = False # check as long there are unseen files while files_ids: new_files_ids = [] for plugin in self.plugins: targets = plugin.getTargets(files_ids) if targets: self.logDebug("Targets for %s: %s" % (plugin.__name__, targets)) matched = True for target, fid in targets: if target in extracted: self.logDebug(basename(target), "skipped") continue extracted.append(target) # prevent extracting same file twice klass = plugin(self, target, out, self.getConfig("fullpath"), self.getConfig("overwrite"), self.getConfig("renice")) klass.init() self.logInfo(basename(target), _("Extract to %s") % out) new_files = self.startExtracting(klass, fid, p.password.strip().splitlines(), thread) self.logDebug("Extracted: %s" % new_files) self.setPermissions(new_files) for file in new_files: if not exists(file): self.logDebug("new file %s does not exists" % file) continue if self.getConfig("recursive") and isfile(file): new_files_ids.append((file, fid)) # append as new target files_ids = new_files_ids # also check extracted files if not matched: self.logInfo(_("No files found to extract"))