def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it :param url: url as string :param get: GET as dict :param post: POST as dict, list or string :param ref: Set HTTP_REFERER header :param cookies: use saved cookies :param just_header: if True only the header will be retrieved and returned as dict :param decode: Whether to decode the output according to http header, should be True in most cases :return: Loaded content """ if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") self.checkAbort() res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.core.debug: from inspect import currentframe frame = currentframe() if not exists(join("tmp", self.__name__)): makedirs(join("tmp", self.__name__)) f = open( join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)) , "wb") del frame # delete the frame or it wont be cleaned try: tmp = res.encode("utf8") except: tmp = res f.write(tmp) f.close() if just_header: #parse header header = {"code": self.req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") key = key.lower().strip() value = value.strip() if key in header: if type(header[key]) == list: header[key].append(value) else: header[key] = [header[key], value] else: header[key] = value res = header return res
def setUp(self): PluginTester.setUp(self) for f in self.files: if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f)) # folder for reports report = join("tmp", self.__class__.__name__) if exists(report): for f in listdir(report): remove(join(report, f))
def ask_lang(self): langs = self.config.getMetaData("general", "language").type.split(";") self.lang = self.ask(u"Choose your Language / Wähle deine Sprache", "en", langs) gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) translation = gettext.translation("setup", join(self.path, "locale"), languages=[self.lang, "en"], fallback=True) translation.install(True) #l10n Input shorthand for yes self.yes = _("y") #l10n Input shorthand for no self.no = _("n")
def set_user(self): gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) translation = gettext.translation("setup", join(self.path, "locale"), languages=[self.config["general"]["language"], "en"], fallback=True) translation.install(True) from module.database import DatabaseBackend db = DatabaseBackend(None) db.setup() noaction = True try: while True: print _("Select action") print _("1 - Create/Edit user") print _("2 - List users") print _("3 - Remove user") print _("4 - Quit") action = raw_input("[1]/2/3/4: ") if not action in ("1", "2", "3", "4"): continue elif action == "1": print "" username = self.ask(_("Username"), "User") password = self.ask("", "", password=True) db.addUser(username, password) noaction = False elif action == "2": print "" print _("Users") print "-----" users = db.getAllUserData() noaction = False for user in users.itervalues(): print user.name print "-----" print "" elif action == "3": print "" username = self.ask(_("Username"), "") if username: db.removeUser(username) noaction = False elif action == "4": db.syncSave() break finally: if not noaction: db.shutdown()
def setup(self): self.scripts = {} folders = ['download_preparing', 'download_finished', 'package_finished', 'before_reconnect', 'after_reconnect', 'unrar_finished', 'all_dls_finished', 'all_dls_processed'] for folder in folders: self.scripts[folder] = [] self.initPluginType(folder, join(pypath, 'scripts', folder)) self.initPluginType(folder, join('scripts', folder)) for script_type, names in self.scripts.iteritems(): if names: self.logInfo((_("Installed scripts for %s: ") % script_type ) + ", ".join([basename(x) for x in names]))
def initPluginType(self, folder, path): if not exists(path): try: makedirs(path) except: self.logDebug("Script folder %s not created" % folder) return for f in listdir(path): if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"): continue if not access(join(path, f), X_OK): self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f)) self.scripts[folder].append(join(path, f))
def test_plugin(self, name, url, flag): # Print to stdout to see whats going on print "%s: %s, %s" % (name, url.encode("utf8"), flag) log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag) # url and plugin should be only important thing pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0) pyfile.initPlugin() self.thread.pyfile = pyfile self.thread.plugin = pyfile.plugin try: a = time() pyfile.plugin.preprocessing(self.thread) log(DEBUG, "downloading took %ds" % (time() - a)) log(DEBUG, "size %d kb" % (pyfile.size / 1024)) if flag == "offline": raise Exception("No offline Exception raised.") if pyfile.name not in self.files: raise Exception("Filename %s not recognized." % pyfile.name) if not exists(save_join(DL_DIR, pyfile.name)): raise Exception("File %s does not exists." % pyfile.name) hash = md5() f = open(save_join(DL_DIR, pyfile.name), "rb") while True: buf = f.read(4096) if not buf: break hash.update(buf) f.close() if hash.hexdigest() != self.files[pyfile.name]: log(DEBUG, "Hash is %s" % hash.hexdigest()) size = stat(f.name).st_size if size < 1024 * 1024 * 10: # 10MB # Copy for debug report log(DEBUG, "Downloaded file copied to report") move(f.name, join("tmp", plugin, f.name)) raise Exception("Hash does not match.") except Exception, e: if isinstance(e, Fail) and flag == "fail": pass elif isinstance(e, Fail) and flag == "offline" and e.message == "offline": pass else: raise
def writeDebugReport(self, name, pyfile=None, plugin=None): """ writes a debug report to disk """ dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S")) if pyfile: dump = self.getFileDump(pyfile) else: dump = self.getPluginDump(plugin) try: import zipfile zip = zipfile.ZipFile(dump_name, "w") if exists(join("tmp", name)): for f in listdir(join("tmp", name)): try: # avoid encoding errors zip.write(join("tmp", name, f), save_join(name, f)) except: pass info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime()) info.external_attr = 0644 << 16L # change permissions zip.writestr(info, dump) info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime()) info.external_attr = 0644 << 16L zip.writestr(info, self.getSystemDump()) zip.close() if not stat(dump_name).st_size: raise Exception("Empty Zipfile") except Exception, e: self.log.debug("Error creating zip file: %s" % e) dump_name = dump_name.replace(".zip", ".txt") f = open(dump_name, "wb") f.write(dump) f.close()
def uploadContainer(self, filename, data): """Uploads and adds a container file to pyLoad. :param filename: filename, extension is important so it can correctly decrypted :param data: file content """ th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb") th.write(str(data)) th.close() return self.addPackage(th.name, [th.name])
def checkOnlineStatusContainer(self, urls, container, data): """ checks online status of urls and a submitted container file :param urls: list of urls :param container: container file name :param data: file content :return: :class:`OnlineCheck` """ th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb") th.write(str(data)) th.close() urls.append(th.name) return self.checkOnlineStatus(urls)
def conf_path(self, trans=False): if trans: gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) translation = gettext.translation("setup", join(self.path, "locale"), languages=[self.config["general"]["language"], "en"], fallback=True) translation.install(True) print _("Setting new configpath, current configuration will not be transferred!") path = self.ask(_("Configpath"), abspath("")) try: path = join(pypath, path) if not exists(path): makedirs(path) f = open(join(pypath, "module", "config", "configdir"), "wb") f.write(path) f.close() print _("Configpath changed, setup will now close, please restart to go on.") print _("Press Enter to exit.") raw_input() exit() except Exception, e: print _("Setting config path failed: %s") % str(e)
def getLog(self, offset=0): """Returns most recent log entries. :param offset: line offset :return: List of log entries """ filename = join(self.core.config['log']['log_folder'], 'log.txt') try: fh = open(filename, "r") lines = fh.readlines() fh.close() if offset >= len(lines): return [] return lines[offset:] except: return ['No log available']
class Hoster(Base): """ Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading. """ @staticmethod def getInfo(urls): """This method is used to retrieve the online status of files for hoster plugins. It has to *yield* list of tuples with the result in this format (name, size, status, url), where status is one of API pyfile statuses. :param urls: List of urls :return: yield list of tuple with results (name, size, status, url) """ pass def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) self.wantReconnect = False #: enables simultaneous processing of multiple downloads self.limitDL = 0 #: chunk limit self.chunkLimit = 1 #: enables resume (will be ignored if server dont accept chunks) self.resumeDownload = False #: plugin is waiting self.waiting = False self.ocr = None #captcha reader instance #: account handler instance, see :py:class:`Account` self.account = self.core.accountManager.getAccountForPlugin( self.__name__) #: premium status self.premium = False #: username/login self.user = None if self.account and not self.account.isUsable(): self.account = None if self.account: self.user = self.account.loginname #: Browser instance, see `network.Browser` self.req = self.account.getAccountRequest() # Default: -1, True, True self.chunkLimit, self.limitDL, self.resumeDownload = self.account.getDownloadSettings( ) self.premium = self.account.isPremium() else: self.req = self.core.requestFactory.getRequest(self.__name__) #: associated pyfile instance, see `PyFile` self.pyfile = pyfile self.thread = None # holds thread in future #: location where the last call to download was saved self.lastDownload = "" #: re match of the last call to `checkDownload` self.lastCheck = None #: js engine, see `JsEngine` self.js = self.core.js self.retries = 0 # amount of retries already made self.html = None # some plugins store html code here self.init() def getMultiDL(self): return self.limitDL <= 0 def setMultiDL(self, val): self.limitDL = 0 if val else 1 #: virtual attribute using self.limitDL on behind multiDL = property(getMultiDL, setMultiDL) def getChunkCount(self): if self.chunkLimit <= 0: return self.config["download"]["chunks"] return min(self.config["download"]["chunks"], self.chunkLimit) def getDownloadLimit(self): if self.account: limit = self.account.options.get("limitDL", 0) if limit == "": limit = 0 if self.limitDL > 0: # a limit is already set, we use the minimum return min(int(limit), self.limitDL) else: return int(limit) else: return self.limitDL def __call__(self): return self.__name__ def init(self): """initialize the plugin (in addition to `__init__`)""" pass def setup(self): """ setup for environment and other things, called before downloading (possibly more than one time)""" pass def preprocessing(self, thread): """ handles important things to do before starting """ self.thread = thread if self.account: # will force a re-login or reload of account info if necessary self.account.getAccountInfo() else: self.req.clearCookies() self.setup() self.pyfile.setStatus("starting") return self.process(self.pyfile) def process(self, pyfile): """the 'main' method of every plugin, you **have to** overwrite it""" raise NotImplementedError def abort(self): return self.pyfile.abort def resetAccount(self): """ don't use account and retry download """ self.account = None self.req = self.core.requestFactory.getRequest(self.__name__) self.retry() def checksum(self, local_file=None): """ return codes: 0 - checksum ok 1 - checksum wrong 5 - can't get checksum 10 - not implemented 20 - unknown error """ #@TODO checksum check addon return True, 10 def setWait(self, seconds, reconnect=False): """Set a specific wait time later used with `wait` :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ if reconnect: self.wantReconnect = True self.pyfile.waitUntil = time() + int(seconds) def wait(self): """ waits the time previously set """ self.waiting = True self.pyfile.setStatus("waiting") while self.pyfile.waitUntil > time(): self.thread.m.reconnecting.wait(2) self.checkAbort() if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False raise Reconnect self.waiting = False self.pyfile.setStatus("starting") def offline(self): """ fail and indicate file is offline """ raise Fail("offline") def tempOffline(self): """ fail and indicates file ist temporary offline, the core may take consequences """ raise Fail("temp. offline") def retry(self, max_tries=3, wait_time=1, reason=""): """Retries and begin again from the beginning :param max_tries: number of maximum retries :param wait_time: time to wait in seconds :param reason: reason for retrying, will be passed to fail if max_tries reached """ if 0 < max_tries <= self.retries: if not reason: reason = "Max retries reached" raise Fail(reason) self.wantReconnect = False self.setWait(wait_time) self.wait() self.retries += 1 raise Retry(reason) def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder :param url: :param get: :param post: :param ref: :param cookies: :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ self.checkForSameFiles() self.checkAbort() self.pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception, e: self.log.warning( _("Setting User and Group failed: %s") % str(e)) # convert back to unicode location = fs_decode(location) name = save_filename(self.pyfile.name) filename = join(location, name) self.core.addonManager.dispatchEvent("download:start", self.pyfile, url, filename) try: newname = self.req.httpDownload(url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.getChunkCount(), resume=self.resumeDownload, disposition=disposition) finally: self.pyfile.size = self.req.size if disposition and newname and newname != name: #triple check, just to be sure self.log.info("%(name)s saved as %(newname)s" % { "name": name, "newname": newname }) self.pyfile.name = newname filename = join(location, newname) fs_filename = fs_encode(filename) if self.core.config["permission"]["change_file"]: chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(fs_filename, uid, gid) except Exception, e: self.log.warning( _("Setting User and Group failed: %s") % str(e))
#l10n yes, true,t are inputs for booleans with value true if input.lower().strip() in [ self.yes, _("yes"), _("true"), _("t"), "yes" ]: return True #l10n no, false,f are inputs for booleans with value false elif input.lower().strip() in [ self.no, _("no"), _("false"), _("f"), "no" ]: return False else: print _("Invalid Input") continue if not answers: return input else: if input in answers: return input else: print _("Invalid Input") if __name__ == "__main__": test = Setup(join(abspath(dirname(__file__)), ".."), None) test.start()
def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input :param url: url of captcha image :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled :param forceUser: if True, ocr is not used :param imgtype: Type of the Image :param result_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha :return: result of decrypting """ img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time())[-6:].replace(".", "") temp_file = open( join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") temp_file.write(img) temp_file.close() name = "%sOCR" % self.__name__ has_plugin = name in self.core.pluginManager.getPlugins("internal") if self.core.captcha: OCR = self.core.pluginManager.loadClass("internal", name) else: OCR = None if OCR and not forceUser: sleep(randint(3000, 5000) / 1000.0) self.checkAbort() ocr = OCR() result = ocr.get_captcha(temp_file.name) else: task = self.im.createCaptchaTask(img, imgtype, temp_file.name, self.__name__, result_type) self.task = task while task.isWaiting(): if self.abort(): self.im.removeTask(task) raise Abort() sleep(1) #TODO task handling self.im.removeTask(task) if task.error and has_plugin: #ignore default error message since the user could use OCR self.fail( _("Pil and tesseract not installed and no Client connected for captcha decrypting" )) elif task.error: self.fail(task.error) elif not task.result: self.fail(_("No captcha result obtained in appropriate time.")) result = task.result self.log.debug("Received captcha result: %s" % str(result)) if not self.core.debug: try: remove(temp_file.name) except: pass return result
def getPath(self, name=""): return join(self.m.core.config["general"]["download_folder"], name)
def getPath(self, name=""): self.timestamp = time() return join(self.m.getPackage(self.root).getPath(), self.folder, name)
input = raw_input(qst + " %s: " % info) input = input.decode(self.stdin_encoding) if input.strip() == "": input = default if bool: #l10n yes, true,t are inputs for booleans with value true if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]: return True #l10n no, false,f are inputs for booleans with value false elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]: return False else: print _("Invalid Input") continue if not answers: return input else: if input in answers: return input else: print _("Invalid Input") if __name__ == "__main__": test = Setup(join(abspath(dirname(__file__)), ".."), None) test.start()
def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input :param url: url of captcha image :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled :param forceUser: if True, ocr is not used :param imgtype: Type of the Image :param result_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha :return: result of decrypting """ img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time())[-6:].replace(".", "") temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") temp_file.write(img) temp_file.close() name = "%sOCR" % self.__name__ has_plugin = name in self.core.pluginManager.getPlugins("internal") if self.core.captcha: OCR = self.core.pluginManager.loadClass("internal", name) else: OCR = None if OCR and not forceUser: sleep(randint(3000, 5000) / 1000.0) self.checkAbort() ocr = OCR() result = ocr.get_captcha(temp_file.name) else: task = self.im.createCaptchaTask(img, imgtype, temp_file.name, self.__name__, result_type) self.task = task while task.isWaiting(): if self.abort(): self.im.removeTask(task) raise Abort() sleep(1) #TODO task handling self.im.removeTask(task) if task.error and has_plugin: #ignore default error message since the user could use OCR self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting")) elif task.error: self.fail(task.error) elif not task.result: self.fail(_("No captcha result obtained in appropriate time.")) result = task.result self.log.debug("Received captcha result: %s" % str(result)) if not self.core.debug: try: remove(temp_file.name) except: pass return result
from hashlib import md5 from time import time from shutil import move import codecs from nose.tools import nottest from helper.Stubs import Core from helper.PluginTester import PluginTester from module.datatypes.PyFile import PyFile from module.plugins.Base import Fail from module.utils import accumulate from module.utils.fs import save_join, join, exists, listdir, remove, stat DL_DIR = join("Downloads", "tmp") class HosterPluginTester(PluginTester): files = {} def setUp(self): PluginTester.setUp(self) for f in self.files: if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f)) # folder for reports report = join("tmp", self.__class__.__name__) if exists(report): for f in listdir(report): remove(join(report, f))
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it :param url: url as string :param get: GET as dict :param post: POST as dict, list or string :param ref: Set HTTP_REFERER header :param cookies: use saved cookies :param just_header: if True only the header will be retrieved and returned as dict :param decode: Whether to decode the output according to http header, should be True in most cases :return: Loaded content """ if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.") self.checkAbort() res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.core.debug: from inspect import currentframe frame = currentframe() if not exists(join("tmp", self.__name__)): makedirs(join("tmp", self.__name__)) f = open( join( "tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)), "wb") del frame # delete the frame or it wont be cleaned try: tmp = res.encode("utf8") except: tmp = res f.write(tmp) f.close() if just_header: #parse header header = {"code": self.req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") key = key.lower().strip() value = value.strip() if key in header: if type(header[key]) == list: header[key].append(value) else: header[key] = [header[key], value] else: header[key] = value res = header return res