def downloads(): root = PYLOAD.getConfigValue("general", "download_folder") if not isdir(root): return base([_('Download directory not found.')]) data = { 'folder': [], 'files': [] } items = listdir(fs_encode(root)) for item in sorted([fs_decode(x) for x in items]): if isdir(save_join(root, item)): folder = { 'name': item, 'path': item, 'files': [] } files = listdir(save_join(root, item)) for file in sorted([fs_decode(x) for x in files]): try: if isfile(save_join(root, item, file)): folder['files'].append(file) except: pass data['folder'].append(folder) elif isfile(join(root, item)): data['files'].append(item) return render_to_response('downloads.html', {'files': data}, [pre_processor])
def downloads(): root = PYLOAD.getConfigValue("general", "download_folder") if not isdir(root): return base([_('Download directory not found.')]) data = {'folder': [], 'files': []} items = listdir(fs_encode(root)) for item in sorted([fs_decode(x) for x in items]): if isdir(save_join(root, item)): folder = {'name': item, 'path': item, 'files': []} files = listdir(save_join(root, item)) for file in sorted([fs_decode(x) for x in files]): try: if isfile(save_join(root, item, file)): folder['files'].append(file) except: pass data['folder'].append(folder) elif isfile(join(root, item)): data['files'].append(item) return render_to_response('downloads.html', {'files': data}, [pre_processor])
def list(self, password=None): command = "vb" if self.fullpath else "lb" p = self.call_cmd(command, "-v", fs_encode(self.filename), password=password) out, err = p.communicate() if "Cannot open" in err: raise ArchiveError(_("Cannot open file")) if err.strip(): #: only log error at this point self.manager.logError(err.strip()) result = set() if not self.fullpath and self.VERSION.startswith('5'): # NOTE: Unrar 5 always list full path for f in fs_decode(out).splitlines(): f = save_join(self.out, os.path.basename(f.strip())) if os.path.isfile(f): result.add(save_join(self.out, os.path.basename(f))) else: for f in fs_decode(out).splitlines(): f = f.strip() result.add(save_join(self.out, f)) return list(result)
def downloads(): root = PYLOAD.getConfigValue("general", "download_folder") if not isdir(root): return base([_("Download directory not found.")]) data = {"folder": [], "files": []} items = listdir(fs_encode(root)) for item in sorted([fs_decode(x) for x in items]): if isdir(safe_join(root, item)): folder = {"name": item, "path": item, "files": []} files = listdir(safe_join(root, item)) for file in sorted([fs_decode(x) for x in files]): try: if isfile(safe_join(root, item, file)): folder["files"].append(file) except: pass data["folder"].append(folder) elif isfile(join(root, item)): data["files"].append(item) return render_to_response("downloads.html", {"files": data}, [pre_processor])
class MegaCoNz(Hoster): __name__ = "MegaCoNz" __type__ = "hoster" __version__ = "0.30" __pattern__ = r'(https?://(?:www\.)?mega(\.co)?\.nz/|mega:|chrome:.+?)#(?P<TYPE>N|)!(?P<ID>[\w^_]+)!(?P<KEY>[\w,-]+)' __description__ = """Mega.co.nz hoster plugin""" __license__ = "GPLv3" __authors__ = [("RaNaN", "*****@*****.**"), ("Walter Purcaro", "*****@*****.**")] API_URL = "https://eu.api.mega.co.nz/cs" FILE_SUFFIX = ".crypted" def b64_decode(self, data): data = data.replace("-", "+").replace("_", "/") return standard_b64decode(data + '=' * (-len(data) % 4)) def getCipherKey(self, key): """ Construct the cipher key from the given data """ a = array.array("I", self.b64_decode(key)) k = array.array("I", (a[0] ^ a[4], a[1] ^ a[5], a[2] ^ a[6], a[3] ^ a[7])) iv = a[4:6] + array.array("I", (0, 0)) meta_mac = a[6:8] return k, iv, meta_mac def api_response(self, **kwargs): """ Dispatch a call to the api, see https://mega.co.nz/#developers """ # generate a session id, no idea where to obtain elsewhere uid = random.randint(10 << 9, 10 ** 10) res = self.load(self.API_URL, get={'id': uid}, post=json_dumps([kwargs])) self.logDebug("Api Response: " + res) return json_loads(res) def decryptAttr(self, data, key): k, iv, meta_mac = self.getCipherKey(key) cbc = AES.new(k, AES.MODE_CBC, "\0" * 16) attr = decode(cbc.decrypt(self.b64_decode(data))) self.logDebug("Decrypted Attr: %s" % attr) if not attr.startswith("MEGA"): self.fail(_("Decryption failed")) # Data is padded, 0-bytes must be stripped return json_loads(re.search(r'{.+?}', attr).group(0)) def decryptFile(self, key): """ Decrypts the file at lastDownload` """ # upper 64 bit of counter start n = self.b64_decode(key)[16:24] # convert counter to long and shift bytes k, iv, meta_mac = self.getCipherKey(key) ctr = Counter.new(128, initial_value=long(n.encode("hex"), 16) << 64) cipher = AES.new(k, AES.MODE_CTR, counter=ctr) self.pyfile.setStatus("decrypting") self.pyfile.setProgress(0) file_crypted = fs_encode(self.lastDownload) file_decrypted = file_crypted.rsplit(self.FILE_SUFFIX)[0] try: f = open(file_crypted, "rb") df = open(file_decrypted, "wb") except IOError, e: self.fail(e) chunk_size = 2 ** 15 # buffer size, 32k # file_mac = [0, 0, 0, 0] # calculate CBC-MAC for checksum chunks = os.path.getsize(file_crypted) / chunk_size + 1 for i in xrange(chunks): buf = f.read(chunk_size) if not buf: break chunk = cipher.decrypt(buf) df.write(chunk) self.pyfile.setProgress(int((100.0 / chunks) * i)) # chunk_mac = [iv[0], iv[1], iv[0], iv[1]] # for i in xrange(0, chunk_size, 16): # block = chunk[i:i+16] # if len(block) % 16: # block += '=' * (16 - (len(block) % 16)) # block = array.array("I", block) # chunk_mac = [chunk_mac[0] ^ a_[0], chunk_mac[1] ^ block[1], chunk_mac[2] ^ block[2], chunk_mac[3] ^ block[3]] # chunk_mac = aes_cbc_encrypt_a32(chunk_mac, k) # file_mac = [file_mac[0] ^ chunk_mac[0], file_mac[1] ^ chunk_mac[1], file_mac[2] ^ chunk_mac[2], file_mac[3] ^ chunk_mac[3]] # file_mac = aes_cbc_encrypt_a32(file_mac, k) self.pyfile.setProgress(100) f.close() df.close() # if file_mac[0] ^ file_mac[1], file_mac[2] ^ file_mac[3] != meta_mac: # os.remove(file_decrypted) # self.fail(_("Checksum mismatch")) os.remove(file_crypted) self.lastDownload = fs_decode(file_decrypted)
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder :param url: :param get: :param post: :param ref: :param cookies: :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ self.checkForSameFiles() self.pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and not IS_WINDOWS: try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception as e: self.log.warning( _("Setting User and Group failed: %s") % str(e)) # convert back to unicode location = fs_decode(location) name = save_path(self.pyfile.name) filename = join(location, name) get_hook_manager().dispatchEvent("downloadStarts", self.pyfile, url, filename) try: newname = self.req.httpDownload( url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.getChunkCount(), resume=self.resumeDownload, progressNotify=self.pyfile.setProgress, disposition=disposition) finally: self.pyfile.size = self.req.size if disposition and newname and newname != name: #triple check, just to be sure self.log.info("%(name)s saved as %(newname)s" % { "name": name, "newname": newname }) self.pyfile.name = newname filename = join(location, newname) fs_filename = fs_encode(filename) if self.core.config["permission"]["change_file"]: chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) if self.core.config["permission"]["change_dl"] and not IS_WINDOWS: try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(fs_filename, uid, gid) except Exception as e: self.log.warning( _("Setting User and Group failed: %s") % str(e)) self.lastDownload = filename return self.lastDownload
class Plugin(Base): __name__ = "Plugin" __type__ = "hoster" __version__ = "0.10" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] __description__ = """Base plugin""" __license__ = "GPLv3" __authors__ = [("RaNaN", "*****@*****.**"), ("spoob", "*****@*****.**"), ("mkaay", "*****@*****.**")] def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) self.wantReconnect = False #: enables simultaneous processing of multiple downloads self.multiDL = True self.limitDL = 0 #: chunk limit self.chunkLimit = 1 self.resumeDownload = False #: time() + wait in seconds self.waitUntil = 0 self.waiting = False self.ocr = None #captcha reader instance #: account handler instance, see :py:class:`Account` self.account = pyfile.m.core.accountManager.getAccountPlugin( self.__name__) #: premium status self.premium = False #: username/login self.user = None if self.account and not self.account.canUse(): self.account = None if self.account: self.user, data = self.account.selectAccount() #: Browser instance, see `network.Browser` self.req = self.account.getAccountRequest(self.user) self.chunkLimit = -1 # chunk limit, -1 for unlimited #: enables resume (will be ignored if server dont accept chunks) self.resumeDownload = True self.multiDL = True #every hoster with account should provide multiple downloads #: premium status self.premium = self.account.isPremium(self.user) else: self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) #: associated pyfile instance, see `PyFile` self.pyfile = pyfile self.thread = None # holds thread in future #: location where the last call to download was saved self.lastDownload = "" #: re match of the last call to `checkDownload` self.lastCheck = None #: js engine, see `JsEngine` self.js = self.core.js self.cTask = None #captcha task self.retries = 0 # amount of retries already made self.html = None # some plugins store html code here self.init() def getChunkCount(self): if self.chunkLimit <= 0: return self.config["download"]["chunks"] return min(self.config["download"]["chunks"], self.chunkLimit) def __call__(self): return self.__name__ def init(self): """initialize the plugin (in addition to `__init__`)""" pass def setup(self): """ setup for enviroment and other things, called before downloading (possibly more than one time)""" pass def preprocessing(self, thread): """ handles important things to do before starting """ self.thread = thread if self.account: self.account.checkLogin(self.user) else: self.req.clearCookies() self.setup() self.pyfile.setStatus("starting") return self.process(self.pyfile) def process(self, pyfile): """the 'main' method of every plugin, you **have to** overwrite it""" raise NotImplementedError def resetAccount(self): """ dont use account and retry download """ self.account = None self.req = self.core.requestFactory.getRequest(self.__name__) self.retry() def checksum(self, local_file=None): """ return codes: 0 - checksum ok 1 - checksum wrong 5 - can't get checksum 10 - not implemented 20 - unknown error """ #@TODO checksum check hook return True, 10 def setWait(self, seconds, reconnect=False): """Set a specific wait time later used with `wait` :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ if reconnect: self.wantReconnect = True self.pyfile.waitUntil = time() + int(seconds) def wait(self): """ waits the time previously set """ self.waiting = True self.pyfile.setStatus("waiting") while self.pyfile.waitUntil > time(): self.thread.m.reconnecting.wait(2) if self.pyfile.abort: raise Abort if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False raise Reconnect self.waiting = False self.pyfile.setStatus("starting") def offline(self): """ fail and indicate file is offline """ raise Fail("offline") def tempOffline(self): """ fail and indicates file ist temporary offline, the core may take consequences """ raise Fail("temp. offline") def skip(self, reason): raise Skip(reason) def retry(self, max_tries=3, wait_time=1, reason=""): """Retries and begin again from the beginning :param max_tries: number of maximum retries :param wait_time: time to wait in seconds :param reason: reason for retrying, will be passed to fail if max_tries reached """ if 0 < max_tries <= self.retries: if not reason: reason = "Max retries reached" raise Fail(reason) self.wantReconnect = False self.setWait(wait_time) self.wait() self.retries += 1 raise Retry(reason) def invalidCaptcha(self): if self.cTask: self.cTask.invalid() def correctCaptcha(self): if self.cTask: self.cTask.correct() def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input :param url: url of captcha image :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled :param forceUser: if True, ocr is not used :param imgtype: Type of the Image :param result_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha :return: result of decrypting """ img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time())[-6:].replace(".", "") temp_file = open( join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") temp_file.write(img) temp_file.close() has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins if self.core.captcha: Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) else: Ocr = None if Ocr and not forceUser: sleep(randint(3000, 5000) / 1000.0) if self.pyfile.abort: raise Abort ocr = Ocr() result = ocr.get_captcha(temp_file.name) else: captchaManager = self.core.captchaManager task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) self.cTask = task captchaManager.handleCaptcha(task) while task.isWaiting(): if self.pyfile.abort: captchaManager.removeTask(task) raise Abort sleep(1) captchaManager.removeTask(task) if task.error and has_plugin: #ignore default error message since the user could use OCR self.fail( _("Pil and tesseract not installed and no Client connected for captcha decrypting" )) elif task.error: self.fail(task.error) elif not task.result: self.fail( _("No captcha result obtained in appropiate time by any of the plugins." )) result = task.result self.log.debug("Received captcha result: %s" % str(result)) if not self.core.debug: try: remove(temp_file.name) except: pass return result def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it :param url: :param get: :param post: :param ref: :param cookies: :param just_header: if True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content """ if self.pyfile.abort: raise Abort #utf8 vs decode -> please use decode attribute in all future plugins if type(url) == unicode: url = str(url) res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.core.debug: from inspect import currentframe frame = currentframe() if not exists(join("tmp", self.__name__)): makedirs(join("tmp", self.__name__)) f = open( join( "tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)), "wb") del frame # delete the frame or it wont be cleaned try: tmp = res.encode("utf8") except: tmp = res f.write(tmp) f.close() if just_header: #parse header header = {"code": self.req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") key = key.lower().strip() value = value.strip() if key in header: if type(header[key]) == list: header[key].append(value) else: header[key] = [header[key], value] else: header[key] = value res = header return res def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder :param url: :param get: :param post: :param ref: :param cookies: :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ self.checkForSameFiles() self.pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception, e: self.log.warning( _("Setting User and Group failed: %s") % str(e)) # convert back to unicode location = fs_decode(location) name = save_path(self.pyfile.name) filename = join(location, name) self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) try: newname = self.req.httpDownload( url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.getChunkCount(), resume=self.resumeDownload, progressNotify=self.pyfile.setProgress, disposition=disposition) finally: self.pyfile.size = self.req.size if disposition and newname and newname != name: #triple check, just to be sure self.log.info("%(name)s saved as %(newname)s" % { "name": name, "newname": newname }) self.pyfile.name = newname filename = join(location, newname) fs_filename = fs_encode(filename) if self.core.config["permission"]["change_file"]: chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(fs_filename, uid, gid) except Exception, e: self.log.warning( _("Setting User and Group failed: %s") % str(e))