def process(self, pyfile): site = self.load(pyfile.url) avail_videos = re.findall(r"""mediaCollection.addMediaStream\(0, ([0-9]*), "([^\"]*)", "([^\"]*)", "[^\"]*"\);""", site) avail_videos.sort(key=lambda videodesc: int(videodesc[0]), reverse=True) # The higher the number, the better the quality quality, url, playpath = avail_videos[0] pyfile.name = re.search(r"<h1>([^<]*)</h1>", site).group(1) if url.startswith("http"): # Best quality is available over HTTP. Very rare. self.download(url) else: pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, pyfile.package().folder) if not os.path.exists(location): os.makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception, e: self.log.warning(_("Setting User and Group failed: %s") % str(e)) output_file = save_join(location, save_path(pyfile.name)) + os.path.splitext(playpath)[1] RTMP.download_rtmp_stream(url, playpath=playpath, output_file=output_file)
def periodical(self): if not self.configIsValid(): self.setConfig("activated", False) self.logWarning('deactivated because of invalid config.') return # sleep 10 secs in case pc just woke up and # network connection is not yet established sleep(10) # prepare names of preferred hosters self.preferredHosters = self.getConfig( "preferredHosters").strip().split(',') if len(self.preferredHosters[0]) == 0: self.preferredHosters = {} if len(self.preferredHosters) > 0: for i in range(len(self.preferredHosters)): self.preferredHosters[i] = self.preferredHosters[i].strip() seriesCfg = ConfigParser.RawConfigParser() parsedCfgFiles = seriesCfg.read(self.getConfig("showsCfgFile")) self.logInfo('Processed config file(s): %s' % str.join(',', parsedCfgFiles)) self.printSeriesCfgInfo(seriesCfg) for showName in seriesCfg.sections(): if not seriesCfg.getboolean(showName, 'active'): continue if not self.seriesCfgIsValid(seriesCfg, showName): continue self.logInfo('Syncronizing %s' % showName) showUrl = seriesCfg.get(showName, 'url') showHdPreferred = seriesCfg.getboolean(showName, 'hdPreferred') showExclEpisodes = re.findall( r'\w+', seriesCfg.get(showName, 'excludedEpisodes').lower()) showExclSeasons = re.findall( r'\w+', seriesCfg.get(showName, 'excludedSeasons')) showDirFmt = self.getConf('format') showsBaseDir = self.getConfig("showsBaseDir") showDir = save_join( self.core.api.getConfigValue('general', 'download_folder'), showsBaseDir, save_path(showName)) queue = seriesCfg.getboolean(showName, 'queue') self.logDebug("%s queue=%s" % (showName, queue)) show = Show(self, showName, showDir, showUrl, showHdPreferred, showExclSeasons, showExclEpisodes, showDirFmt, queue) self.logDebug(show) show.syncronize() self.logInfo('Finished')
def _create_packages(self): """Create new packages from self.packages""" package_folder = self.pyfile.package().folder package_password = self.pyfile.package().password package_queue = self.pyfile.package().queue folder_per_package = self.core.config.get('general', 'folder_per_package') use_subfolder = self.getConfig('use_subfolder', folder_per_package) subfolder_per_package = self.getConfig('subfolder_per_package', True) for name, links, folder in self.packages: self.logDebug( "Parsed package: %s" % name, "%d links" % len(links), "Saved to folder: %s" % folder if folder else "Saved to download folder") links = map(decode, links) pid = self.core.api.addPackage(name, links, package_queue) if package_password: self.core.api.setPackageData(pid, {"password": package_password}) setFolder = lambda x: self.core.api.setPackageData( pid, {"folder": x or ""} ) #@NOTE: Workaround to do not break API addPackage method if use_subfolder: if not subfolder_per_package: setFolder(package_folder) self.logDebug( "Set package %(name)s folder to: %(folder)s" % { "name": name, "folder": folder }) elif not folder_per_package or name != folder: if not folder: folder = urlparse.urlparse(name).path.split("/")[-1] setFolder(save_path(folder)) self.logDebug( "Set package %(name)s folder to: %(folder)s" % { "name": name, "folder": folder }) elif folder_per_package: setFolder(None)
def process(self, pyfile): site = self.load(pyfile.url) avail_videos = re.findall( r"""mediaCollection.addMediaStream\(0, ([0-9]*), "([^\"]*)", "([^\"]*)", "[^\"]*"\);""", site) avail_videos.sort( key=lambda videodesc: int(videodesc[0]), reverse=True) # The higher the number, the better the quality quality, url, playpath = avail_videos[0] pyfile.name = re.search(r"<h1>([^<]*)</h1>", site).group(1) if url.startswith("http"): # Best quality is available over HTTP. Very rare. self.download(url) else: pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, pyfile.package().folder) if not os.path.exists(location): os.makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"][ "change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception, e: self.log.warning( _("Setting User and Group failed: %s") % str(e)) output_file = save_join(location, save_path( pyfile.name)) + os.path.splitext(playpath)[1] RTMP.download_rtmp_stream(url, playpath=playpath, output_file=output_file)
def folder(self): return save_path(self._folder)
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder :param url: :param get: :param post: :param ref: :param cookies: :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ self.checkForSameFiles() self.pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and not IS_WINDOWS: try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception as e: self.log.warning( _("Setting User and Group failed: %s") % str(e)) # convert back to unicode location = fs_decode(location) name = save_path(self.pyfile.name) filename = join(location, name) get_hook_manager().dispatchEvent("downloadStarts", self.pyfile, url, filename) try: newname = self.req.httpDownload( url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.getChunkCount(), resume=self.resumeDownload, progressNotify=self.pyfile.setProgress, disposition=disposition) finally: self.pyfile.size = self.req.size if disposition and newname and newname != name: #triple check, just to be sure self.log.info("%(name)s saved as %(newname)s" % { "name": name, "newname": newname }) self.pyfile.name = newname filename = join(location, newname) fs_filename = fs_encode(filename) if self.core.config["permission"]["change_file"]: chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) if self.core.config["permission"]["change_dl"] and not IS_WINDOWS: try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(fs_filename, uid, gid) except Exception as e: self.log.warning( _("Setting User and Group failed: %s") % str(e)) self.lastDownload = filename return self.lastDownload
class Plugin(Base): __name__ = "Plugin" __type__ = "hoster" __version__ = "0.10" __pattern__ = r'^unmatchable$' __config__ = [] #: [("name", "type", "desc", "default")] __description__ = """Base plugin""" __license__ = "GPLv3" __authors__ = [("RaNaN", "*****@*****.**"), ("spoob", "*****@*****.**"), ("mkaay", "*****@*****.**")] def __init__(self, pyfile): Base.__init__(self, pyfile.m.core) self.wantReconnect = False #: enables simultaneous processing of multiple downloads self.multiDL = True self.limitDL = 0 #: chunk limit self.chunkLimit = 1 self.resumeDownload = False #: time() + wait in seconds self.waitUntil = 0 self.waiting = False self.ocr = None #captcha reader instance #: account handler instance, see :py:class:`Account` self.account = pyfile.m.core.accountManager.getAccountPlugin( self.__name__) #: premium status self.premium = False #: username/login self.user = None if self.account and not self.account.canUse(): self.account = None if self.account: self.user, data = self.account.selectAccount() #: Browser instance, see `network.Browser` self.req = self.account.getAccountRequest(self.user) self.chunkLimit = -1 # chunk limit, -1 for unlimited #: enables resume (will be ignored if server dont accept chunks) self.resumeDownload = True self.multiDL = True #every hoster with account should provide multiple downloads #: premium status self.premium = self.account.isPremium(self.user) else: self.req = pyfile.m.core.requestFactory.getRequest(self.__name__) #: associated pyfile instance, see `PyFile` self.pyfile = pyfile self.thread = None # holds thread in future #: location where the last call to download was saved self.lastDownload = "" #: re match of the last call to `checkDownload` self.lastCheck = None #: js engine, see `JsEngine` self.js = self.core.js self.cTask = None #captcha task self.retries = 0 # amount of retries already made self.html = None # some plugins store html code here self.init() def getChunkCount(self): if self.chunkLimit <= 0: return self.config["download"]["chunks"] return min(self.config["download"]["chunks"], self.chunkLimit) def __call__(self): return self.__name__ def init(self): """initialize the plugin (in addition to `__init__`)""" pass def setup(self): """ setup for enviroment and other things, called before downloading (possibly more than one time)""" pass def preprocessing(self, thread): """ handles important things to do before starting """ self.thread = thread if self.account: self.account.checkLogin(self.user) else: self.req.clearCookies() self.setup() self.pyfile.setStatus("starting") return self.process(self.pyfile) def process(self, pyfile): """the 'main' method of every plugin, you **have to** overwrite it""" raise NotImplementedError def resetAccount(self): """ dont use account and retry download """ self.account = None self.req = self.core.requestFactory.getRequest(self.__name__) self.retry() def checksum(self, local_file=None): """ return codes: 0 - checksum ok 1 - checksum wrong 5 - can't get checksum 10 - not implemented 20 - unknown error """ #@TODO checksum check hook return True, 10 def setWait(self, seconds, reconnect=False): """Set a specific wait time later used with `wait` :param seconds: wait time in seconds :param reconnect: True if a reconnect would avoid wait time """ if reconnect: self.wantReconnect = True self.pyfile.waitUntil = time() + int(seconds) def wait(self): """ waits the time previously set """ self.waiting = True self.pyfile.setStatus("waiting") while self.pyfile.waitUntil > time(): self.thread.m.reconnecting.wait(2) if self.pyfile.abort: raise Abort if self.thread.m.reconnecting.isSet(): self.waiting = False self.wantReconnect = False raise Reconnect self.waiting = False self.pyfile.setStatus("starting") def offline(self): """ fail and indicate file is offline """ raise Fail("offline") def tempOffline(self): """ fail and indicates file ist temporary offline, the core may take consequences """ raise Fail("temp. offline") def skip(self, reason): raise Skip(reason) def retry(self, max_tries=3, wait_time=1, reason=""): """Retries and begin again from the beginning :param max_tries: number of maximum retries :param wait_time: time to wait in seconds :param reason: reason for retrying, will be passed to fail if max_tries reached """ if 0 < max_tries <= self.retries: if not reason: reason = "Max retries reached" raise Fail(reason) self.wantReconnect = False self.setWait(wait_time) self.wait() self.retries += 1 raise Retry(reason) def invalidCaptcha(self): if self.cTask: self.cTask.invalid() def correctCaptcha(self): if self.cTask: self.cTask.correct() def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg', result_type='textual'): """ Loads a captcha and decrypts it with ocr, plugin, user input :param url: url of captcha image :param get: get part for request :param post: post part for request :param cookies: True if cookies should be enabled :param forceUser: if True, ocr is not used :param imgtype: Type of the Image :param result_type: 'textual' if text is written on the captcha\ or 'positional' for captcha where the user have to click\ on a specific region on the captcha :return: result of decrypting """ img = self.load(url, get=get, post=post, cookies=cookies) id = ("%.2f" % time())[-6:].replace(".", "") temp_file = open( join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb") temp_file.write(img) temp_file.close() has_plugin = self.__name__ in self.core.pluginManager.captchaPlugins if self.core.captcha: Ocr = self.core.pluginManager.loadClass("captcha", self.__name__) else: Ocr = None if Ocr and not forceUser: sleep(randint(3000, 5000) / 1000.0) if self.pyfile.abort: raise Abort ocr = Ocr() result = ocr.get_captcha(temp_file.name) else: captchaManager = self.core.captchaManager task = captchaManager.newTask(img, imgtype, temp_file.name, result_type) self.cTask = task captchaManager.handleCaptcha(task) while task.isWaiting(): if self.pyfile.abort: captchaManager.removeTask(task) raise Abort sleep(1) captchaManager.removeTask(task) if task.error and has_plugin: #ignore default error message since the user could use OCR self.fail( _("Pil and tesseract not installed and no Client connected for captcha decrypting" )) elif task.error: self.fail(task.error) elif not task.result: self.fail( _("No captcha result obtained in appropiate time by any of the plugins." )) result = task.result self.log.debug("Received captcha result: %s" % str(result)) if not self.core.debug: try: remove(temp_file.name) except: pass return result def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False): """Load content at url and returns it :param url: :param get: :param post: :param ref: :param cookies: :param just_header: if True only the header will be retrieved and returned as dict :param decode: Wether to decode the output according to http header, should be True in most cases :return: Loaded content """ if self.pyfile.abort: raise Abort #utf8 vs decode -> please use decode attribute in all future plugins if type(url) == unicode: url = str(url) res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode) if self.core.debug: from inspect import currentframe frame = currentframe() if not exists(join("tmp", self.__name__)): makedirs(join("tmp", self.__name__)) f = open( join( "tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno)), "wb") del frame # delete the frame or it wont be cleaned try: tmp = res.encode("utf8") except: tmp = res f.write(tmp) f.close() if just_header: #parse header header = {"code": self.req.code} for line in res.splitlines(): line = line.strip() if not line or ":" not in line: continue key, none, value = line.partition(":") key = key.lower().strip() value = value.strip() if key in header: if type(header[key]) == list: header[key].append(value) else: header[key] = [header[key], value] else: header[key] = value res = header return res def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False): """Downloads the content at url to download folder :param url: :param get: :param post: :param ref: :param cookies: :param disposition: if True and server provides content-disposition header\ the filename will be changed if needed :return: The location where the file was saved """ self.checkForSameFiles() self.pyfile.setStatus("downloading") download_folder = self.config['general']['download_folder'] location = save_join(download_folder, self.pyfile.package().folder) if not exists(location): makedirs(location, int(self.core.config["permission"]["folder"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(location, uid, gid) except Exception, e: self.log.warning( _("Setting User and Group failed: %s") % str(e)) # convert back to unicode location = fs_decode(location) name = save_path(self.pyfile.name) filename = join(location, name) self.core.hookManager.dispatchEvent("downloadStarts", self.pyfile, url, filename) try: newname = self.req.httpDownload( url, filename, get=get, post=post, ref=ref, cookies=cookies, chunks=self.getChunkCount(), resume=self.resumeDownload, progressNotify=self.pyfile.setProgress, disposition=disposition) finally: self.pyfile.size = self.req.size if disposition and newname and newname != name: #triple check, just to be sure self.log.info("%(name)s saved as %(newname)s" % { "name": name, "newname": newname }) self.pyfile.name = newname filename = join(location, newname) fs_filename = fs_encode(filename) if self.core.config["permission"]["change_file"]: chmod(fs_filename, int(self.core.config["permission"]["file"], 8)) if self.core.config["permission"]["change_dl"] and os.name != "nt": try: uid = getpwnam(self.config["permission"]["user"])[2] gid = getgrnam(self.config["permission"]["group"])[2] chown(fs_filename, uid, gid) except Exception, e: self.log.warning( _("Setting User and Group failed: %s") % str(e))