def checkAndAddPackages(self, links, dest=Destination.Queue): """Checks online status, retrieves names, and will add packages.\ Because of this packages are not added immediatly, only for internal use. :param links: list of urls :param dest: `Destination` :return: None """ data = get_plugin_manager().parseUrls(links) get_thread_manager().createResultThread(data, True)
def reCheckPackage(self, pid): """ recheck links in package """ data = self.db.getPackageData(pid) urls = [] for pyfile in six.itervalues(data): if pyfile["status"] not in (0, 12, 13): urls.append((pyfile["url"], pyfile["plugin"])) get_thread_manager().createInfoThread(urls, pid)
def addLinks(self, urls, package): """adds links""" get_hook_manager().dispatchEvent("linksAdded", urls, package) data = get_plugin_manager().parseUrls(urls) self.db.addLinks(data, package) get_thread_manager().createInfoThread(data, package) #@TODO change from reloadAll event to package update event get_pull_manager().addEvent(ReloadAllEvent("collector"))
def shutdown(self): self.log.info(_("shutting down...")) try: if self.config['webinterface']['activated'] and hasattr(self, "webserver"): self.webserver.quit() for thread in get_thread_manager().threads: thread.put("quit") pyfiles = self.files.cache.values() for pyfile in pyfiles: pyfile.abortDownload() get_hook_manager().coreExiting() except: if self.debug: print_exc() self.log.info(_("error while shutting down")) finally: self.files.syncSave() self.shuttedDown = True self.deletePidFile()
def update_plugins(self): server_data = self.server_response() if not server_data or server_data[0] != self.pyload.api.getServerVersion(): return 0 updated = self._update_plugins(server_data) if updated: self.log_info(_("*** Plugins updated ***")) if get_plugin_manager().reloadPlugins(updated): exitcode = 1 else: self.log_warning(_("You have to restart pyLoad to use the updated plugins")) self.info['plugins'] = True exitcode = 2 paused = get_thread_manager().pause self.pyload.api.pauseServer() self.manager.dispatchEvent("plugin_updated", updated) if not paused: self.pyload.api.unpauseServer() else: self.log_info(_("All plugins are up to date!")) exitcode = 0 #: Exit codes: #: 0 = No plugin updated #: 1 = Plugins updated #: 2 = Plugins updated, but restart required return exitcode
def statusServer(self): """Some general information about the current status of pyLoad. :return: `ServerStatus` """ thread_manager = get_thread_manager() serverStatus = ServerStatus( thread_manager.pause, len(thread_manager.processingIds()), self.core.files.getQueueCount(), self.core.files.getFileCount(), 0, not thread_manager.pause and self.isTimeDownload(), self.core.config['reconnect']['activated'] and self.isTimeReconnect(), ) for pyfile in [ x.active for x in thread_manager.threads if x.active and isinstance(x.active, PyFile) ]: serverStatus.speed += pyfile.getSpeed() #bytes/s return serverStatus
def deleteLink(self, id): """deletes links""" f = self.getFile(id) if not f: return None pid = f.packageid e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue") oldorder = f.order if id in get_thread_manager().processingIds(): self.cache[id].abortDownload() if id in self.cache: del self.cache[id] self.db.deleteLink(f) get_pull_manager().addEvent(e) p = self.getPackage(pid) if not len(p.getChildren()): p.delete() pyfiles = self.cache.values() for pyfile in pyfiles: if pyfile.packageid == pid and pyfile.order > oldorder: pyfile.order -= 1 pyfile.notifyChange()
def togglePause(self): """Toggle pause state. :return: new pause state """ thread_manager = get_thread_manager() thread_manager.pause ^= True return thread_manager.pause
def finishIfDone(self): """set status to finish and release file if every thread is finished with it""" if self.id in get_thread_manager().processingIds(): return False self.setStatus("finished") self.release() self.m.checkAllLinksFinished() return True
def pollResults(self, rid): """ Polls the result available for ResultID :param rid: `ResultID` :return: `OnlineCheck`, if rid is -1 then no more data available """ result = get_thread_manager().getInfoResult(rid) if "ALL_INFO_FETCHED" in result: del result["ALL_INFO_FETCHED"] return OnlineCheck(-1, result) else: return OnlineCheck(rid, result)
def abortDownload(self): """abort pyfile if possible""" while self.id in get_thread_manager().processingIds(): self.abort = True if self.plugin and self.plugin.req: self.plugin.req.abortDownloads() sleep(0.1) self.abort = False if self.hasPlugin() and self.plugin.req: self.plugin.req.abortDownloads() self.release()
def periodical_task(self): if self.config.get('hdd') is False: return thread_manager = get_thread_manager() if (thread_manager.pause or not self.pyload.api.isTimeDownload() or not thread_manager.getActiveFiles()): return dl_folder = self.pyload.config.get('general', 'download_folder') if (self.max_mtime(dl_folder) - self.mtime) < self.periodical.interval: return self.touch(self.TMP_FILE)
def periodical_task(self): if self.pyload.debug: if self.config.get('reloadplugins'): self.autoreload_plugins() if self.config.get('nodebugupdate'): return if self.config.get('checkperiod') and \ time.time() - max(self.CHECK_INTERVAL, self.config.get('checkinterval') * 60 * 60) > self.info['last_check']: self.update() if self.do_restart is True: if get_thread_manager().pause and not self.pyload.api.statusDownloads(): self.pyload.api.restart()
def statusDownloads(self): """ Status off all currently running downloads. :return: list of `DownloadStatus` """ data = [] for pyfile in get_thread_manager().getActiveFiles(): if not isinstance(pyfile, PyFile): continue data.append( DownloadInfo(pyfile.id, pyfile.name, pyfile.getSpeed(), pyfile.getETA(), pyfile.formatETA(), pyfile.getBytesLeft(), pyfile.getSize(), pyfile.formatSize(), pyfile.getPercent(), pyfile.status, pyfile.getStatusName(), pyfile.formatWait(), pyfile.waitUntil, pyfile.packageid, pyfile.package().name, pyfile.pluginname)) return data
def checkOnlineStatus(self, urls): """ initiates online status check :param urls: :return: initial set of data as `OnlineCheck` instance containing the result id """ data = get_plugin_manager().parseUrls(urls) rid = get_thread_manager().createResultThread(data, False) tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0))) for url, pluginname in data] data = parseNames(tmp) result = {} for k, v in six.iteritems(data): for url, status in v: status.packagename = k result[url] = status return OnlineCheck(rid, result)
def startThread(self, function, *args, **kwargs): t = HookThread(get_thread_manager(), function, args, kwargs)
def start(self, rpc=True, web=True): """ starts the fun :D """ self.version = CURRENT_VERSION if not exists("pyload.conf"): from module.setup import Setup print("This is your first start, running configuration assistent now.") self.config = ConfigParser() s = Setup(pypath, self.config) res = False try: res = s.start() except SystemExit: pass except KeyboardInterrupt: print("\nSetup interrupted") except: res = False print_exc() print("Setup failed") if not res: remove("pyload.conf") exit() try: signal.signal(signal.SIGQUIT, self.quit) except: pass self.config = ConfigParser() gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None]) translation = gettext.translation("pyLoad", self.path("locale"), languages=[self.config['general']['language'],"en"],fallback=True) install_translation(translation) self.debug = self.doDebug or self.config['general']['debug_mode'] self.remote &= self.config['remote']['activated'] pid = self.isAlreadyRunning() if pid: print(_("pyLoad already running with pid %s") % pid) exit() if not IS_WINDOWS and self.config["general"]["renice"]: os.system("renice %d %d" % (self.config["general"]["renice"], os.getpid())) if self.config["permission"]["change_group"]: if not IS_WINDOWS: try: from grp import getgrnam group = getgrnam(self.config["permission"]["group"]) os.setgid(group[2]) except Exception as e: print(_("Failed changing group: %s") % e) if self.config["permission"]["change_user"]: if not IS_WINDOWS: try: from pwd import getpwnam user = getpwnam(self.config["permission"]["user"]) os.setuid(user[2]) except Exception as e: print(_("Failed changing user: %s") % e) self.check_file( self.config['log']['log_folder'], _("folder for logs"), is_folder=True, ) if self.debug: self.init_logger(logging.DEBUG) # logging level else: self.init_logger(logging.INFO) # logging level sys.excepthook = exceptHook self.do_kill = False self.do_restart = False self.shuttedDown = False self.log.info(_("Starting") + " pyLoad %s" % CURRENT_VERSION) self.log.info(_("Using home directory: %s") % getcwd()) self.writePidFile() #@TODO refractor remote.activated = self.remote self.log.debug("Remote activated: %s" % self.remote) self.check_install("Crypto", _("pycrypto to decode container files")) #img = self.check_install("Image", _("Python Image Libary (PIL) for captcha reading")) #self.check_install("pycurl", _("pycurl to download any files"), True, True) self.check_file("tmp", _("folder for temporary files"), is_folder=True) #tesser = self.check_install("tesseract", _("tesseract for captcha reading"), False) if not IS_WINDOWS else True self.captcha = True # checks seems to fail, althoug tesseract is available self.check_file( self.config['general']['download_folder'], _("folder for downloads"), is_folder=True, ) if self.config['ssl']['activated']: self.check_install("OpenSSL", _("OpenSSL for secure connection")) self.setupDB() if self.deleteLinks: self.log.info(_("All links removed")) self.db.purgeLinks() set_request_factory(RequestFactory(self)) self.lastClientConnected = 0 # later imported because they would trigger api import, and remote value not set correctly from module import Api from module.HookManager import HookManager from module.ThreadManager import ThreadManager if Api.activated != self.remote: self.log.warning("Import error: API remote status not correct.") self.api = Api.Api(self) self.scheduler = Scheduler(self) #hell yeah, so many important managers :D set_plugin_manager(PluginManager(self)) set_pull_manager(PullManager(self)) set_thread_manager(ThreadManager(self)) set_account_manager(AccountManager(self)) set_captcha_manager(CaptchaManager(self)) # HookManager sets itself as a singleton HookManager(self) set_remote_manager(RemoteManager(self)) thread_manager = get_thread_manager() self.js = JsEngine() self.log.info(_("Downloadtime: %s") % self.api.isTimeDownload()) if rpc: get_remote_manager().startBackends() if web: self.init_webserver() spaceLeft = freeSpace(self.config["general"]["download_folder"]) self.log.info(_("Free space: %s") % formatSize(spaceLeft)) self.config.save() #save so config files gets filled link_file = join(pypath, "links.txt") if exists(link_file): f = open(link_file, "rb") if f.read().strip(): self.api.addPackage("links.txt", [link_file], 1) f.close() link_file = "links.txt" if exists(link_file): f = open(link_file, "rb") if f.read().strip(): self.api.addPackage("links.txt", [link_file], 1) f.close() #self.scheduler.addJob(0, get_account_manager().getAccountInfos) self.log.info(_("Activating Accounts...")) get_account_manager().getAccountInfos() thread_manager.pause = False self.running = True self.log.info(_("Activating Plugins...")) get_hook_manager().coreReady() self.log.info(_("pyLoad is up and running")) #test api # from module.common.APIExerciser import startApiExerciser # startApiExerciser(self, 3) #some memory stats # from guppy import hpy # hp=hpy() # import objgraph # objgraph.show_most_common_types(limit=20) # import memdebug # memdebug.start(8002) locals().clear() while True: try: sleep(2) except IOError as e: if e.errno != 4: # errno.EINTR raise if self.do_restart: self.log.info(_("restarting pyLoad")) self.restart() if self.do_kill: self.shutdown() self.log.info(_("pyLoad quits")) self.removeLogger() _exit(0) #@TODO thrift blocks shutdown thread_manager.work() self.scheduler.work()
def unpauseServer(self): """Unpause server: New Downloads will be started.""" get_thread_manager().pause = False
def toggle_pause(self): thread_manager = get_thread_manager() thread_manager.pause = not thread_manager.pause return thread_manager.pause
def pauseServer(self): """Pause server: Tt wont start any new downloads, but nothing gets aborted.""" get_thread_manager().pause = True