def checkAndAddPackages(self, links, dest=Destination.Queue):
"""Checks online status, retrieves names, and will add packages.\
Because of this packages are not added immediatly, only for internal use.
:param links: list of urls
:param dest: `Destination`
:return: None
"""
data = get_plugin_manager().parseUrls(links)
get_thread_manager().createResultThread(data, True)
def reCheckPackage(self, pid):
""" recheck links in package """
data = self.db.getPackageData(pid)
urls = []
for pyfile in six.itervalues(data):
if pyfile["status"] not in (0, 12, 13):
urls.append((pyfile["url"], pyfile["plugin"]))
get_thread_manager().createInfoThread(urls, pid)
def addLinks(self, urls, package):
"""adds links"""
get_hook_manager().dispatchEvent("linksAdded", urls, package)
data = get_plugin_manager().parseUrls(urls)
self.db.addLinks(data, package)
get_thread_manager().createInfoThread(data, package)
#@TODO change from reloadAll event to package update event
get_pull_manager().addEvent(ReloadAllEvent("collector"))
def shutdown(self):
self.log.info(_("shutting down..."))
try:
if self.config['webinterface']['activated'] and hasattr(self, "webserver"):
self.webserver.quit()
for thread in get_thread_manager().threads:
thread.put("quit")
pyfiles = self.files.cache.values()
for pyfile in pyfiles:
pyfile.abortDownload()
get_hook_manager().coreExiting()
except:
if self.debug:
print_exc()
self.log.info(_("error while shutting down"))
finally:
self.files.syncSave()
self.shuttedDown = True
self.deletePidFile()
def update_plugins(self):
server_data = self.server_response()
if not server_data or server_data[0] != self.pyload.api.getServerVersion():
return 0
updated = self._update_plugins(server_data)
if updated:
self.log_info(_("*** Plugins updated ***"))
if get_plugin_manager().reloadPlugins(updated):
exitcode = 1
else:
self.log_warning(_("You have to restart pyLoad to use the updated plugins"))
self.info['plugins'] = True
exitcode = 2
paused = get_thread_manager().pause
self.pyload.api.pauseServer()
self.manager.dispatchEvent("plugin_updated", updated)
if not paused:
self.pyload.api.unpauseServer()
else:
self.log_info(_("All plugins are up to date!"))
exitcode = 0
#: Exit codes:
#: 0 = No plugin updated
#: 1 = Plugins updated
#: 2 = Plugins updated, but restart required
return exitcode
def statusServer(self):
"""Some general information about the current status of pyLoad.
:return: `ServerStatus`
"""
thread_manager = get_thread_manager()
serverStatus = ServerStatus(
thread_manager.pause,
len(thread_manager.processingIds()),
self.core.files.getQueueCount(),
self.core.files.getFileCount(),
0,
not thread_manager.pause and self.isTimeDownload(),
self.core.config['reconnect']['activated']
and self.isTimeReconnect(),
)
for pyfile in [
x.active for x in thread_manager.threads
if x.active and isinstance(x.active, PyFile)
]:
serverStatus.speed += pyfile.getSpeed() #bytes/s
return serverStatus
def deleteLink(self, id):
"""deletes links"""
f = self.getFile(id)
if not f:
return None
pid = f.packageid
e = RemoveEvent("file", id, "collector" if not f.package().queue else "queue")
oldorder = f.order
if id in get_thread_manager().processingIds():
self.cache[id].abortDownload()
if id in self.cache:
del self.cache[id]
self.db.deleteLink(f)
get_pull_manager().addEvent(e)
p = self.getPackage(pid)
if not len(p.getChildren()):
p.delete()
pyfiles = self.cache.values()
for pyfile in pyfiles:
if pyfile.packageid == pid and pyfile.order > oldorder:
pyfile.order -= 1
pyfile.notifyChange()
def togglePause(self):
"""Toggle pause state.
:return: new pause state
"""
thread_manager = get_thread_manager()
thread_manager.pause ^= True
return thread_manager.pause
def finishIfDone(self):
"""set status to finish and release file if every thread is finished with it"""
if self.id in get_thread_manager().processingIds():
return False
self.setStatus("finished")
self.release()
self.m.checkAllLinksFinished()
return True
def pollResults(self, rid):
""" Polls the result available for ResultID
:param rid: `ResultID`
:return: `OnlineCheck`, if rid is -1 then no more data available
"""
result = get_thread_manager().getInfoResult(rid)
if "ALL_INFO_FETCHED" in result:
del result["ALL_INFO_FETCHED"]
return OnlineCheck(-1, result)
else:
return OnlineCheck(rid, result)
def abortDownload(self):
"""abort pyfile if possible"""
while self.id in get_thread_manager().processingIds():
self.abort = True
if self.plugin and self.plugin.req:
self.plugin.req.abortDownloads()
sleep(0.1)
self.abort = False
if self.hasPlugin() and self.plugin.req:
self.plugin.req.abortDownloads()
self.release()
def periodical_task(self):
if self.config.get('hdd') is False:
return
thread_manager = get_thread_manager()
if (thread_manager.pause or not self.pyload.api.isTimeDownload()
or not thread_manager.getActiveFiles()):
return
dl_folder = self.pyload.config.get('general', 'download_folder')
if (self.max_mtime(dl_folder) - self.mtime) < self.periodical.interval:
return
self.touch(self.TMP_FILE)
def periodical_task(self):
if self.pyload.debug:
if self.config.get('reloadplugins'):
self.autoreload_plugins()
if self.config.get('nodebugupdate'):
return
if self.config.get('checkperiod') and \
time.time() - max(self.CHECK_INTERVAL, self.config.get('checkinterval') * 60 * 60) > self.info['last_check']:
self.update()
if self.do_restart is True:
if get_thread_manager().pause and not self.pyload.api.statusDownloads():
self.pyload.api.restart()
def statusDownloads(self):
""" Status off all currently running downloads.
:return: list of `DownloadStatus`
"""
data = []
for pyfile in get_thread_manager().getActiveFiles():
if not isinstance(pyfile, PyFile):
continue
data.append(
DownloadInfo(pyfile.id, pyfile.name, pyfile.getSpeed(),
pyfile.getETA(), pyfile.formatETA(),
pyfile.getBytesLeft(), pyfile.getSize(),
pyfile.formatSize(), pyfile.getPercent(),
pyfile.status, pyfile.getStatusName(),
pyfile.formatWait(), pyfile.waitUntil,
pyfile.packageid,
pyfile.package().name, pyfile.pluginname))
return data
def checkOnlineStatus(self, urls):
""" initiates online status check
:param urls:
:return: initial set of data as `OnlineCheck` instance containing the result id
"""
data = get_plugin_manager().parseUrls(urls)
rid = get_thread_manager().createResultThread(data, False)
tmp = [(url, (url, OnlineStatus(url, pluginname, "unknown", 3, 0)))
for url, pluginname in data]
data = parseNames(tmp)
result = {}
for k, v in six.iteritems(data):
for url, status in v:
status.packagename = k
result[url] = status
return OnlineCheck(rid, result)
def startThread(self, function, *args, **kwargs):
t = HookThread(get_thread_manager(), function, args, kwargs)
def start(self, rpc=True, web=True):
""" starts the fun :D """
self.version = CURRENT_VERSION
if not exists("pyload.conf"):
from module.setup import Setup
print("This is your first start, running configuration assistent now.")
self.config = ConfigParser()
s = Setup(pypath, self.config)
res = False
try:
res = s.start()
except SystemExit:
pass
except KeyboardInterrupt:
print("\nSetup interrupted")
except:
res = False
print_exc()
print("Setup failed")
if not res:
remove("pyload.conf")
exit()
try: signal.signal(signal.SIGQUIT, self.quit)
except: pass
self.config = ConfigParser()
gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation("pyLoad", self.path("locale"),
languages=[self.config['general']['language'],"en"],fallback=True)
install_translation(translation)
self.debug = self.doDebug or self.config['general']['debug_mode']
self.remote &= self.config['remote']['activated']
pid = self.isAlreadyRunning()
if pid:
print(_("pyLoad already running with pid %s") % pid)
exit()
if not IS_WINDOWS and self.config["general"]["renice"]:
os.system("renice %d %d" % (self.config["general"]["renice"], os.getpid()))
if self.config["permission"]["change_group"]:
if not IS_WINDOWS:
try:
from grp import getgrnam
group = getgrnam(self.config["permission"]["group"])
os.setgid(group[2])
except Exception as e:
print(_("Failed changing group: %s") % e)
if self.config["permission"]["change_user"]:
if not IS_WINDOWS:
try:
from pwd import getpwnam
user = getpwnam(self.config["permission"]["user"])
os.setuid(user[2])
except Exception as e:
print(_("Failed changing user: %s") % e)
self.check_file(
self.config['log']['log_folder'],
_("folder for logs"),
is_folder=True,
)
if self.debug:
self.init_logger(logging.DEBUG) # logging level
else:
self.init_logger(logging.INFO) # logging level
sys.excepthook = exceptHook
self.do_kill = False
self.do_restart = False
self.shuttedDown = False
self.log.info(_("Starting") + " pyLoad %s" % CURRENT_VERSION)
self.log.info(_("Using home directory: %s") % getcwd())
self.writePidFile()
#@TODO refractor
remote.activated = self.remote
self.log.debug("Remote activated: %s" % self.remote)
self.check_install("Crypto", _("pycrypto to decode container files"))
#img = self.check_install("Image", _("Python Image Libary (PIL) for captcha reading"))
#self.check_install("pycurl", _("pycurl to download any files"), True, True)
self.check_file("tmp", _("folder for temporary files"), is_folder=True)
#tesser = self.check_install("tesseract", _("tesseract for captcha reading"), False) if not IS_WINDOWS else True
self.captcha = True # checks seems to fail, althoug tesseract is available
self.check_file(
self.config['general']['download_folder'],
_("folder for downloads"),
is_folder=True,
)
if self.config['ssl']['activated']:
self.check_install("OpenSSL", _("OpenSSL for secure connection"))
self.setupDB()
if self.deleteLinks:
self.log.info(_("All links removed"))
self.db.purgeLinks()
set_request_factory(RequestFactory(self))
self.lastClientConnected = 0
# later imported because they would trigger api import, and remote value not set correctly
from module import Api
from module.HookManager import HookManager
from module.ThreadManager import ThreadManager
if Api.activated != self.remote:
self.log.warning("Import error: API remote status not correct.")
self.api = Api.Api(self)
self.scheduler = Scheduler(self)
#hell yeah, so many important managers :D
set_plugin_manager(PluginManager(self))
set_pull_manager(PullManager(self))
set_thread_manager(ThreadManager(self))
set_account_manager(AccountManager(self))
set_captcha_manager(CaptchaManager(self))
# HookManager sets itself as a singleton
HookManager(self)
set_remote_manager(RemoteManager(self))
thread_manager = get_thread_manager()
self.js = JsEngine()
self.log.info(_("Downloadtime: %s") % self.api.isTimeDownload())
if rpc:
get_remote_manager().startBackends()
if web:
self.init_webserver()
spaceLeft = freeSpace(self.config["general"]["download_folder"])
self.log.info(_("Free space: %s") % formatSize(spaceLeft))
self.config.save() #save so config files gets filled
link_file = join(pypath, "links.txt")
if exists(link_file):
f = open(link_file, "rb")
if f.read().strip():
self.api.addPackage("links.txt", [link_file], 1)
f.close()
link_file = "links.txt"
if exists(link_file):
f = open(link_file, "rb")
if f.read().strip():
self.api.addPackage("links.txt", [link_file], 1)
f.close()
#self.scheduler.addJob(0, get_account_manager().getAccountInfos)
self.log.info(_("Activating Accounts..."))
get_account_manager().getAccountInfos()
thread_manager.pause = False
self.running = True
self.log.info(_("Activating Plugins..."))
get_hook_manager().coreReady()
self.log.info(_("pyLoad is up and running"))
#test api
# from module.common.APIExerciser import startApiExerciser
# startApiExerciser(self, 3)
#some memory stats
# from guppy import hpy
# hp=hpy()
# import objgraph
# objgraph.show_most_common_types(limit=20)
# import memdebug
# memdebug.start(8002)
locals().clear()
while True:
try:
sleep(2)
except IOError as e:
if e.errno != 4: # errno.EINTR
raise
if self.do_restart:
self.log.info(_("restarting pyLoad"))
self.restart()
if self.do_kill:
self.shutdown()
self.log.info(_("pyLoad quits"))
self.removeLogger()
_exit(0) #@TODO thrift blocks shutdown
thread_manager.work()
self.scheduler.work()
def unpauseServer(self):
"""Unpause server: New Downloads will be started."""
get_thread_manager().pause = False
def toggle_pause(self):
thread_manager = get_thread_manager()
thread_manager.pause = not thread_manager.pause
return thread_manager.pause
def pauseServer(self):
"""Pause server: Tt wont start any new downloads, but nothing gets aborted."""
get_thread_manager().pause = True
