def assignJob(self):
"""assing a job to a thread if possible"""
if self.pause or not self.core.api.isTimeDownload(): return
# if self.downloaded > 20:
# if not self.cleanPyCurl(): return
free = [x for x in self.threads if not x.active]
inuse = set([(x.active.pluginname, self.getLimit(x)) for x in self.threads if
x.active and x.active.hasPlugin() and x.active.plugin.account])
inuse = map(lambda x: (x[0], x[1], len([y for y in self.threads if y.active and y.active.pluginname == x[0]])),
inuse)
onlimit = [x[0] for x in inuse if 0 < x[1] <= x[2]]
occ = [x.active.pluginname for x in self.threads if
x.active and x.active.hasPlugin() and not x.active.plugin.multiDL] + onlimit
occ.sort()
occ = tuple(set(occ))
job = self.core.files.getJob(occ)
if job:
try:
job.initPlugin()
except Exception, e:
self.log.critical(str(e))
print_exc()
job.setStatus("failed")
job.error = str(e)
job.release()
return
if job.plugin.__type__ == "hoster":
spaceLeft = freeSpace(self.core.config["general"]["download_folder"]) / 1024 / 1024
if spaceLeft < self.core.config["general"]["min_free_space"]:
self.log.warning(_("Not enough space left on device"))
self.pause = True
if free and not self.pause:
thread = free[0]
# self.downloaded += 1
thread.put(job)
else:
# put job back
if occ not in self.core.files.jobCache:
self.core.files.jobCache[occ] = []
self.core.files.jobCache[occ].append(job.id)
# check for decrypt jobs
job = self.core.files.getDecryptJob()
if job:
job.initPlugin()
thread = PluginThread.DecrypterThread(self, job)
else:
thread = PluginThread.DecrypterThread(self, job)
def assignJob(self):
"""assing a job to a thread if possible"""
if self.pause or not self.core.api.isTimeDownload(): return
#if self.downloaded > 20:
# if not self.cleanPyCurl(): return
free = [x for x in self.threads if not x.active]
inuse = set([
(x.active.pluginname, self.getLimit(x)) for x in self.threads
if x.active and x.active.hasPlugin() and x.active.plugin.account
])
inuse = map(
lambda x: (x[0], x[1],
len([
y for y in self.threads
if y.active and y.active.pluginname == x[0]
])), inuse)
onlimit = [x[0] for x in inuse if x[1] > 0 and x[2] >= x[1]]
occ = [
x.active.pluginname for x in self.threads if x.active
and x.active.hasPlugin() and not x.active.plugin.multiDL
] + onlimit
occ.sort()
occ = tuple(set(occ))
job = self.core.files.getJob(occ)
if job:
try:
job.initPlugin()
except Exception, e:
self.log.critical(str(e))
print_exc()
job.setStatus("failed")
job.error = str(e)
job.release()
return
if job.plugin.__type__ == "hoster":
spaceLeft = freeSpace(self.core.config["general"]
["download_folder"]) / 1024 / 1024
if spaceLeft < self.core.config["general"]["min_free_space"]:
self.log.warning(_("Not enough space left on device"))
self.pause = True
if free and not self.pause:
thread = free[0]
#self.downloaded += 1
thread.put(job)
else:
#put job back
if occ not in self.core.files.jobCache:
self.core.files.jobCache[occ] = []
self.core.files.jobCache[occ].append(job.id)
#check for decrypt jobs
job = self.core.files.getDecryptJob()
if job:
job.initPlugin()
thread = PluginThread.DecrypterThread(self, job)
else:
thread = PluginThread.DecrypterThread(self, job)
def freeSpace(self):
return freeSpace(self.core.config["general"]["download_folder"])
def freeSpace(self):
return freeSpace(self.core.config["general"]["download_folder"])
def start(self, rpc=True, web=True):
""" starts the fun :D """
self.version = CURRENT_VERSION
if not exists("pyload.conf"):
from module.setup import Setup
print("This is your first start, running configuration assistent now.")
self.config = ConfigParser()
s = Setup(pypath, self.config)
res = False
try:
res = s.start()
except SystemExit:
pass
except KeyboardInterrupt:
print("\nSetup interrupted")
except:
res = False
print_exc()
print("Setup failed")
if not res:
remove("pyload.conf")
exit()
try: signal.signal(signal.SIGQUIT, self.quit)
except: pass
self.config = ConfigParser()
gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation("pyLoad", self.path("locale"),
languages=[self.config['general']['language'],"en"],fallback=True)
install_translation(translation)
self.debug = self.doDebug or self.config['general']['debug_mode']
self.remote &= self.config['remote']['activated']
pid = self.isAlreadyRunning()
if pid:
print(_("pyLoad already running with pid %s") % pid)
exit()
if not IS_WINDOWS and self.config["general"]["renice"]:
os.system("renice %d %d" % (self.config["general"]["renice"], os.getpid()))
if self.config["permission"]["change_group"]:
if not IS_WINDOWS:
try:
from grp import getgrnam
group = getgrnam(self.config["permission"]["group"])
os.setgid(group[2])
except Exception as e:
print(_("Failed changing group: %s") % e)
if self.config["permission"]["change_user"]:
if not IS_WINDOWS:
try:
from pwd import getpwnam
user = getpwnam(self.config["permission"]["user"])
os.setuid(user[2])
except Exception as e:
print(_("Failed changing user: %s") % e)
self.check_file(
self.config['log']['log_folder'],
_("folder for logs"),
is_folder=True,
)
if self.debug:
self.init_logger(logging.DEBUG) # logging level
else:
self.init_logger(logging.INFO) # logging level
sys.excepthook = exceptHook
self.do_kill = False
self.do_restart = False
self.shuttedDown = False
self.log.info(_("Starting") + " pyLoad %s" % CURRENT_VERSION)
self.log.info(_("Using home directory: %s") % getcwd())
self.writePidFile()
#@TODO refractor
remote.activated = self.remote
self.log.debug("Remote activated: %s" % self.remote)
self.check_install("Crypto", _("pycrypto to decode container files"))
#img = self.check_install("Image", _("Python Image Libary (PIL) for captcha reading"))
#self.check_install("pycurl", _("pycurl to download any files"), True, True)
self.check_file("tmp", _("folder for temporary files"), is_folder=True)
#tesser = self.check_install("tesseract", _("tesseract for captcha reading"), False) if not IS_WINDOWS else True
self.captcha = True # checks seems to fail, althoug tesseract is available
self.check_file(
self.config['general']['download_folder'],
_("folder for downloads"),
is_folder=True,
)
if self.config['ssl']['activated']:
self.check_install("OpenSSL", _("OpenSSL for secure connection"))
self.setupDB()
if self.deleteLinks:
self.log.info(_("All links removed"))
self.db.purgeLinks()
set_request_factory(RequestFactory(self))
self.lastClientConnected = 0
# later imported because they would trigger api import, and remote value not set correctly
from module import Api
from module.HookManager import HookManager
from module.ThreadManager import ThreadManager
if Api.activated != self.remote:
self.log.warning("Import error: API remote status not correct.")
self.api = Api.Api(self)
self.scheduler = Scheduler(self)
#hell yeah, so many important managers :D
set_plugin_manager(PluginManager(self))
set_pull_manager(PullManager(self))
set_thread_manager(ThreadManager(self))
set_account_manager(AccountManager(self))
set_captcha_manager(CaptchaManager(self))
# HookManager sets itself as a singleton
HookManager(self)
set_remote_manager(RemoteManager(self))
thread_manager = get_thread_manager()
self.js = JsEngine()
self.log.info(_("Downloadtime: %s") % self.api.isTimeDownload())
if rpc:
get_remote_manager().startBackends()
if web:
self.init_webserver()
spaceLeft = freeSpace(self.config["general"]["download_folder"])
self.log.info(_("Free space: %s") % formatSize(spaceLeft))
self.config.save() #save so config files gets filled
link_file = join(pypath, "links.txt")
if exists(link_file):
f = open(link_file, "rb")
if f.read().strip():
self.api.addPackage("links.txt", [link_file], 1)
f.close()
link_file = "links.txt"
if exists(link_file):
f = open(link_file, "rb")
if f.read().strip():
self.api.addPackage("links.txt", [link_file], 1)
f.close()
#self.scheduler.addJob(0, get_account_manager().getAccountInfos)
self.log.info(_("Activating Accounts..."))
get_account_manager().getAccountInfos()
thread_manager.pause = False
self.running = True
self.log.info(_("Activating Plugins..."))
get_hook_manager().coreReady()
self.log.info(_("pyLoad is up and running"))
#test api
# from module.common.APIExerciser import startApiExerciser
# startApiExerciser(self, 3)
#some memory stats
# from guppy import hpy
# hp=hpy()
# import objgraph
# objgraph.show_most_common_types(limit=20)
# import memdebug
# memdebug.start(8002)
locals().clear()
while True:
try:
sleep(2)
except IOError as e:
if e.errno != 4: # errno.EINTR
raise
if self.do_restart:
self.log.info(_("restarting pyLoad"))
self.restart()
if self.do_kill:
self.shutdown()
self.log.info(_("pyLoad quits"))
self.removeLogger()
_exit(0) #@TODO thrift blocks shutdown
thread_manager.work()
self.scheduler.work()
