def conf_path(self, trans=False):
if trans:
gettext.setpaths(
[join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation(
"setup",
join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"],
fallback=True)
translation.install(True)
print _(
"Setting new configpath, current configuration will not be transferred!"
)
path = self.ask(_("Config path"), abspath(""))
try:
path = join(pypath, path)
if not exists(path):
makedirs(path)
f = open(join(pypath, "pyload", "config", "configdir"), "wb")
f.write(path)
f.close()
print _(
"Config path changed, setup will now close, please restart to go on."
)
print _("Press Enter to exit.")
raw_input()
exit()
except Exception, e:
print _("Setting config path failed: %s") % str(e)
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
"""Load content at url and returns it
:param url: url as string
:param get: GET as dict
:param post: POST as dict, list or string
:param ref: Set HTTP_REFERER header
:param cookies: use saved cookies
:param just_header: if True only the header will be retrieved and returned as dict
:param decode: Whether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.")
self.checkAbort()
res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
if not exists(join("tmp", self.__name__)):
makedirs(join("tmp", self.__name__))
f = open(
join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
, "wb")
del frame # delete the frame or it wont be cleaned
try:
tmp = res.encode("utf8")
except:
tmp = res
f.write(tmp)
f.close()
if just_header:
#parse header
header = {"code": self.req.code}
for line in res.splitlines():
line = line.strip()
if not line or ":" not in line: continue
key, none, value = line.partition(":")
key = key.lower().strip()
value = value.strip()
if key in header:
if type(header[key]) == list:
header[key].append(value)
else:
header[key] = [header[key], value]
else:
header[key] = value
res = header
return res
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def ask_lang(self):
langs = self.config.getMetaData("general", "language").type.split(";")
self.lang = self.ask(u"Choose your Language / Wähle deine Sprache", "en", langs)
gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation("setup", join(self.path, "locale"), languages=[self.lang, "en"], fallback=True)
translation.install(True)
#l10n Input shorthand for yes
self.yes = _("y")
#l10n Input shorthand for no
self.no = _("n")
def set_user(self):
gettext.setpaths(
[join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation(
"setup",
join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"],
fallback=True)
translation.install(True)
from pyload.database import DatabaseBackend
db = DatabaseBackend(None)
db.setup()
noaction = True
try:
while True:
print _("Select action")
print _("1 - Create/Edit user")
print _("2 - List users")
print _("3 - Remove user")
print _("4 - Quit")
action = raw_input("[1]/2/3/4: ")
if not action in ("1", "2", "3", "4"):
continue
elif action == "1":
print ""
username = self.ask(_("Username"), "User")
password = self.ask("", "", password=True)
db.addUser(username, password)
noaction = False
elif action == "2":
print ""
print _("Users")
print "-----"
users = db.getAllUserData()
noaction = False
for user in users.itervalues():
print user.name
print "-----"
print ""
elif action == "3":
print ""
username = self.ask(_("Username"), "")
if username:
db.removeUser(username)
noaction = False
elif action == "4":
db.syncSave()
break
finally:
if not noaction:
db.shutdown()
def set_user(self):
gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation("setup", join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"], fallback=True)
translation.install(True)
from pyload.database import DatabaseBackend
db = DatabaseBackend(None)
db.setup()
noaction = True
try:
while True:
print _("Select action")
print _("1 - Create/Edit user")
print _("2 - List users")
print _("3 - Remove user")
print _("4 - Quit")
action = raw_input("[1]/2/3/4: ")
if not action in ("1", "2", "3", "4"):
continue
elif action == "1":
print ""
username = self.ask(_("Username"), "User")
password = self.ask("", "", password=True)
db.addUser(username, password)
noaction = False
elif action == "2":
print ""
print _("Users")
print "-----"
users = db.getAllUserData()
noaction = False
for user in users.itervalues():
print user.name
print "-----"
print ""
elif action == "3":
print ""
username = self.ask(_("Username"), "")
if username:
db.removeUserByName(username)
noaction = False
elif action == "4":
db.syncSave()
break
finally:
if not noaction:
db.shutdown()
def ask_lang(self):
langs = self.config.getMetaData("general", "language").type.split(";")
self.lang = self.ask(u"Choose your Language / Wähle deine Sprache",
"en", langs)
gettext.setpaths(
[join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation("setup",
join(self.path, "locale"),
languages=[self.lang, "en"],
fallback=True)
translation.install(True)
#l10n Input shorthand for yes
self.yes = _("y")
#l10n Input shorthand for no
self.no = _("n")
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getPluginDump(pyfile.plugin) + "\n"
dump += self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"),
gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"),
gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf:
break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf: break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(
e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getPluginDump(pyfile.plugin) + "\n"
dump += self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def uploadContainer(self, filename, data):
"""Uploads and adds a container file to pyLoad.
:param filename: filename, extension is important so it can correctly decrypted
:param data: file content
"""
th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb")
th.write(str(data))
th.close()
return self.addPackage(th.name, [th.name])
def uploadContainer(self, filename, data):
"""Uploads and adds a container file to pyLoad.
:param filename: filename, extension is important so it can correctly decrypted
:param data: file content
"""
th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb")
th.write(str(data))
th.close()
return self.addPackage(th.name, [th.name])
def checkContainer(self, filename, data):
""" checks online status of urls and a submitted container file
:param filename: name of the file
:param data: file content
:return: :class:`OnlineCheck`
"""
th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb")
th.write(str(data))
th.close()
return self.checkLinks([th.name])
def conf_path(self, trans=False):
if trans:
translation = gettext.translation("setup", join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"], fallback=True)
translation.install(True)
print _("Setting new configpath, current configuration will not be transferred!")
path = self.ask(_("Config path"), abspath(""))
try:
path = join(pypath, path)
if not exists(path):
makedirs(path)
f = open(join(pypath, "pyload", "config", "configdir"), "wb")
f.write(path)
f.close()
print _("Config path changed, setup will now close, please restart to go on.")
print _("Press Enter to exit.")
raw_input()
exit()
except Exception, e:
print _("Setting config path failed: %s") % str(e)
def checkOnlineStatusContainer(self, urls, container, data):
""" checks online status of urls and a submitted container file
:param urls: list of urls
:param container: container file name
:param data: file content
:return: :class:`OnlineCheck`
"""
th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb")
th.write(str(data))
th.close()
urls.append(th.name)
return self.checkOnlineStatus(urls)
def checkContainer(self, filename, data):
""" checks online status of urls and a submitted container file
:param filename: name of the file
:param data: file content
:return: :class:`OnlineCheck`
"""
th = open(
join(self.core.config["general"]["download_folder"],
"tmp_" + filename), "wb")
th.write(str(data))
th.close()
return self.checkLinks([th.name])
def checkOnlineStatusContainer(self, urls, container, data):
""" checks online status of urls and a submitted container file
:param urls: list of urls
:param container: container file name
:param data: file content
:return: :class:`OnlineCheck`
"""
th = open(
join(self.core.config["general"]["download_folder"],
"tmp_" + container), "wb")
th.write(str(data))
th.close()
urls.append(th.name)
return self.checkOnlineStatus(urls)
def getLog(self, offset=0):
"""Returns most recent log entries.
:param offset: line offset
:return: List of log entries
"""
filename = join(self.core.config['log']['log_folder'], 'log.txt')
try:
fh = open(filename, "r")
lines = fh.readlines()
fh.close()
if offset >= len(lines):
return []
return lines[offset:]
except:
return ['No log available']
def set_user(self):
translation = gettext.translation("setup", join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"], fallback=True)
translation.install(True)
self.openDB()
try:
while True:
print _("Select action")
print _("1 - Create/Edit user")
print _("2 - List users")
print _("3 - Remove user")
print _("4 - Quit")
action = raw_input("[1]/2/3/4: ")
if not action in ("1", "2", "3", "4"):
continue
elif action == "1":
print ""
username = self.ask(_("Username"), "User")
password = self.ask("", "", password=True)
admin = self.ask("Admin?", self.yes, bool=True)
self.db.addUser(username, password, Role.Admin if admin else Role.User, int('1111111', 2))
elif action == "2":
print ""
print _("Users")
print "-----"
users = self.db.getAllUserData()
for user in users.itervalues():
print user.name
print "-----"
print ""
elif action == "3":
print ""
username = self.ask(_("Username"), "")
if username:
self.db.removeUserByName(username)
elif action == "4":
self.db.syncSave()
break
finally:
self.closeDB()
def decryptCaptcha(self,
url,
get={},
post={},
cookies=False,
forceUser=False,
imgtype='jpg',
result_type='textual'):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
:param forceUser: if True, ocr is not used
:param imgtype: Type of the Image
:param result_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
:return: result of decrypting
"""
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".", "")
temp_file = open(
join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)),
"wb")
temp_file.write(img)
temp_file.close()
name = "%sOCR" % self.__name__
has_plugin = name in self.core.pluginManager.getPlugins("internal")
if self.core.captcha:
OCR = self.core.pluginManager.loadClass("internal", name)
else:
OCR = None
if OCR and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
self.checkAbort()
ocr = OCR()
result = ocr.get_captcha(temp_file.name)
else:
task = self.im.createCaptchaTask(img, imgtype, temp_file.name,
self.__name__, result_type)
self.task = task
while task.isWaiting():
if self.abort():
self.im.removeTask(task)
raise Abort()
sleep(1)
#TODO task handling
self.im.removeTask(task)
if task.error and has_plugin: #ignore default error message since the user could use OCR
self.fail(
_("Pil and tesseract not installed and no Client connected for captcha decrypting"
))
elif task.error:
self.fail(task.error)
elif not task.result:
self.fail(_("No captcha result obtained in appropriate time."))
result = task.result
self.log.debug("Received captcha result: %s" % str(result))
if not self.core.debug:
try:
remove(temp_file.name)
except:
pass
return result
from hashlib import md5
from time import time
from shutil import move
import codecs
from nose.tools import nottest
from helper.Stubs import Core
from helper.PluginTester import PluginTester
from pyload.datatypes.PyFile import PyFile
from pyload.plugins.Base import Fail
from pyload.utils import accumulate
from pyload.utils.fs import save_join, join, exists, listdir, remove, stat
DL_DIR = join("Downloads", "tmp")
class HosterPluginTester(PluginTester):
files = {}
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)):
remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
#l10n yes, true,t are inputs for booleans with value true
if input.lower().strip() in [
self.yes, _("yes"),
_("true"),
_("t"), "yes"
]:
return True
#l10n no, false,f are inputs for booleans with value false
elif input.lower().strip() in [
self.no, _("no"),
_("false"), _("f"), "no"
]:
return False
else:
print _("Invalid Input")
continue
if not answers:
return input
else:
if input in answers:
return input
else:
print _("Invalid Input")
if __name__ == "__main__":
test = Setup(join(abspath(dirname(__file__)), ".."), None)
test.start()
from hashlib import md5
from time import time
from shutil import move
from nose.tools import nottest
from helper.Stubs import Core
from helper.parser import parse_config
from helper.PluginTester import PluginTester
from pyload.datatypes.PyFile import PyFile, statusMap
from pyload.plugins.Base import Fail
from pyload.utils import accumulate
from pyload.utils.fs import save_join, join, exists, listdir, remove, stat
DL_DIR = join("Downloads", "tmp")
class HosterPluginTester(PluginTester):
files = {}
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
input = raw_input(qst + " %s: " % info)
input = input.decode(self.stdin_encoding)
if input.strip() == "":
input = default
if bool:
#l10n yes, true,t are inputs for booleans with value true
if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]:
return True
#l10n no, false,f are inputs for booleans with value false
elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]:
return False
else:
print _("Invalid Input")
continue
if not answers:
return input
else:
if input in answers:
return input
else:
print _("Invalid Input")
if __name__ == "__main__":
test = Setup(join(abspath(dirname(__file__)), ".."), None)
test.start()
def getPath(self, name=""):
self.timestamp = time()
return join(self.m.getPackage(self.root).getPath(), self.folder, name)
def getPath(self, name=""):
return join(self.m.core.config["general"]["download_folder"], name)
def decryptCaptcha(
self, url, get={}, post={}, cookies=False, forceUser=False, imgtype="jpg", result_type="textual"
):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
:param forceUser: if True, ocr is not used
:param imgtype: Type of the Image
:param result_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
:return: result of decrypting
"""
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".", "")
temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb")
temp_file.write(img)
temp_file.close()
name = "%sOCR" % self.__name__
has_plugin = name in self.core.pluginManager.getPlugins("internal")
if self.core.captcha:
OCR = self.core.pluginManager.loadClass("internal", name)
else:
OCR = None
if OCR and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
self.checkAbort()
ocr = OCR()
result = ocr.get_captcha(temp_file.name)
else:
task = self.im.createCaptchaTask(img, imgtype, temp_file.name, self.__name__, result_type)
self.task = task
while task.isWaiting():
if self.abort():
self.im.removeTask(task)
raise Abort()
sleep(1)
# TODO task handling
self.im.removeTask(task)
if task.error and has_plugin: # ignore default error message since the user could use OCR
self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting"))
elif task.error:
self.fail(task.error)
elif not task.result:
self.fail(_("No captcha result obtained in appropriate time."))
result = task.result
self.log.debug("Received captcha result: %s" % str(result))
if not self.core.debug:
try:
remove(temp_file.name)
except:
pass
return result
class Hoster(Base):
"""
Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading.
"""
#: Class used to make requests with `self.load`
REQUEST_CLASS = DefaultRequest
#: Class used to make download
DOWNLOAD_CLASS = DefaultDownload
@staticmethod
def getInfo(urls):
"""This method is used to retrieve the online status of files for hoster plugins.
:param urls: List of urls
:return: yield list of :class:`LinkStatus` as result
"""
pass
__type__ = "hoster"
def __init__(self, pyfile):
# TODO: pyfile.owner, but it's not correct yet
Base.__init__(self, pyfile.m.core)
self.wantReconnect = False
#: enables simultaneous processing of multiple downloads
self.limitDL = 0
#: chunk limit
self.chunkLimit = 1
#: enables resume (will be ignored if server dont accept chunks)
self.resumeDownload = False
#: plugin is waiting
self.waiting = False
self.ocr = None #captcha reader instance
#: account handler instance, see :py:class:`Account`
self.account = self.core.accountManager.selectAccount(
self.__name__, self.owner)
#: premium status
self.premium = False
if self.account:
#: Request instance bound to account
self.req = self.account.getAccountRequest()
# Default: -1, True, True
self.chunkLimit, self.limitDL, self.resumeDownload = self.account.getDownloadSettings(
)
self.premium = self.account.isPremium()
else:
self.req = self.core.requestFactory.getRequest(
klass=self.REQUEST_CLASS)
#: Will hold the download class
self.dl = None
#: associated pyfile instance, see `PyFile`
self.pyfile = pyfile
self.thread = None # holds thread in future
#: location where the last call to download was saved
self.lastDownload = ""
#: re match of the last call to `checkDownload`
self.lastCheck = None
self.retries = 0 # amount of retries already made
self.html = None # some plugins store html code here
self.init()
@property
def user(self):
self.logDebug(
"Deprecated usage of self.user -> use self.account.loginname")
if self.account:
return self.account.loginname
def getMultiDL(self):
return self.limitDL <= 0
def setMultiDL(self, val):
self.limitDL = 0 if val else 1
#: virtual attribute using self.limitDL on behind
multiDL = property(getMultiDL, setMultiDL)
def getChunkCount(self):
if self.chunkLimit <= 0:
return self.config["download"]["chunks"]
return min(self.config["download"]["chunks"], self.chunkLimit)
def getDownloadLimit(self):
if self.account:
limit = self.account.options.get("limitDL", 0)
if limit == "": limit = 0
if self.limitDL > 0: # a limit is already set, we use the minimum
return min(int(limit), self.limitDL)
else:
return int(limit)
else:
return self.limitDL
def __call__(self):
return self.__name__
def init(self):
"""initialize the plugin (in addition to `__init__`)"""
pass
def setup(self):
""" setup for environment and other things, called before downloading (possibly more than one time)"""
pass
def preprocessing(self, thread):
""" handles important things to do before starting """
self.thread = thread
if self.account:
# will force a re-login or reload of account info if necessary
self.account.getAccountInfo()
else:
self.req.reset()
self.setup()
self.pyfile.setStatus("starting")
return self.process(self.pyfile)
def process(self, pyfile):
"""the 'main' method of every plugin, you **have to** overwrite it"""
raise NotImplementedError
def abort(self):
return self.pyfile.abort
def resetAccount(self):
""" don't use account and retry download """
self.account = None
self.req = self.core.requestFactory.getRequest(self.__name__)
self.retry()
def checksum(self, local_file=None):
"""
return codes:
0 - checksum ok
1 - checksum wrong
5 - can't get checksum
10 - not implemented
20 - unknown error
"""
#@TODO checksum check addon
return True, 10
def setWait(self, seconds, reconnect=None):
"""Set a specific wait time later used with `wait`
:param seconds: wait time in seconds
:param reconnect: True if a reconnect would avoid wait time
"""
if reconnect is not None:
self.wantReconnect = reconnect
self.pyfile.waitUntil = time() + int(seconds)
def wait(self, seconds=None, reconnect=None):
""" Waits the time previously set or use these from arguments. See `setWait`
"""
if seconds is not None:
self.setWait(seconds, reconnect)
self._wait()
def _wait(self):
self.waiting = True
self.pyfile.setStatus("waiting")
while self.pyfile.waitUntil > time():
self.thread.m.reconnecting.wait(2)
self.checkAbort()
if self.thread.m.reconnecting.isSet():
self.waiting = False
self.wantReconnect = False
raise Reconnect
self.waiting = False
self.pyfile.setStatus("starting")
def offline(self):
""" fail and indicate file is offline """
raise Fail("offline")
def tempOffline(self):
""" fail and indicates file ist temporary offline, the core may take consequences """
raise Fail("temp. offline")
def retry(self,
max_tries=3,
wait_time=1,
reason="",
backoff=lambda x, y: x):
"""Retries and begin again from the beginning
:param max_tries: number of maximum retries
:param wait_time: time to wait in seconds
:param reason: reason for retrying, will be passed to fail if max_tries reached
:param backoff: Function to backoff the wait time, takes initial time and number of retry as argument.
defaults to no backoff / fixed wait time
"""
if 0 < max_tries <= self.retries:
if not reason: reason = "Max retries reached"
raise Fail(reason)
self.wantReconnect = False
self.retries += 1
self.setWait(backoff(wait_time, self.retries))
self.wait()
raise Retry(reason)
def download(self,
url,
get={},
post={},
ref=True,
cookies=True,
disposition=False):
"""Downloads the content at url to download folder
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.checkAbort()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"],
8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
# convert back to unicode
location = fs_decode(location)
name = self.pyfile.name
filename = join(location, name)
self.core.addonManager.dispatchEvent("download:start", self.pyfile,
url, filename)
# Create the class used for downloading
self.dl = self.core.requestFactory.getDownloadRequest(
self.req, self.DOWNLOAD_CLASS)
try:
# TODO: hardcoded arguments
newname = self.dl.download(url,
filename,
get=get,
post=post,
referer=ref,
chunks=self.getChunkCount(),
resume=self.resumeDownload,
cookies=cookies,
disposition=disposition)
finally:
self.dl.close()
self.pyfile.size = self.dl.size
if disposition and newname and newname != name: #triple check, just to be sure
self.log.info("%(name)s saved as %(newname)s" % {
"name": name,
"newname": newname
})
self.pyfile.name = newname
filename = join(location, newname)
fs_filename = fs_encode(filename)
if self.core.config["permission"]["change_file"]:
chmod(fs_filename, int(self.core.config["permission"]["file"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(fs_filename, uid, gid)
except Exception, e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
from hashlib import md5
from time import time
from shutil import move
import codecs
from nose.tools import nottest
from helper.Stubs import Core
from helper.PluginTester import PluginTester
from pyload.datatypes.PyFile import PyFile
from pyload.plugins.Base import Fail
from pyload.utils import accumulate
from pyload.utils.fs import save_join, join, exists, listdir, remove, stat
DL_DIR = join("Downloads", "tmp")
class HosterPluginTester(PluginTester):
files = {}
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
