def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
"""Load content at url and returns it
:param url: url as string
:param get: GET as dict
:param post: POST as dict, list or string
:param ref: Set HTTP_REFERER header
:param cookies: use saved cookies
:param just_header: if True only the header will be retrieved and returned as dict
:param decode: Whether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.")
self.checkAbort()
res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
if not exists(join("tmp", self.__name__)):
makedirs(join("tmp", self.__name__))
f = open(
join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
, "wb")
del frame # delete the frame or it wont be cleaned
try:
tmp = res.encode("utf8")
except:
tmp = res
f.write(tmp)
f.close()
if just_header:
#parse header
header = {"code": self.req.code}
for line in res.splitlines():
line = line.strip()
if not line or ":" not in line: continue
key, none, value = line.partition(":")
key = key.lower().strip()
value = value.strip()
if key in header:
if type(header[key]) == list:
header[key].append(value)
else:
header[key] = [header[key], value]
else:
header[key] = value
res = header
return res
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def ask_lang(self):
langs = self.config.getMetaData("general", "language").type.split(";")
self.lang = self.ask(u"Choose your Language / Wähle deine Sprache", "en", langs)
gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation("setup", join(self.path, "locale"), languages=[self.lang, "en"], fallback=True)
translation.install(True)
#l10n Input shorthand for yes
self.yes = _("y")
#l10n Input shorthand for no
self.no = _("n")
def set_user(self):
gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation("setup", join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"], fallback=True)
translation.install(True)
from module.database import DatabaseBackend
db = DatabaseBackend(None)
db.setup()
noaction = True
try:
while True:
print _("Select action")
print _("1 - Create/Edit user")
print _("2 - List users")
print _("3 - Remove user")
print _("4 - Quit")
action = raw_input("[1]/2/3/4: ")
if not action in ("1", "2", "3", "4"):
continue
elif action == "1":
print ""
username = self.ask(_("Username"), "User")
password = self.ask("", "", password=True)
db.addUser(username, password)
noaction = False
elif action == "2":
print ""
print _("Users")
print "-----"
users = db.getAllUserData()
noaction = False
for user in users.itervalues():
print user.name
print "-----"
print ""
elif action == "3":
print ""
username = self.ask(_("Username"), "")
if username:
db.removeUser(username)
noaction = False
elif action == "4":
db.syncSave()
break
finally:
if not noaction:
db.shutdown()
def setup(self):
self.scripts = {}
folders = ['download_preparing', 'download_finished', 'package_finished',
'before_reconnect', 'after_reconnect', 'unrar_finished',
'all_dls_finished', 'all_dls_processed']
for folder in folders:
self.scripts[folder] = []
self.initPluginType(folder, join(pypath, 'scripts', folder))
self.initPluginType(folder, join('scripts', folder))
for script_type, names in self.scripts.iteritems():
if names:
self.logInfo((_("Installed scripts for %s: ") % script_type ) + ", ".join([basename(x) for x in names]))
def initPluginType(self, folder, path):
if not exists(path):
try:
makedirs(path)
except:
self.logDebug("Script folder %s not created" % folder)
return
for f in listdir(path):
if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"):
continue
if not access(join(path, f), X_OK):
self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f))
self.scripts[folder].append(join(path, f))
def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf: break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def uploadContainer(self, filename, data):
"""Uploads and adds a container file to pyLoad.
:param filename: filename, extension is important so it can correctly decrypted
:param data: file content
"""
th = open(join(self.core.config["general"]["download_folder"], "tmp_" + filename), "wb")
th.write(str(data))
th.close()
return self.addPackage(th.name, [th.name])
def checkOnlineStatusContainer(self, urls, container, data):
""" checks online status of urls and a submitted container file
:param urls: list of urls
:param container: container file name
:param data: file content
:return: :class:`OnlineCheck`
"""
th = open(join(self.core.config["general"]["download_folder"], "tmp_" + container), "wb")
th.write(str(data))
th.close()
urls.append(th.name)
return self.checkOnlineStatus(urls)
def conf_path(self, trans=False):
if trans:
gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation("setup", join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"], fallback=True)
translation.install(True)
print _("Setting new configpath, current configuration will not be transferred!")
path = self.ask(_("Configpath"), abspath(""))
try:
path = join(pypath, path)
if not exists(path):
makedirs(path)
f = open(join(pypath, "module", "config", "configdir"), "wb")
f.write(path)
f.close()
print _("Configpath changed, setup will now close, please restart to go on.")
print _("Press Enter to exit.")
raw_input()
exit()
except Exception, e:
print _("Setting config path failed: %s") % str(e)
def getLog(self, offset=0):
"""Returns most recent log entries.
:param offset: line offset
:return: List of log entries
"""
filename = join(self.core.config['log']['log_folder'], 'log.txt')
try:
fh = open(filename, "r")
lines = fh.readlines()
fh.close()
if offset >= len(lines):
return []
return lines[offset:]
except:
return ['No log available']
class Hoster(Base):
"""
Base plugin for hoster plugin. Overwrite getInfo for online status retrieval, process for downloading.
"""
@staticmethod
def getInfo(urls):
"""This method is used to retrieve the online status of files for hoster plugins.
It has to *yield* list of tuples with the result in this format (name, size, status, url),
where status is one of API pyfile statuses.
:param urls: List of urls
:return: yield list of tuple with results (name, size, status, url)
"""
pass
def __init__(self, pyfile):
Base.__init__(self, pyfile.m.core)
self.wantReconnect = False
#: enables simultaneous processing of multiple downloads
self.limitDL = 0
#: chunk limit
self.chunkLimit = 1
#: enables resume (will be ignored if server dont accept chunks)
self.resumeDownload = False
#: plugin is waiting
self.waiting = False
self.ocr = None #captcha reader instance
#: account handler instance, see :py:class:`Account`
self.account = self.core.accountManager.getAccountForPlugin(
self.__name__)
#: premium status
self.premium = False
#: username/login
self.user = None
if self.account and not self.account.isUsable(): self.account = None
if self.account:
self.user = self.account.loginname
#: Browser instance, see `network.Browser`
self.req = self.account.getAccountRequest()
# Default: -1, True, True
self.chunkLimit, self.limitDL, self.resumeDownload = self.account.getDownloadSettings(
)
self.premium = self.account.isPremium()
else:
self.req = self.core.requestFactory.getRequest(self.__name__)
#: associated pyfile instance, see `PyFile`
self.pyfile = pyfile
self.thread = None # holds thread in future
#: location where the last call to download was saved
self.lastDownload = ""
#: re match of the last call to `checkDownload`
self.lastCheck = None
#: js engine, see `JsEngine`
self.js = self.core.js
self.retries = 0 # amount of retries already made
self.html = None # some plugins store html code here
self.init()
def getMultiDL(self):
return self.limitDL <= 0
def setMultiDL(self, val):
self.limitDL = 0 if val else 1
#: virtual attribute using self.limitDL on behind
multiDL = property(getMultiDL, setMultiDL)
def getChunkCount(self):
if self.chunkLimit <= 0:
return self.config["download"]["chunks"]
return min(self.config["download"]["chunks"], self.chunkLimit)
def getDownloadLimit(self):
if self.account:
limit = self.account.options.get("limitDL", 0)
if limit == "": limit = 0
if self.limitDL > 0: # a limit is already set, we use the minimum
return min(int(limit), self.limitDL)
else:
return int(limit)
else:
return self.limitDL
def __call__(self):
return self.__name__
def init(self):
"""initialize the plugin (in addition to `__init__`)"""
pass
def setup(self):
""" setup for environment and other things, called before downloading (possibly more than one time)"""
pass
def preprocessing(self, thread):
""" handles important things to do before starting """
self.thread = thread
if self.account:
# will force a re-login or reload of account info if necessary
self.account.getAccountInfo()
else:
self.req.clearCookies()
self.setup()
self.pyfile.setStatus("starting")
return self.process(self.pyfile)
def process(self, pyfile):
"""the 'main' method of every plugin, you **have to** overwrite it"""
raise NotImplementedError
def abort(self):
return self.pyfile.abort
def resetAccount(self):
""" don't use account and retry download """
self.account = None
self.req = self.core.requestFactory.getRequest(self.__name__)
self.retry()
def checksum(self, local_file=None):
"""
return codes:
0 - checksum ok
1 - checksum wrong
5 - can't get checksum
10 - not implemented
20 - unknown error
"""
#@TODO checksum check addon
return True, 10
def setWait(self, seconds, reconnect=False):
"""Set a specific wait time later used with `wait`
:param seconds: wait time in seconds
:param reconnect: True if a reconnect would avoid wait time
"""
if reconnect:
self.wantReconnect = True
self.pyfile.waitUntil = time() + int(seconds)
def wait(self):
""" waits the time previously set """
self.waiting = True
self.pyfile.setStatus("waiting")
while self.pyfile.waitUntil > time():
self.thread.m.reconnecting.wait(2)
self.checkAbort()
if self.thread.m.reconnecting.isSet():
self.waiting = False
self.wantReconnect = False
raise Reconnect
self.waiting = False
self.pyfile.setStatus("starting")
def offline(self):
""" fail and indicate file is offline """
raise Fail("offline")
def tempOffline(self):
""" fail and indicates file ist temporary offline, the core may take consequences """
raise Fail("temp. offline")
def retry(self, max_tries=3, wait_time=1, reason=""):
"""Retries and begin again from the beginning
:param max_tries: number of maximum retries
:param wait_time: time to wait in seconds
:param reason: reason for retrying, will be passed to fail if max_tries reached
"""
if 0 < max_tries <= self.retries:
if not reason: reason = "Max retries reached"
raise Fail(reason)
self.wantReconnect = False
self.setWait(wait_time)
self.wait()
self.retries += 1
raise Retry(reason)
def download(self,
url,
get={},
post={},
ref=True,
cookies=True,
disposition=False):
"""Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.checkAbort()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"],
8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
# convert back to unicode
location = fs_decode(location)
name = save_filename(self.pyfile.name)
filename = join(location, name)
self.core.addonManager.dispatchEvent("download:start", self.pyfile,
url, filename)
try:
newname = self.req.httpDownload(url,
filename,
get=get,
post=post,
ref=ref,
cookies=cookies,
chunks=self.getChunkCount(),
resume=self.resumeDownload,
disposition=disposition)
finally:
self.pyfile.size = self.req.size
if disposition and newname and newname != name: #triple check, just to be sure
self.log.info("%(name)s saved as %(newname)s" % {
"name": name,
"newname": newname
})
self.pyfile.name = newname
filename = join(location, newname)
fs_filename = fs_encode(filename)
if self.core.config["permission"]["change_file"]:
chmod(fs_filename, int(self.core.config["permission"]["file"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(fs_filename, uid, gid)
except Exception, e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
#l10n yes, true,t are inputs for booleans with value true
if input.lower().strip() in [
self.yes, _("yes"),
_("true"),
_("t"), "yes"
]:
return True
#l10n no, false,f are inputs for booleans with value false
elif input.lower().strip() in [
self.no, _("no"),
_("false"), _("f"), "no"
]:
return False
else:
print _("Invalid Input")
continue
if not answers:
return input
else:
if input in answers:
return input
else:
print _("Invalid Input")
if __name__ == "__main__":
test = Setup(join(abspath(dirname(__file__)), ".."), None)
test.start()
def decryptCaptcha(self,
url,
get={},
post={},
cookies=False,
forceUser=False,
imgtype='jpg',
result_type='textual'):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
:param forceUser: if True, ocr is not used
:param imgtype: Type of the Image
:param result_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
:return: result of decrypting
"""
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".", "")
temp_file = open(
join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)),
"wb")
temp_file.write(img)
temp_file.close()
name = "%sOCR" % self.__name__
has_plugin = name in self.core.pluginManager.getPlugins("internal")
if self.core.captcha:
OCR = self.core.pluginManager.loadClass("internal", name)
else:
OCR = None
if OCR and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
self.checkAbort()
ocr = OCR()
result = ocr.get_captcha(temp_file.name)
else:
task = self.im.createCaptchaTask(img, imgtype, temp_file.name,
self.__name__, result_type)
self.task = task
while task.isWaiting():
if self.abort():
self.im.removeTask(task)
raise Abort()
sleep(1)
#TODO task handling
self.im.removeTask(task)
if task.error and has_plugin: #ignore default error message since the user could use OCR
self.fail(
_("Pil and tesseract not installed and no Client connected for captcha decrypting"
))
elif task.error:
self.fail(task.error)
elif not task.result:
self.fail(_("No captcha result obtained in appropriate time."))
result = task.result
self.log.debug("Received captcha result: %s" % str(result))
if not self.core.debug:
try:
remove(temp_file.name)
except:
pass
return result
def getPath(self, name=""):
return join(self.m.core.config["general"]["download_folder"], name)
def getPath(self, name=""):
return join(self.m.core.config["general"]["download_folder"], name)
def getPath(self, name=""):
self.timestamp = time()
return join(self.m.getPackage(self.root).getPath(), self.folder, name)
input = raw_input(qst + " %s: " % info)
input = input.decode(self.stdin_encoding)
if input.strip() == "":
input = default
if bool:
#l10n yes, true,t are inputs for booleans with value true
if input.lower().strip() in [self.yes, _("yes"), _("true"), _("t"), "yes"]:
return True
#l10n no, false,f are inputs for booleans with value false
elif input.lower().strip() in [self.no, _("no"), _("false"), _("f"), "no"]:
return False
else:
print _("Invalid Input")
continue
if not answers:
return input
else:
if input in answers:
return input
else:
print _("Invalid Input")
if __name__ == "__main__":
test = Setup(join(abspath(dirname(__file__)), ".."), None)
test.start()
def decryptCaptcha(self, url, get={}, post={}, cookies=False, forceUser=False, imgtype='jpg',
result_type='textual'):
""" Loads a captcha and decrypts it with ocr, plugin, user input
:param url: url of captcha image
:param get: get part for request
:param post: post part for request
:param cookies: True if cookies should be enabled
:param forceUser: if True, ocr is not used
:param imgtype: Type of the Image
:param result_type: 'textual' if text is written on the captcha\
or 'positional' for captcha where the user have to click\
on a specific region on the captcha
:return: result of decrypting
"""
img = self.load(url, get=get, post=post, cookies=cookies)
id = ("%.2f" % time())[-6:].replace(".", "")
temp_file = open(join("tmp", "tmpCaptcha_%s_%s.%s" % (self.__name__, id, imgtype)), "wb")
temp_file.write(img)
temp_file.close()
name = "%sOCR" % self.__name__
has_plugin = name in self.core.pluginManager.getPlugins("internal")
if self.core.captcha:
OCR = self.core.pluginManager.loadClass("internal", name)
else:
OCR = None
if OCR and not forceUser:
sleep(randint(3000, 5000) / 1000.0)
self.checkAbort()
ocr = OCR()
result = ocr.get_captcha(temp_file.name)
else:
task = self.im.createCaptchaTask(img, imgtype, temp_file.name, self.__name__, result_type)
self.task = task
while task.isWaiting():
if self.abort():
self.im.removeTask(task)
raise Abort()
sleep(1)
#TODO task handling
self.im.removeTask(task)
if task.error and has_plugin: #ignore default error message since the user could use OCR
self.fail(_("Pil and tesseract not installed and no Client connected for captcha decrypting"))
elif task.error:
self.fail(task.error)
elif not task.result:
self.fail(_("No captcha result obtained in appropriate time."))
result = task.result
self.log.debug("Received captcha result: %s" % str(result))
if not self.core.debug:
try:
remove(temp_file.name)
except:
pass
return result
from hashlib import md5
from time import time
from shutil import move
import codecs
from nose.tools import nottest
from helper.Stubs import Core
from helper.PluginTester import PluginTester
from module.datatypes.PyFile import PyFile
from module.plugins.Base import Fail
from module.utils import accumulate
from module.utils.fs import save_join, join, exists, listdir, remove, stat
DL_DIR = join("Downloads", "tmp")
class HosterPluginTester(PluginTester):
files = {}
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def load(self,
url,
get={},
post={},
ref=True,
cookies=True,
just_header=False,
decode=False):
"""Load content at url and returns it
:param url: url as string
:param get: GET as dict
:param post: POST as dict, list or string
:param ref: Set HTTP_REFERER header
:param cookies: use saved cookies
:param just_header: if True only the header will be retrieved and returned as dict
:param decode: Whether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
if not hasattr(self, "req"):
raise Exception("Plugin type does not have Request attribute.")
self.checkAbort()
res = self.req.load(url,
get,
post,
ref,
cookies,
just_header,
decode=decode)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
if not exists(join("tmp", self.__name__)):
makedirs(join("tmp", self.__name__))
f = open(
join(
"tmp", self.__name__, "%s_line%s.dump.html" %
(frame.f_back.f_code.co_name, frame.f_back.f_lineno)),
"wb")
del frame # delete the frame or it wont be cleaned
try:
tmp = res.encode("utf8")
except:
tmp = res
f.write(tmp)
f.close()
if just_header:
#parse header
header = {"code": self.req.code}
for line in res.splitlines():
line = line.strip()
if not line or ":" not in line: continue
key, none, value = line.partition(":")
key = key.lower().strip()
value = value.strip()
if key in header:
if type(header[key]) == list:
header[key].append(value)
else:
header[key] = [header[key], value]
else:
header[key] = value
res = header
return res
