def checkForSameFiles(self, starting=False):
""" checks if same file was/is downloaded within same package
:param starting: indicates that the current download is going to start
:raises SkipDownload:
"""
pack = self.pyfile.package()
for pyfile in self.core.files.cachedFiles():
if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
if pyfile.status in (0, 12): #finished or downloading
raise SkipDownload(pyfile.pluginname)
elif pyfile.status in (
5, 7) and starting: #a download is waiting/starting and was apparently started before
raise SkipDownload(pyfile.pluginname)
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, pack.folder, self.pyfile.name)
if starting and self.core.config['download']['skip_existing'] and exists(location):
size = os.stat(location).st_size
if size >= self.pyfile.size:
raise SkipDownload("File exists.")
pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
if pyfile:
if exists(location):
raise SkipDownload(pyfile[0])
self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
def checkForSameFiles(self, starting=False):
""" checks if same file was/is downloaded within same package
:param starting: indicates that the current download is going to start
:raises SkipDownload:
"""
pack = self.pyfile.package()
for pyfile in self.core.files.cachedFiles():
if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
if pyfile.status in (0, 12): #finished or downloading
raise SkipDownload(pyfile.pluginname)
elif pyfile.status in (
5, 7) and starting: #a download is waiting/starting and was apparently started before
raise SkipDownload(pyfile.pluginname)
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, pack.folder, self.pyfile.name)
if starting and self.core.config['download']['skip_existing'] and exists(location):
size = os.stat(location).st_size
if size >= self.pyfile.size:
raise SkipDownload("File exists.")
pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
if pyfile:
if exists(location):
raise SkipDownload(pyfile[0])
self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
def run(self):
self.running = True
import webinterface
global webinterface
if self.https:
if not exists(self.cert) or not exists(self.key):
log.warning(_("SSL certificates not found."))
self.https = False
prefer = None
# These cases covers all settings
if self.server == "threaded":
prefer = "threaded"
elif self.server == "fastcgi":
prefer = "flup"
elif self.server == "fallback":
prefer = "wsgiref"
server = self.select_server(prefer)
try:
self.start_server(server)
except Exception, e:
log.error(_("Failed starting webserver: " + e.message))
self.error = e
if core:
core.print_exc()
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def isWSSecure(self):
# needs to use TLS when either requested or webUI is also using encryption
if not self.core.config['ssl']['activated'] or self.core.config['webUI']['https']:
return False
if not exists(self.core.config['ssl']['cert']) or not exists(self.core.config['ssl']['key']):
self.core.log.warning(_('SSL key or certificate not found'))
return False
return True
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
"""Downloads the content at url to download folder
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.checkAbort()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(_("Setting User and Group failed: %s") % str(e))
def download(self,
url,
get={},
post={},
ref=True,
cookies=True,
disposition=False):
"""Downloads the content at url to download folder
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.checkAbort()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"],
8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
def conf_path(self, trans=False):
if trans:
gettext.setpaths(
[join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation(
"setup",
join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"],
fallback=True)
translation.install(True)
print _(
"Setting new configpath, current configuration will not be transferred!"
)
path = self.ask(_("Config path"), abspath(""))
try:
path = join(pypath, path)
if not exists(path):
makedirs(path)
f = open(join(pypath, "pyload", "config", "configdir"), "wb")
f.write(path)
f.close()
print _(
"Config path changed, setup will now close, please restart to go on."
)
print _("Press Enter to exit.")
raw_input()
exit()
except Exception, e:
print _("Setting config path failed: %s") % str(e)
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
"""Load content at url and returns it
:param url: url as string
:param get: GET as dict
:param post: POST as dict, list or string
:param ref: Set HTTP_REFERER header
:param cookies: use saved cookies
:param just_header: if True only the header will be retrieved and returned as dict
:param decode: Whether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.")
self.checkAbort()
res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
if not exists(join("tmp", self.__name__)):
makedirs(join("tmp", self.__name__))
f = open(
join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
, "wb")
del frame # delete the frame or it wont be cleaned
try:
tmp = res.encode("utf8")
except:
tmp = res
f.write(tmp)
f.close()
if just_header:
#parse header
header = {"code": self.req.code}
for line in res.splitlines():
line = line.strip()
if not line or ":" not in line: continue
key, none, value = line.partition(":")
key = key.lower().strip()
value = value.strip()
if key in header:
if type(header[key]) == list:
header[key].append(value)
else:
header[key] = [header[key], value]
else:
header[key] = value
res = header
return res
def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf:
break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf: break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(
e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def getLocalContent(self, urls):
"""Load files from disk and separate to file content and url list
:param urls:
:return: list of (filename, content), remote urls
"""
content = []
# do nothing if no decryptFile method
if hasattr(self.__class__, "decryptFile"):
remote = []
for url in urls:
path = None
if url.startswith("http"): # skip urls directly
pass
elif url.startswith(self.CONTENT_PREFIX):
path = url
elif exists(url):
path = url
elif exists(self.core.path(url)):
path = self.core.path(url)
if path:
try:
if path.startswith(self.CONTENT_PREFIX):
content.append(
("", path[len(self.CONTENT_PREFIX)]))
else:
f = open(fs_encode(path), "rb")
content.append((f.name, f.read()))
f.close()
except IOError, e:
self.logError("IOError", e)
else:
remote.append(url)
#swap filtered url list
urls = remote
def run(self):
self.running = True
import webinterface
global webinterface
if self.https:
if not exists(self.cert) or not exists(self.key):
log.warning(_("SSL certificates not found."))
self.https = False
if webinterface.UNAVAILALBE:
log.warning(_("WebUI built is not available"))
elif webinterface.APP_PATH == "app":
log.info(_("Running webUI in development mode"))
prefer = None
# These cases covers all settings
if self.server == "threaded":
prefer = "threaded"
elif self.server == "fastcgi":
prefer = "flup"
elif self.server == "fallback":
prefer = "wsgiref"
server = self.select_server(prefer)
try:
self.start_server(server)
except Exception, e:
log.error(_("Failed starting webserver: " + e.message))
self.error = e
if core:
core.print_exc()
def run(self):
self.running = True
import webinterface
global webinterface
if self.https:
if not exists(self.cert) or not exists(self.key):
log.warning(_("SSL certificates not found."))
self.https = False
if webinterface.UNAVAILALBE:
log.warning(_("WebUI built is not available"))
elif webinterface.APP_PATH == "app":
log.info(_("Running webUI in development mode"))
prefer = None
# These cases covers all settings
if self.server == "threaded":
prefer = "threaded"
elif self.server == "fastcgi":
prefer = "flup"
elif self.server == "fallback":
prefer = "wsgiref"
server = self.select_server(prefer)
try:
self.start_server(server)
except Exception, e:
log.error(_("Failed starting webserver: " + e.message))
self.error = e
if core:
core.print_exc()
def getLocalContent(self, urls):
"""Load files from disk and separate to file content and url list
:param urls:
:return: list of (filename, content), remote urls
"""
content = []
# do nothing if no decryptFile method
if hasattr(self.__class__, "decryptFile"):
remote = []
for url in urls:
path = None
if url.startswith("http"): # skip urls directly
pass
elif url.startswith(self.CONTENT_PREFIX):
path = url
elif exists(url):
path = url
elif exists(self.core.path(url)):
path = self.core.path(url)
if path:
try:
if path.startswith(self.CONTENT_PREFIX):
content.append(("", path[len(self.CONTENT_PREFIX)]))
else:
f = open(fs_encode(path), "rb")
content.append((f.name, f.read()))
f.close()
except IOError, e:
self.logError("IOError", e)
else:
remote.append(url)
#swap filtered url list
urls = remote
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getPluginDump(pyfile.plugin) + "\n"
dump += self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"),
gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"),
gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def checkDownload(self,
rules,
api_size=0,
max_size=50000,
delete=True,
read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param api_size: expected file size
:param max_size: if the file is larger then it wont be checked
:param delete: delete if matched
:param read_size: amount of bytes to read from files larger then max_size
:return: dictionary key of the first rule that matched
"""
lastDownload = fs_encode(self.lastDownload)
if not exists(lastDownload): return None
size = stat(lastDownload)
size = size.st_size
if api_size and api_size <= size:
return None
elif size > max_size and not read_size:
return None
self.log.debug("Download Check triggered")
f = open(lastDownload, "rb")
content = f.read(read_size if read_size else -1)
f.close()
#produces encoding errors, better log to other file in the future?
#self.log.debug("Content: %s" % content)
for name, rule in rules.iteritems():
if type(rule) in (str, unicode):
if rule in content:
if delete:
remove(lastDownload)
return name
elif hasattr(rule, "search"):
m = rule.search(content)
if m:
if delete:
remove(lastDownload)
self.lastCheck = m
return name
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getPluginDump(pyfile.plugin) + "\n"
dump += self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def conf_path(self, trans=False):
if trans:
translation = gettext.translation("setup", join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"], fallback=True)
translation.install(True)
print _("Setting new configpath, current configuration will not be transferred!")
path = self.ask(_("Config path"), abspath(""))
try:
path = join(pypath, path)
if not exists(path):
makedirs(path)
f = open(join(pypath, "pyload", "config", "configdir"), "wb")
f.write(path)
f.close()
print _("Config path changed, setup will now close, please restart to go on.")
print _("Press Enter to exit.")
raw_input()
exit()
except Exception, e:
print _("Setting config path failed: %s") % str(e)
def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param api_size: expected file size
:param max_size: if the file is larger then it wont be checked
:param delete: delete if matched
:param read_size: amount of bytes to read from files larger then max_size
:return: dictionary key of the first rule that matched
"""
lastDownload = fs_encode(self.lastDownload)
if not exists(lastDownload):
return None
size = stat(lastDownload)
size = size.st_size
if api_size and api_size <= size:
return None
elif size > max_size and not read_size:
return None
self.log.debug("Download Check triggered")
f = open(lastDownload, "rb")
content = f.read(read_size if read_size else -1)
f.close()
# produces encoding errors, better log to other file in the future?
# self.log.debug("Content: %s" % content)
for name, rule in rules.iteritems():
if type(rule) in (str, unicode):
if rule in content:
if delete:
remove(lastDownload)
return name
elif hasattr(rule, "search"):
m = rule.search(content)
if m:
if delete:
remove(lastDownload)
self.lastCheck = m
return name
