def run(self):
self.running = True
import webinterface
global webinterface
if self.https:
if not exists(self.cert) or not exists(self.key):
log.warning(_("SSL certificates not found."))
self.https = False
prefer = None
# These cases covers all settings
if self.server == "threaded":
prefer = "threaded"
elif self.server == "fastcgi":
prefer = "flup"
elif self.server == "fallback":
prefer = "wsgiref"
server = self.select_server(prefer)
try:
self.start_server(server)
except Exception, e:
log.error(_("Failed starting webserver: " + e.message))
self.error = e
if core:
core.print_exc()
def run(self):
self.running = True
import webinterface
global webinterface
if self.https:
if not exists(self.cert) or not exists(self.key):
log.warning(_("SSL certificates not found."))
self.https = False
prefer = None
# These cases covers all settings
if self.server == "threaded":
prefer = "threaded"
elif self.server == "fastcgi":
prefer = "flup"
elif self.server == "fallback":
prefer = "wsgiref"
server = self.select_server(prefer)
try:
self.start_server(server)
except Exception, e:
log.error(_("Failed starting webserver: " + e.message))
self.error = e
if core:
core.print_exc()
def run(self):
self.running = True
import webinterface
global webinterface
if self.https:
if not exists(self.cert) or not exists(self.key):
log.warning(_("SSL certificates not found."))
self.https = False
if self.server == "fastcgi":
try:
import flup
except:
log.warning(_("Can't use %(server)s, python-flup is not installed!") % {
"server": self.server})
self.server = "builtin"
elif self.server == "lightweight":
try:
import bjoern
except Exception, e:
log.error(_("Error importing lightweight server: %s") % e)
log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern"))
log.warning(_("Copy the boern.so to module/lib folder or use setup.py install"))
log.warning(_("Of course you need to be familiar with linux and know how to compile software"))
self.server = "builtin"
def run(self):
self.running = True
import webinterface
global webinterface
if self.https:
if not exists(self.cert) or not exists(self.key):
log.warning(_("SSL certificates not found."))
self.https = False
if self.server == "fastcgi":
try:
import flup
except:
log.warning(_("Can't use %(server)s, python-flup is not installed!") % {"server": self.server})
self.server = "builtin"
elif self.server == "lightweight":
try:
import bjoern
except Exception, e:
log.error(_("Error importing lightweight server: %s") % e)
log.warning(_("You need to download and compile bjoern, https://github.com/jonashaag/bjoern"))
log.warning(_("Copy the boern.so to module/lib folder or use setup.py install"))
log.warning(_("Of course you need to be familiar with linux and know how to compile software"))
self.server = "builtin"
def checkForSameFiles(self, starting=False):
""" checks if same file was/is downloaded within same package
:param starting: indicates that the current download is going to start
:raises SkipDownload:
"""
pack = self.pyfile.package()
for pyfile in self.core.files.cachedFiles():
if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
if pyfile.status in (0, 12): #finished or downloading
raise SkipDownload(pyfile.pluginname)
elif pyfile.status in (
5, 7) and starting: #a download is waiting/starting and was apparently started before
raise SkipDownload(pyfile.pluginname)
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, pack.folder, self.pyfile.name)
if starting and self.core.config['download']['skip_existing'] and exists(location):
size = os.stat(location).st_size
if size >= self.pyfile.size:
raise SkipDownload("File exists.")
pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
if pyfile:
if exists(location):
raise SkipDownload(pyfile[0])
self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def conf_path(self, trans=False):
if trans:
gettext.setpaths(
[join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation(
"setup",
join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"],
fallback=True)
translation.install(True)
print _(
"Setting new configpath, current configuration will not be transferred!"
)
path = self.ask(_("Configpath"), abspath(""))
try:
path = join(pypath, path)
if not exists(path):
makedirs(path)
f = open(join(pypath, "module", "config", "configdir"), "wb")
f.write(path)
f.close()
print _(
"Configpath changed, setup will now close, please restart to go on."
)
print _("Press Enter to exit.")
raw_input()
exit()
except Exception, e:
print _("Setting config path failed: %s") % str(e)
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
"""Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.checkAbort()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(_("Setting User and Group failed: %s") % str(e))
def startExtracting(self, plugin, fid, passwords, thread):
pyfile = self.core.files.getFile(fid)
if not pyfile:
return []
pyfile.setCustomStatus(_("extracting"))
thread.addActive(pyfile) # keep this file until everything is done
try:
progress = lambda x: pyfile.setProgress(x)
success = False
if not plugin.checkArchive():
plugin.extract(progress)
success = True
else:
self.logInfo(basename(plugin.file), _("Password protected"))
self.logDebug("Passwords: %s" % str(passwords))
pwlist = copy(self.getPasswords())
#remove already supplied pws from list (only local)
for pw in passwords:
if pw in pwlist:
pwlist.remove(pw)
for pw in passwords + pwlist:
try:
self.logDebug("Try password: %s" % pw)
if plugin.checkPassword(pw):
plugin.extract(progress, pw)
self.addPassword(pw)
success = True
break
except WrongPassword:
self.logDebug("Password was wrong")
if not success:
self.logError(basename(plugin.file), _("Wrong password"))
return []
if self.core.debug:
self.logDebug("Would delete: %s" % ", ".join(plugin.getDeleteFiles()))
if self.getConfig("deletearchive"):
files = plugin.getDeleteFiles()
self.logInfo(_("Deleting %s files") % len(files))
for f in files:
if exists(f):
remove(f)
else:
self.logDebug("%s does not exists" % f)
self.logInfo(basename(plugin.file), _("Extracting finished"))
self.manager.dispatchEvent("unrarFinished", plugin.out, plugin.file)
return plugin.getExtractedFiles()
except ArchiveError, e:
self.logError(basename(plugin.file), _("Archive Error"), str(e))
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
"""Load content at url and returns it
:param url: url as string
:param get: GET as dict
:param post: POST as dict, list or string
:param ref: Set HTTP_REFERER header
:param cookies: use saved cookies
:param just_header: if True only the header will be retrieved and returned as dict
:param decode: Whether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.")
self.checkAbort()
res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
if not exists(join("tmp", self.__name__)):
makedirs(join("tmp", self.__name__))
f = open(
join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
, "wb")
del frame # delete the frame or it wont be cleaned
try:
tmp = res.encode("utf8")
except:
tmp = res
f.write(tmp)
f.close()
if just_header:
#parse header
header = {"code": self.req.code}
for line in res.splitlines():
line = line.strip()
if not line or ":" not in line: continue
key, none, value = line.partition(":")
key = key.lower().strip()
value = value.strip()
if key in header:
if type(header[key]) == list:
header[key].append(value)
else:
header[key] = [header[key], value]
else:
header[key] = value
res = header
return res
def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
"""Load content at url and returns it
:param url: url as string
:param get: GET as dict
:param post: POST as dict, list or string
:param ref: Set HTTP_REFERER header
:param cookies: use saved cookies
:param just_header: if True only the header will be retrieved and returned as dict
:param decode: Whether to decode the output according to http header, should be True in most cases
:return: Loaded content
"""
if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.")
self.checkAbort()
res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)
if self.core.debug:
from inspect import currentframe
frame = currentframe()
if not exists(join("tmp", self.__name__)):
makedirs(join("tmp", self.__name__))
f = open(
join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
, "wb")
del frame # delete the frame or it wont be cleaned
try:
tmp = res.encode("utf8")
except:
tmp = res
f.write(tmp)
f.close()
if just_header:
#parse header
header = {"code": self.req.code}
for line in res.splitlines():
line = line.strip()
if not line or ":" not in line: continue
key, none, value = line.partition(":")
key = key.lower().strip()
value = value.strip()
if key in header:
if type(header[key]) == list:
header[key].append(value)
else:
header[key] = [header[key], value]
else:
header[key] = value
res = header
return res
def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf: break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf: break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(
e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def reloadPasswords(self):
pwfile = self.getConfig("passwordfile")
if not exists(pwfile):
open(pwfile, "wb").close()
passwords = []
f = open(pwfile, "rb")
for pw in f.read().splitlines():
passwords.append(pw)
f.close()
self.passwords = passwords
def _checkVersion(self):
""" get db version"""
if not exists(self.VERSION_FILE):
f = open(self.VERSION_FILE, "wb")
f.write(str(DB_VERSION))
f.close()
return
f = open(self.VERSION_FILE, "rb")
v = int(f.read().strip())
f.close()
return v
def checkForSameFiles(self, starting=False):
""" checks if same file was/is downloaded within same package
:param starting: indicates that the current download is going to start
:raises SkipDownload:
"""
pack = self.pyfile.package()
for pyfile in self.core.files.cachedFiles():
if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package(
).folder == pack.folder:
if pyfile.status in (0, 12): #finished or downloading
raise SkipDownload(pyfile.pluginname)
elif pyfile.status in (
5, 7
) and starting: #a download is waiting/starting and was apparently started before
raise SkipDownload(pyfile.pluginname)
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, pack.folder, self.pyfile.name)
if starting and self.core.config['download'][
'skip_existing'] and exists(location):
size = os.stat(location).st_size
if size >= self.pyfile.size:
raise SkipDownload("File exists.")
pyfile = self.core.db.findDuplicates(self.pyfile.id,
self.pyfile.package().folder,
self.pyfile.name)
if pyfile:
if exists(location):
raise SkipDownload(pyfile[0])
self.log.debug("File %s not skipped, because it does not exists." %
self.pyfile.name)
def getLocalContent(self, urls):
"""Load files from disk and separate to file content and url list
:param urls:
:return: list of (filename, content), remote urls
"""
content = []
# do nothing if no decryptFile method
if hasattr(self.__class__, "decryptFile"):
remote = []
for url in urls:
path = None
if url.startswith("http"): # skip urls directly
pass
elif url.startswith(self.CONTENT_PREFIX):
path = url
elif exists(url):
path = url
elif exists(self.core.path(url)):
path = self.core.path(url)
if path:
try:
if path.startswith(self.CONTENT_PREFIX):
content.append(
("", path[len(self.CONTENT_PREFIX)]))
else:
f = open(fs_encode(path), "rb")
content.append((f.name, f.read()))
f.close()
except IOError, e:
self.logError("IOError", e)
else:
remote.append(url)
#swap filtered url list
urls = remote
def getLocalContent(self, urls):
"""Load files from disk and separate to file content and url list
:param urls:
:return: list of (filename, content), remote urls
"""
content = []
# do nothing if no decryptFile method
if hasattr(self.__class__, "decryptFile"):
remote = []
for url in urls:
path = None
if url.startswith("http"): # skip urls directly
pass
elif url.startswith(self.CONTENT_PREFIX):
path = url
elif exists(url):
path = url
elif exists(self.core.path(url)):
path = self.core.path(url)
if path:
try:
if path.startswith(self.CONTENT_PREFIX):
content.append(("", path[len(self.CONTENT_PREFIX)]))
else:
f = open(fs_encode(path), "rb")
content.append((f.name, f.read()))
f.close()
except IOError, e:
self.logError("IOError", e)
else:
remote.append(url)
#swap filtered url list
urls = remote
def setPermissions(self, files):
for f in files:
if not exists(f): continue
try:
if self.core.config["permission"]["change_file"]:
if isfile(f):
chmod(f, int(self.core.config["permission"]["file"], 8))
elif isdir(f):
chmod(f, int(self.core.config["permission"]["folder"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(f, uid, gid)
except Exception, e:
self.log.warning(_("Setting User and Group failed"), e)
def initPluginType(self, folder, path):
if not exists(path):
try:
makedirs(path)
except :
self.logDebug("Script folder %s not created" % folder)
return
for f in listdir(path):
if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"):
continue
if not access(join(path,f), X_OK):
self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f))
self.scripts[folder].append(join(path, f))
def initPluginType(self, folder, path):
if not exists(path):
try:
makedirs(path)
except:
self.logDebug("Script folder %s not created" % folder)
return
for f in listdir(path):
if f.startswith("#") or f.startswith(".") or f.startswith("_") or f.endswith("~") or f.endswith(".swp"):
continue
if not access(join(path, f), X_OK):
self.logWarning(_("Script not executable:") + " %s/%s" % (folder, f))
self.scripts[folder].append(join(path, f))
def setPermissions(self, files):
for f in files:
if not exists(f): continue
try:
if self.core.config["permission"]["change_file"]:
if isfile(f):
chmod(f, int(self.core.config["permission"]["file"], 8))
elif isdir(f):
chmod(f, int(self.core.config["permission"]["folder"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(f, uid, gid)
except Exception, e:
self.log.warning(_("Setting User and Group failed"), e)
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"),
gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"),
gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def checkDownload(self,
rules,
api_size=0,
max_size=50000,
delete=True,
read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param api_size: expected file size
:param max_size: if the file is larger then it wont be checked
:param delete: delete if matched
:param read_size: amount of bytes to read from files larger then max_size
:return: dictionary key of the first rule that matched
"""
lastDownload = fs_encode(self.lastDownload)
if not exists(lastDownload): return None
size = stat(lastDownload)
size = size.st_size
if api_size and api_size <= size: return None
elif size > max_size and not read_size: return None
self.log.debug("Download Check triggered")
f = open(lastDownload, "rb")
content = f.read(read_size if read_size else -1)
f.close()
#produces encoding errors, better log to other file in the future?
#self.log.debug("Content: %s" % content)
for name, rule in rules.iteritems():
if type(rule) in (str, unicode):
if rule in content:
if delete:
remove(lastDownload)
return name
elif hasattr(rule, "search"):
m = rule.search(content)
if m:
if delete:
remove(lastDownload)
self.lastCheck = m
return name
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def download(self,
url,
get={},
post={},
ref=True,
cookies=True,
disposition=False):
"""Downloads the content at url to download folder
:param url:
:param get:
:param post:
:param ref:
:param cookies:
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.checkAbort()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"],
8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
def conf_path(self, trans=False):
if trans:
gettext.setpaths([join(os.sep, "usr", "share", "pyload", "locale"), None])
translation = gettext.translation("setup", join(self.path, "locale"),
languages=[self.config["general"]["language"], "en"], fallback=True)
translation.install(True)
print _("Setting new configpath, current configuration will not be transferred!")
path = self.ask(_("Configpath"), abspath(""))
try:
path = join(pypath, path)
if not exists(path):
makedirs(path)
f = open(join(pypath, "module", "config", "configdir"), "wb")
f.write(path)
f.close()
print _("Configpath changed, setup will now close, please restart to go on.")
print _("Press Enter to exit.")
raw_input()
exit()
except Exception, e:
print _("Setting config path failed: %s") % str(e)
def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param api_size: expected file size
:param max_size: if the file is larger then it wont be checked
:param delete: delete if matched
:param read_size: amount of bytes to read from files larger then max_size
:return: dictionary key of the first rule that matched
"""
lastDownload = fs_encode(self.lastDownload)
if not exists(lastDownload): return None
size = stat(lastDownload)
size = size.st_size
if api_size and api_size <= size: return None
elif size > max_size and not read_size: return None
self.log.debug("Download Check triggered")
f = open(lastDownload, "rb")
content = f.read(read_size if read_size else -1)
f.close()
#produces encoding errors, better log to other file in the future?
#self.log.debug("Content: %s" % content)
for name, rule in rules.iteritems():
if type(rule) in (str, unicode):
if rule in content:
if delete:
remove(lastDownload)
return name
elif hasattr(rule, "search"):
m = rule.search(content)
if m:
if delete:
remove(lastDownload)
self.lastCheck = m
return name
def extract(self, ids, thread=None):
# reload from txt file
self.reloadPasswords()
# dl folder
dl = self.config['general']['download_folder']
extracted = []
#iterate packages -> plugins -> targets
for pid in ids:
p = self.core.files.getPackage(pid)
self.logInfo(_("Check package %s") % p.name)
if not p:
continue
# determine output folder
out = save_join(dl, p.folder, "")
# force trailing slash
if self.getConfig("destination") and self.getConfig("destination").lower() != "none":
out = save_join(dl, p.folder, self.getConfig("destination"), "")
#relative to package folder if destination is relative, otherwise absolute path overwrites them
if self.getConfig("subfolder"):
out = join(out, fs_encode(p.folder))
if not exists(out):
makedirs(out)
files_ids = [(save_join(dl, p.folder, x["name"]), x["id"]) for x in p.getChildren().itervalues()]
matched = False
# check as long there are unseen files
while files_ids:
new_files_ids = []
for plugin in self.plugins:
targets = plugin.getTargets(files_ids)
if targets:
self.logDebug("Targets for %s: %s" % (plugin.__name__, targets))
matched = True
for target, fid in targets:
if target in extracted:
self.logDebug(basename(target), "skipped")
continue
extracted.append(target) # prevent extracting same file twice
klass = plugin(self, target, out, self.getConfig("fullpath"), self.getConfig("overwrite"),
self.getConfig("renice"))
klass.init()
self.logInfo(basename(target), _("Extract to %s") % out)
new_files = self.startExtracting(klass, fid, p.password.strip().splitlines(), thread)
self.logDebug("Extracted: %s" % new_files)
self.setPermissions(new_files)
for file in new_files:
if not exists(file):
self.logDebug("new file %s does not exists" % file)
continue
if self.getConfig("recursive") and isfile(file):
new_files_ids.append((file, fid)) # append as new target
files_ids = new_files_ids # also check extracted files
if not matched:
self.logInfo(_("No files found to extract"))
