def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def setUp(self):
PluginTester.setUp(self)
for f in self.files:
if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))
# folder for reports
report = join("tmp", self.__class__.__name__)
if exists(report):
for f in listdir(report):
remove(join(report, f))
def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf: break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(
e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf:
break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def _copyChunks(self):
init = fs_encode(self.info.getChunkName(0)) #initial chunk name
if self.info.getCount() > 1:
fo = open(init, "rb+") #first chunkfile
for i in range(1, self.info.getCount()):
#input file
fo.seek(
self.info.getChunkRange(i - 1)[1] + 1) #seek to beginning of chunk, to get rid of overlapping chunks
fname = fs_encode("%s.chunk%d" % (self.path, i))
fi = open(fname, "rb")
buf = 32 * 1024
while True: #copy in chunks, consumes less memory
data = fi.read(buf)
if not data:
break
fo.write(data)
fi.close()
if fo.tell() < self.info.getChunkRange(i)[1]:
fo.close()
remove(init)
self.info.remove() #there are probably invalid chunks
raise Exception("Downloaded content was smaller than expected. Try to reduce download connections.")
remove(fname) #remove chunk
fo.close()
if self.name:
self.path = save_join(dirname(self.path), self.name)
move(init, fs_encode(self.path))
self.info.remove() #remove info file
def download(self,
url,
get={},
post={},
ref=True,
cookies=True,
disposition=False):
"""Downloads the content at url to download folder
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.checkAbort()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"],
8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(
_("Setting User and Group failed: %s") % str(e))
def checkForSameFiles(self, starting=False):
""" checks if same file was/is downloaded within same package
:param starting: indicates that the current download is going to start
:raises SkipDownload:
"""
pack = self.pyfile.package()
for pyfile in self.core.files.cachedFiles():
if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
if pyfile.status in (0, 12): #finished or downloading
raise SkipDownload(pyfile.pluginname)
elif pyfile.status in (
5, 7) and starting: #a download is waiting/starting and was apparently started before
raise SkipDownload(pyfile.pluginname)
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, pack.folder, self.pyfile.name)
if starting and self.core.config['download']['skip_existing'] and exists(location):
size = os.stat(location).st_size
if size >= self.pyfile.size:
raise SkipDownload("File exists.")
pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
if pyfile:
if exists(location):
raise SkipDownload(pyfile[0])
self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
"""Downloads the content at url to download folder
:param disposition: if True and server provides content-disposition header\
the filename will be changed if needed
:return: The location where the file was saved
"""
self.checkForSameFiles()
self.checkAbort()
self.pyfile.setStatus("downloading")
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, self.pyfile.package().folder)
if not exists(location):
makedirs(location, int(self.core.config["permission"]["folder"], 8))
if self.core.config["permission"]["change_dl"] and os.name != "nt":
try:
uid = getpwnam(self.config["permission"]["user"])[2]
gid = getgrnam(self.config["permission"]["group"])[2]
chown(location, uid, gid)
except Exception, e:
self.log.warning(_("Setting User and Group failed: %s") % str(e))
def _copyChunks(self):
init = fs_encode(self.info.getChunkName(0)) #initial chunk name
if self.info.getCount() > 1:
fo = open(init, "rb+") #first chunkfile
for i in range(1, self.info.getCount()):
#input file
fo.seek(
self.info.getChunkRange(i - 1)[1] + 1
) #seek to beginning of chunk, to get rid of overlapping chunks
fname = fs_encode("%s.chunk%d" % (self.path, i))
fi = open(fname, "rb")
buf = 32 * 1024
while True: #copy in chunks, consumes less memory
data = fi.read(buf)
if not data:
break
fo.write(data)
fi.close()
if fo.tell() < self.info.getChunkRange(i)[1]:
fo.close()
remove(init)
self.info.remove() #there are probably invalid chunks
raise Exception(
"Downloaded content was smaller than expected. Try to reduce download connections."
)
remove(fname) #remove chunk
fo.close()
if self.name:
self.path = save_join(dirname(self.path), self.name)
move(init, fs_encode(self.path))
self.info.remove() #remove info file
def checkForSameFiles(self, starting=False):
""" checks if same file was/is downloaded within same package
:param starting: indicates that the current download is going to start
:raises SkipDownload:
"""
pack = self.pyfile.package()
for pyfile in self.core.files.cachedFiles():
if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
if pyfile.status in (0, 12): #finished or downloading
raise SkipDownload(pyfile.pluginname)
elif pyfile.status in (
5, 7) and starting: #a download is waiting/starting and was apparently started before
raise SkipDownload(pyfile.pluginname)
download_folder = self.config['general']['download_folder']
location = save_join(download_folder, pack.folder, self.pyfile.name)
if starting and self.core.config['download']['skip_existing'] and exists(location):
size = os.stat(location).st_size
if size >= self.pyfile.size:
raise SkipDownload("File exists.")
pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
if pyfile:
if exists(location):
raise SkipDownload(pyfile[0])
self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getPluginDump(pyfile.plugin) + "\n"
dump += self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"),
gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"),
gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getPluginDump(pyfile.plugin) + "\n"
dump += self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def extract(self, ids, thread=None):
# reload from txt file
self.reloadPasswords()
# dl folder
dl = self.config['general']['download_folder']
extracted = []
#iterate packages -> plugins -> targets
for pid in ids:
p = self.core.files.getPackage(pid)
self.logInfo(_("Check package %s") % p.name)
if not p:
continue
# determine output folder
out = save_join(dl, p.folder, "")
# force trailing slash
if self.getConfig("destination") and self.getConfig(
"destination").lower() != "none":
out = save_join(dl, p.folder, self.getConfig("destination"),
"")
#relative to package folder if destination is relative, otherwise absolute path overwrites them
if self.getConfig("subfolder"):
out = join(out, fs_encode(p.folder))
if not exists(out):
makedirs(out)
files_ids = [(save_join(dl, p.folder, f.name), f.fid)
for f in p.getFiles().itervalues()]
matched = False
# check as long there are unseen files
while files_ids:
new_files_ids = []
for plugin in self.plugins:
targets = plugin.getTargets(files_ids)
if targets:
self.logDebug("Targets for %s: %s" %
(plugin.__name__, targets))
matched = True
for target, fid in targets:
if target in extracted:
self.logDebug(basename(target), "skipped")
continue
extracted.append(
target) # prevent extracting same file twice
klass = plugin(self, target, out,
self.getConfig("fullpath"),
self.getConfig("overwrite"),
self.getConfig("excludefiles"),
self.getConfig("renice"))
klass.init()
self.logInfo(basename(target),
_("Extract to %s") % out)
new_files = self.startExtracting(
klass, fid,
p.password.strip().splitlines(), thread)
self.logDebug("Extracted: %s" % new_files)
self.setPermissions(new_files)
for file in new_files:
if not exists(file):
self.logDebug("new file %s does not exists" %
file)
continue
if self.getConfig("recursive") and isfile(file):
new_files_ids.append(
(file, fid)) # append as new target
files_ids = new_files_ids # also check extracted files
if not matched:
self.logInfo(_("No files found to extract"))
def extract(self, ids, thread=None):
# reload from txt file
self.reloadPasswords()
# dl folder
dl = self.config['general']['download_folder']
extracted = []
#iterate packages -> plugins -> targets
for pid in ids:
p = self.core.files.getPackage(pid)
self.logInfo(_("Check package %s") % p.name)
if not p:
continue
# determine output folder
out = save_join(dl, p.folder, "")
# force trailing slash
if self.getConfig("destination") and self.getConfig("destination").lower() != "none":
out = save_join(dl, p.folder, self.getConfig("destination"), "")
#relative to package folder if destination is relative, otherwise absolute path overwrites them
if self.getConfig("subfolder"):
out = join(out, fs_encode(p.folder))
if not exists(out):
makedirs(out)
files_ids = [(save_join(dl, p.folder, f.name), f.fid) for f in p.getFiles().itervalues()]
matched = False
# check as long there are unseen files
while files_ids:
new_files_ids = []
for plugin in self.plugins:
targets = plugin.getTargets(files_ids)
if targets:
self.logDebug("Targets for %s: %s" % (plugin.__name__, targets))
matched = True
for target, fid in targets:
if target in extracted:
self.logDebug(basename(target), "skipped")
continue
extracted.append(target) # prevent extracting same file twice
klass = plugin(self, target, out, self.getConfig("fullpath"), self.getConfig("overwrite"), self.getConfig("excludefiles"),
self.getConfig("renice"))
klass.init()
self.logInfo(basename(target), _("Extract to %s") % out)
new_files = self.startExtracting(klass, fid, p.password.strip().splitlines(), thread)
self.logDebug("Extracted: %s" % new_files)
self.setPermissions(new_files)
for file in new_files:
if not exists(file):
self.logDebug("new file %s does not exists" % file)
continue
if self.getConfig("recursive") and isfile(file):
new_files_ids.append((file, fid)) # append as new target
files_ids = new_files_ids # also check extracted files
if not matched:
self.logInfo(_("No files found to extract"))
