def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf: break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def test_plugin(self, name, url, flag):
# Print to stdout to see whats going on
print "%s: %s, %s" % (name, url.encode("utf8"), flag)
log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)
# url and plugin should be only important thing
pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
pyfile.initPlugin()
self.thread.pyfile = pyfile
self.thread.plugin = pyfile.plugin
try:
a = time()
pyfile.plugin.preprocessing(self.thread)
log(DEBUG, "downloading took %ds" % (time() - a))
log(DEBUG, "size %d kb" % (pyfile.size / 1024))
if flag == "offline":
raise Exception("No offline Exception raised.")
if pyfile.name not in self.files:
raise Exception("Filename %s not recognized." % pyfile.name)
if not exists(save_join(DL_DIR, pyfile.name)):
raise Exception("File %s does not exists." % pyfile.name)
hash = md5()
f = open(save_join(DL_DIR, pyfile.name), "rb")
while True:
buf = f.read(4096)
if not buf: break
hash.update(buf)
f.close()
if hash.hexdigest() != self.files[pyfile.name]:
log(DEBUG, "Hash is %s" % hash.hexdigest())
size = stat(f.name).st_size
if size < 1024 * 1024 * 10: # 10MB
# Copy for debug report
log(DEBUG, "Downloaded file copied to report")
move(f.name, join("tmp", plugin, f.name))
raise Exception("Hash does not match.")
except Exception, e:
if isinstance(e, Fail) and flag == "fail":
pass
elif isinstance(
e, Fail) and flag == "offline" and e.message == "offline":
pass
else:
raise
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"),
gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"),
gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def writeDebugReport(self, name, pyfile=None, plugin=None):
""" writes a debug report to disk """
dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
if pyfile:
dump = self.getFileDump(pyfile)
else:
dump = self.getPluginDump(plugin)
try:
import zipfile
zip = zipfile.ZipFile(dump_name, "w")
if exists(join("tmp", name)):
for f in listdir(join("tmp", name)):
try:
# avoid encoding errors
zip.write(join("tmp", name, f), save_join(name, f))
except:
pass
info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime())
info.external_attr = 0644 << 16L # change permissions
zip.writestr(info, dump)
info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime())
info.external_attr = 0644 << 16L
zip.writestr(info, self.getSystemDump())
zip.close()
if not stat(dump_name).st_size:
raise Exception("Empty Zipfile")
except Exception, e:
self.log.debug("Error creating zip file: %s" % e)
dump_name = dump_name.replace(".zip", ".txt")
f = open(dump_name, "wb")
f.write(dump)
f.close()
def checkDownload(self,
rules,
api_size=0,
max_size=50000,
delete=True,
read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param api_size: expected file size
:param max_size: if the file is larger then it wont be checked
:param delete: delete if matched
:param read_size: amount of bytes to read from files larger then max_size
:return: dictionary key of the first rule that matched
"""
lastDownload = fs_encode(self.lastDownload)
if not exists(lastDownload): return None
size = stat(lastDownload)
size = size.st_size
if api_size and api_size <= size: return None
elif size > max_size and not read_size: return None
self.log.debug("Download Check triggered")
f = open(lastDownload, "rb")
content = f.read(read_size if read_size else -1)
f.close()
#produces encoding errors, better log to other file in the future?
#self.log.debug("Content: %s" % content)
for name, rule in rules.iteritems():
if type(rule) in (str, unicode):
if rule in content:
if delete:
remove(lastDownload)
return name
elif hasattr(rule, "search"):
m = rule.search(content)
if m:
if delete:
remove(lastDownload)
self.lastCheck = m
return name
def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
""" checks the content of the last downloaded file, re match is saved to `lastCheck`
:param rules: dict with names and rules to match (compiled regexp or strings)
:param api_size: expected file size
:param max_size: if the file is larger then it wont be checked
:param delete: delete if matched
:param read_size: amount of bytes to read from files larger then max_size
:return: dictionary key of the first rule that matched
"""
lastDownload = fs_encode(self.lastDownload)
if not exists(lastDownload): return None
size = stat(lastDownload)
size = size.st_size
if api_size and api_size <= size: return None
elif size > max_size and not read_size: return None
self.log.debug("Download Check triggered")
f = open(lastDownload, "rb")
content = f.read(read_size if read_size else -1)
f.close()
#produces encoding errors, better log to other file in the future?
#self.log.debug("Content: %s" % content)
for name, rule in rules.iteritems():
if type(rule) in (str, unicode):
if rule in content:
if delete:
remove(lastDownload)
return name
elif hasattr(rule, "search"):
m = rule.search(content)
if m:
if delete:
remove(lastDownload)
self.lastCheck = m
return name