def test_plugin(self, name, url, flag):
        # Print to stdout to see whats going on
        print "%s: %s, %s" % (name, url.encode("utf8"), flag)
        log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)

        # url and plugin should be only important thing
        pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
        pyfile.initPlugin()

        self.thread.pyfile = pyfile
        self.thread.plugin = pyfile.plugin

        try:
            a = time()
            pyfile.plugin.preprocessing(self.thread)

            log(DEBUG, "downloading took %ds" % (time() - a))
            log(DEBUG, "size %d kb" % (pyfile.size / 1024))

            if flag == "offline":
                raise Exception("No offline Exception raised.")

            if pyfile.name not in self.files:
                raise Exception("Filename %s not recognized." % pyfile.name)

            if not exists(save_join(DL_DIR, pyfile.name)):
                raise Exception("File %s does not exists." % pyfile.name)

            hash = md5()
            f = open(save_join(DL_DIR, pyfile.name), "rb")
            while True:
                buf = f.read(4096)
                if not buf: break
                hash.update(buf)
            f.close()

            if hash.hexdigest() != self.files[pyfile.name]:
                log(DEBUG, "Hash is %s" % hash.hexdigest())
                
                size = stat(f.name).st_size
                if size < 1024 * 1024 * 10: # 10MB
                    # Copy for debug report
                    log(DEBUG, "Downloaded file copied to report")
                    move(f.name, join("tmp", plugin, f.name))

                raise Exception("Hash does not match.")



        except Exception, e:
            if isinstance(e, Fail) and flag == "fail":
                pass
            elif isinstance(e, Fail) and flag == "offline" and e.message == "offline":
                pass
            else:
                raise
Beispiel #2
0
    def test_plugin(self, name, url, flag):
        # Print to stdout to see whats going on
        print "%s: %s, %s" % (name, url.encode("utf8"), flag)
        log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)

        # url and plugin should be only important thing
        pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
        pyfile.initPlugin()

        self.thread.pyfile = pyfile
        self.thread.plugin = pyfile.plugin

        try:
            a = time()
            pyfile.plugin.preprocessing(self.thread)

            log(DEBUG, "downloading took %ds" % (time() - a))
            log(DEBUG, "size %d kb" % (pyfile.size / 1024))

            if flag == "offline":
                raise Exception("No offline Exception raised.")

            if pyfile.name not in self.files:
                raise Exception("Filename %s not recognized." % pyfile.name)

            if not exists(save_join(DL_DIR, pyfile.name)):
                raise Exception("File %s does not exists." % pyfile.name)

            hash = md5()
            f = open(save_join(DL_DIR, pyfile.name), "rb")
            while True:
                buf = f.read(4096)
                if not buf: break
                hash.update(buf)
            f.close()

            if hash.hexdigest() != self.files[pyfile.name]:
                log(DEBUG, "Hash is %s" % hash.hexdigest())

                size = stat(f.name).st_size
                if size < 1024 * 1024 * 10:  # 10MB
                    # Copy for debug report
                    log(DEBUG, "Downloaded file copied to report")
                    move(f.name, join("tmp", plugin, f.name))

                raise Exception("Hash does not match.")

        except Exception, e:
            if isinstance(e, Fail) and flag == "fail":
                pass
            elif isinstance(
                    e, Fail) and flag == "offline" and e.message == "offline":
                pass
            else:
                raise
Beispiel #3
0
    def writeDebugReport(self, name, pyfile=None, plugin=None):
        """ writes a debug report to disk  """

        dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
        if pyfile:
            dump = self.getFileDump(pyfile)
        else:
            dump = self.getPluginDump(plugin)

        try:
            import zipfile

            zip = zipfile.ZipFile(dump_name, "w")

            if exists(join("tmp", name)):
                for f in listdir(join("tmp", name)):
                    try:
                        # avoid encoding errors
                        zip.write(join("tmp", name, f), save_join(name, f))
                    except:
                        pass

            info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"),
                                   gmtime())
            info.external_attr = 0644 << 16L  # change permissions
            zip.writestr(info, dump)

            info = zipfile.ZipInfo(save_join(name, "system_Report.txt"),
                                   gmtime())
            info.external_attr = 0644 << 16L
            zip.writestr(info, self.getSystemDump())

            zip.close()

            if not stat(dump_name).st_size:
                raise Exception("Empty Zipfile")

        except Exception, e:
            self.log.debug("Error creating zip file: %s" % e)

            dump_name = dump_name.replace(".zip", ".txt")
            f = open(dump_name, "wb")
            f.write(dump)
            f.close()
Beispiel #4
0
    def writeDebugReport(self, name, pyfile=None, plugin=None):
        """ writes a debug report to disk  """

        dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
        if pyfile:
            dump = self.getFileDump(pyfile)
        else:
            dump = self.getPluginDump(plugin)

        try:
            import zipfile

            zip = zipfile.ZipFile(dump_name, "w")

            if exists(join("tmp", name)):
                for f in listdir(join("tmp", name)):
                    try:
                        # avoid encoding errors
                        zip.write(join("tmp", name, f), save_join(name, f))
                    except:
                        pass

            info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime())
            info.external_attr = 0644 << 16L # change permissions
            zip.writestr(info, dump)

            info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime())
            info.external_attr = 0644 << 16L
            zip.writestr(info, self.getSystemDump())

            zip.close()

            if not stat(dump_name).st_size:
                raise Exception("Empty Zipfile")

        except Exception, e:
            self.log.debug("Error creating zip file: %s" % e)

            dump_name = dump_name.replace(".zip", ".txt")
            f = open(dump_name, "wb")
            f.write(dump)
            f.close()
Beispiel #5
0
    def checkDownload(self,
                      rules,
                      api_size=0,
                      max_size=50000,
                      delete=True,
                      read_size=0):
        """ checks the content of the last downloaded file, re match is saved to `lastCheck`
        
        :param rules: dict with names and rules to match (compiled regexp or strings)
        :param api_size: expected file size
        :param max_size: if the file is larger then it wont be checked
        :param delete: delete if matched
        :param read_size: amount of bytes to read from files larger then max_size
        :return: dictionary key of the first rule that matched
        """
        lastDownload = fs_encode(self.lastDownload)
        if not exists(lastDownload): return None

        size = stat(lastDownload)
        size = size.st_size

        if api_size and api_size <= size: return None
        elif size > max_size and not read_size: return None
        self.log.debug("Download Check triggered")
        f = open(lastDownload, "rb")
        content = f.read(read_size if read_size else -1)
        f.close()
        #produces encoding errors, better log to other file in the future?
        #self.log.debug("Content: %s" % content)
        for name, rule in rules.iteritems():
            if type(rule) in (str, unicode):
                if rule in content:
                    if delete:
                        remove(lastDownload)
                    return name
            elif hasattr(rule, "search"):
                m = rule.search(content)
                if m:
                    if delete:
                        remove(lastDownload)
                    self.lastCheck = m
                    return name
Beispiel #6
0
    def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
        """ checks the content of the last downloaded file, re match is saved to `lastCheck`
        
        :param rules: dict with names and rules to match (compiled regexp or strings)
        :param api_size: expected file size
        :param max_size: if the file is larger then it wont be checked
        :param delete: delete if matched
        :param read_size: amount of bytes to read from files larger then max_size
        :return: dictionary key of the first rule that matched
        """
        lastDownload = fs_encode(self.lastDownload)
        if not exists(lastDownload): return None

        size = stat(lastDownload)
        size = size.st_size

        if api_size and api_size <= size: return None
        elif size > max_size and not read_size: return None
        self.log.debug("Download Check triggered")
        f = open(lastDownload, "rb")
        content = f.read(read_size if read_size else -1)
        f.close()
        #produces encoding errors, better log to other file in the future?
        #self.log.debug("Content: %s" % content)
        for name, rule in rules.iteritems():
            if type(rule) in (str, unicode):
                if rule in content:
                    if delete:
                        remove(lastDownload)
                    return name
            elif hasattr(rule, "search"):
                m = rule.search(content)
                if m:
                    if delete:
                        remove(lastDownload)
                    self.lastCheck = m
                    return name