Beispiel #1
0
    def checkForSameFiles(self, starting=False):
        """ checks if same file was/is downloaded within same package

        :param starting: indicates that the current download is going to start
        :raises SkipDownload:
        """

        pack = self.pyfile.package()

        for pyfile in self.core.files.cachedFiles():
            if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
                if pyfile.status in (0, 12): #finished or downloading
                    raise SkipDownload(pyfile.pluginname)
                elif pyfile.status in (
                    5, 7) and starting: #a download is waiting/starting and was apparently started before
                    raise SkipDownload(pyfile.pluginname)

        download_folder = self.config['general']['download_folder']
        location = save_join(download_folder, pack.folder, self.pyfile.name)

        if starting and self.core.config['download']['skip_existing'] and exists(location):
            size = os.stat(location).st_size
            if size >= self.pyfile.size:
                raise SkipDownload("File exists.")

        pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
        if pyfile:
            if exists(location):
                raise SkipDownload(pyfile[0])

            self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
Beispiel #2
0
    def checkForSameFiles(self, starting=False):
        """ checks if same file was/is downloaded within same package

        :param starting: indicates that the current download is going to start
        :raises SkipDownload:
        """

        pack = self.pyfile.package()

        for pyfile in self.core.files.cachedFiles():
            if pyfile != self.pyfile and pyfile.name == self.pyfile.name and pyfile.package().folder == pack.folder:
                if pyfile.status in (0, 12): #finished or downloading
                    raise SkipDownload(pyfile.pluginname)
                elif pyfile.status in (
                    5, 7) and starting: #a download is waiting/starting and was apparently started before
                    raise SkipDownload(pyfile.pluginname)

        download_folder = self.config['general']['download_folder']
        location = save_join(download_folder, pack.folder, self.pyfile.name)

        if starting and self.core.config['download']['skip_existing'] and exists(location):
            size = os.stat(location).st_size
            if size >= self.pyfile.size:
                raise SkipDownload("File exists.")

        pyfile = self.core.db.findDuplicates(self.pyfile.id, self.pyfile.package().folder, self.pyfile.name)
        if pyfile:
            if exists(location):
                raise SkipDownload(pyfile[0])

            self.log.debug("File %s not skipped, because it does not exists." % self.pyfile.name)
Beispiel #3
0
    def run(self):
        self.running = True
        import webinterface
        global webinterface

        if self.https:
            if not exists(self.cert) or not exists(self.key):
                log.warning(_("SSL certificates not found."))
                self.https = False

        prefer = None

        # These cases covers all settings
        if self.server == "threaded":
            prefer = "threaded"
        elif self.server == "fastcgi":
            prefer = "flup"
        elif self.server == "fallback":
            prefer = "wsgiref"

        server = self.select_server(prefer)

        try:
            self.start_server(server)

        except Exception, e:
            log.error(_("Failed starting webserver: " + e.message))
            self.error = e
            if core:
                core.print_exc()
    def setUp(self):
        PluginTester.setUp(self)
        for f in self.files:
            if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))

        # folder for reports
        report = join("tmp", self.__class__.__name__)
        if exists(report):
            for f in listdir(report):
                remove(join(report, f))
Beispiel #5
0
    def isWSSecure(self):
        # needs to use TLS when either requested or webUI is also using encryption
        if not self.core.config['ssl']['activated'] or self.core.config['webUI']['https']:
            return False

        if not exists(self.core.config['ssl']['cert']) or not exists(self.core.config['ssl']['key']):
            self.core.log.warning(_('SSL key or certificate not found'))
            return False

        return True
    def setUp(self):
        PluginTester.setUp(self)
        for f in self.files:
            if exists(save_join(DL_DIR, f)): remove(save_join(DL_DIR, f))

        # folder for reports
        report = join("tmp", self.__class__.__name__)
        if exists(report):
            for f in listdir(report):
                remove(join(report, f))
Beispiel #7
0
    def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
        """Downloads the content at url to download folder

        :param disposition: if True and server provides content-disposition header\
        the filename will be changed if needed
        :return: The location where the file was saved
        """
        self.checkForSameFiles()
        self.checkAbort()

        self.pyfile.setStatus("downloading")

        download_folder = self.config['general']['download_folder']

        location = save_join(download_folder, self.pyfile.package().folder)

        if not exists(location):
            makedirs(location, int(self.core.config["permission"]["folder"], 8))

            if self.core.config["permission"]["change_dl"] and os.name != "nt":
                try:
                    uid = getpwnam(self.config["permission"]["user"])[2]
                    gid = getgrnam(self.config["permission"]["group"])[2]

                    chown(location, uid, gid)
                except Exception, e:
                    self.log.warning(_("Setting User and Group failed: %s") % str(e))
Beispiel #8
0
    def download(self,
                 url,
                 get={},
                 post={},
                 ref=True,
                 cookies=True,
                 disposition=False):
        """Downloads the content at url to download folder

        :param disposition: if True and server provides content-disposition header\
        the filename will be changed if needed
        :return: The location where the file was saved
        """
        self.checkForSameFiles()
        self.checkAbort()

        self.pyfile.setStatus("downloading")

        download_folder = self.config['general']['download_folder']

        location = save_join(download_folder, self.pyfile.package().folder)

        if not exists(location):
            makedirs(location, int(self.core.config["permission"]["folder"],
                                   8))

            if self.core.config["permission"]["change_dl"] and os.name != "nt":
                try:
                    uid = getpwnam(self.config["permission"]["user"])[2]
                    gid = getgrnam(self.config["permission"]["group"])[2]

                    chown(location, uid, gid)
                except Exception, e:
                    self.log.warning(
                        _("Setting User and Group failed: %s") % str(e))
Beispiel #9
0
    def conf_path(self, trans=False):
        if trans:
            gettext.setpaths(
                [join(os.sep, "usr", "share", "pyload", "locale"), None])
            translation = gettext.translation(
                "setup",
                join(self.path, "locale"),
                languages=[self.config["general"]["language"], "en"],
                fallback=True)
            translation.install(True)

        print _(
            "Setting new configpath, current configuration will not be transferred!"
        )
        path = self.ask(_("Config path"), abspath(""))
        try:
            path = join(pypath, path)
            if not exists(path):
                makedirs(path)
            f = open(join(pypath, "pyload", "config", "configdir"), "wb")
            f.write(path)
            f.close()
            print _(
                "Config path changed, setup will now close, please restart to go on."
            )
            print _("Press Enter to exit.")
            raw_input()
            exit()
        except Exception, e:
            print _("Setting config path failed: %s") % str(e)
Beispiel #10
0
    def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False):
        """Load content at url and returns it

        :param url: url as string
        :param get: GET as dict
        :param post: POST as dict, list or string
        :param ref: Set HTTP_REFERER header
        :param cookies: use saved cookies
        :param just_header: if True only the header will be retrieved and returned as dict
        :param decode: Whether to decode the output according to http header, should be True in most cases
        :return: Loaded content
        """
        if not hasattr(self, "req"): raise Exception("Plugin type does not have Request attribute.")
        self.checkAbort()

        res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode)

        if self.core.debug:
            from inspect import currentframe

            frame = currentframe()
            if not exists(join("tmp", self.__name__)):
                makedirs(join("tmp", self.__name__))

            f = open(
                join("tmp", self.__name__, "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
                , "wb")
            del frame # delete the frame or it wont be cleaned

            try:
                tmp = res.encode("utf8")
            except:
                tmp = res

            f.write(tmp)
            f.close()

        if just_header:
            #parse header
            header = {"code": self.req.code}
            for line in res.splitlines():
                line = line.strip()
                if not line or ":" not in line: continue

                key, none, value = line.partition(":")
                key = key.lower().strip()
                value = value.strip()

                if key in header:
                    if type(header[key]) == list:
                        header[key].append(value)
                    else:
                        header[key] = [header[key], value]
                else:
                    header[key] = value
            res = header

        return res
Beispiel #11
0
    def test_plugin(self, name, url, flag):
        # Print to stdout to see whats going on
        print "%s: %s, %s" % (name, url.encode("utf8"), flag)
        log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)

        # url and plugin should be only important thing
        pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
        pyfile.initPlugin()

        self.thread.pyfile = pyfile
        self.thread.plugin = pyfile.plugin

        try:
            a = time()
            pyfile.plugin.preprocessing(self.thread)

            log(DEBUG, "downloading took %ds" % (time() - a))
            log(DEBUG, "size %d kb" % (pyfile.size / 1024))

            if flag == "offline":
                raise Exception("No offline Exception raised.")

            if pyfile.name not in self.files:
                raise Exception("Filename %s not recognized." % pyfile.name)

            if not exists(save_join(DL_DIR, pyfile.name)):
                raise Exception("File %s does not exists." % pyfile.name)

            hash = md5()
            f = open(save_join(DL_DIR, pyfile.name), "rb")
            while True:
                buf = f.read(4096)
                if not buf:
                    break
                hash.update(buf)
            f.close()

            if hash.hexdigest() != self.files[pyfile.name]:
                log(DEBUG, "Hash is %s" % hash.hexdigest())

                size = stat(f.name).st_size
                if size < 1024 * 1024 * 10:  # 10MB
                    # Copy for debug report
                    log(DEBUG, "Downloaded file copied to report")
                    move(f.name, join("tmp", plugin, f.name))

                raise Exception("Hash does not match.")

        except Exception, e:
            if isinstance(e, Fail) and flag == "fail":
                pass
            elif isinstance(e, Fail) and flag == "offline" and e.message == "offline":
                pass
            else:
                raise
Beispiel #12
0
    def test_plugin(self, name, url, flag):
        # Print to stdout to see whats going on
        print "%s: %s, %s" % (name, url.encode("utf8"), flag)
        log(DEBUG, "%s: %s, %s", name, url.encode("utf8"), flag)

        # url and plugin should be only important thing
        pyfile = PyFile(self.core, -1, url, url, 0, 0, "", name, 0, 0)
        pyfile.initPlugin()

        self.thread.pyfile = pyfile
        self.thread.plugin = pyfile.plugin

        try:
            a = time()
            pyfile.plugin.preprocessing(self.thread)

            log(DEBUG, "downloading took %ds" % (time() - a))
            log(DEBUG, "size %d kb" % (pyfile.size / 1024))

            if flag == "offline":
                raise Exception("No offline Exception raised.")

            if pyfile.name not in self.files:
                raise Exception("Filename %s not recognized." % pyfile.name)

            if not exists(save_join(DL_DIR, pyfile.name)):
                raise Exception("File %s does not exists." % pyfile.name)

            hash = md5()
            f = open(save_join(DL_DIR, pyfile.name), "rb")
            while True:
                buf = f.read(4096)
                if not buf: break
                hash.update(buf)
            f.close()

            if hash.hexdigest() != self.files[pyfile.name]:
                log(DEBUG, "Hash is %s" % hash.hexdigest())

                size = stat(f.name).st_size
                if size < 1024 * 1024 * 10:  # 10MB
                    # Copy for debug report
                    log(DEBUG, "Downloaded file copied to report")
                    move(f.name, join("tmp", plugin, f.name))

                raise Exception("Hash does not match.")

        except Exception, e:
            if isinstance(e, Fail) and flag == "fail":
                pass
            elif isinstance(
                    e, Fail) and flag == "offline" and e.message == "offline":
                pass
            else:
                raise
Beispiel #13
0
    def getLocalContent(self, urls):
        """Load files from disk and separate to file content and url list

        :param urls:
        :return: list of (filename, content), remote urls
        """
        content = []
        # do nothing if no decryptFile method
        if hasattr(self.__class__, "decryptFile"):
            remote = []
            for url in urls:
                path = None
                if url.startswith("http"):  # skip urls directly
                    pass
                elif url.startswith(self.CONTENT_PREFIX):
                    path = url
                elif exists(url):
                    path = url
                elif exists(self.core.path(url)):
                    path = self.core.path(url)

                if path:
                    try:
                        if path.startswith(self.CONTENT_PREFIX):
                            content.append(
                                ("", path[len(self.CONTENT_PREFIX)]))
                        else:
                            f = open(fs_encode(path), "rb")
                            content.append((f.name, f.read()))
                            f.close()
                    except IOError, e:
                        self.logError("IOError", e)
                else:
                    remote.append(url)

            #swap filtered url list
            urls = remote
Beispiel #14
0
    def run(self):
        self.running = True
        import webinterface

        global webinterface

        if self.https:
            if not exists(self.cert) or not exists(self.key):
                log.warning(_("SSL certificates not found."))
                self.https = False

        if webinterface.UNAVAILALBE:
            log.warning(_("WebUI built is not available"))
        elif webinterface.APP_PATH == "app":
            log.info(_("Running webUI in development mode"))

        prefer = None

        # These cases covers all settings
        if self.server == "threaded":
            prefer = "threaded"
        elif self.server == "fastcgi":
            prefer = "flup"
        elif self.server == "fallback":
            prefer = "wsgiref"

        server = self.select_server(prefer)

        try:
            self.start_server(server)

        except Exception, e:
            log.error(_("Failed starting webserver: " + e.message))
            self.error = e
            if core:
                core.print_exc()
Beispiel #15
0
    def run(self):
        self.running = True
        import webinterface

        global webinterface

        if self.https:
            if not exists(self.cert) or not exists(self.key):
                log.warning(_("SSL certificates not found."))
                self.https = False

        if webinterface.UNAVAILALBE:
            log.warning(_("WebUI built is not available"))
        elif webinterface.APP_PATH == "app":
            log.info(_("Running webUI in development mode"))

        prefer = None

        # These cases covers all settings
        if self.server == "threaded":
            prefer = "threaded"
        elif self.server == "fastcgi":
            prefer = "flup"
        elif self.server == "fallback":
            prefer = "wsgiref"

        server = self.select_server(prefer)

        try:
            self.start_server(server)

        except Exception, e:
            log.error(_("Failed starting webserver: " + e.message))
            self.error = e
            if core:
                core.print_exc()
Beispiel #16
0
    def getLocalContent(self, urls):
        """Load files from disk and separate to file content and url list

        :param urls:
        :return: list of (filename, content), remote urls
        """
        content = []
        # do nothing if no decryptFile method
        if hasattr(self.__class__, "decryptFile"):
            remote = []
            for url in urls:
                path = None
                if url.startswith("http"): # skip urls directly
                    pass
                elif url.startswith(self.CONTENT_PREFIX):
                    path = url
                elif exists(url):
                    path = url
                elif exists(self.core.path(url)):
                    path = self.core.path(url)

                if path:
                    try:
                        if path.startswith(self.CONTENT_PREFIX):
                            content.append(("", path[len(self.CONTENT_PREFIX)]))
                        else:
                            f = open(fs_encode(path), "rb")
                            content.append((f.name, f.read()))
                            f.close()
                    except IOError, e:
                        self.logError("IOError", e)
                else:
                    remote.append(url)

            #swap filtered url list
            urls = remote
Beispiel #17
0
    def writeDebugReport(self, name, pyfile=None, plugin=None):
        """ writes a debug report to disk  """

        dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
        if pyfile:
            dump = self.getPluginDump(pyfile.plugin) + "\n"
            dump += self.getFileDump(pyfile)
        else:
            dump = self.getPluginDump(plugin)

        try:
            import zipfile
            zip = zipfile.ZipFile(dump_name, "w")

            if exists(join("tmp", name)):
                for f in listdir(join("tmp", name)):
                    try:
                        # avoid encoding errors
                        zip.write(join("tmp", name, f), save_join(name, f))
                    except:
                        pass

            info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"),
                                   gmtime())
            info.external_attr = 0644 << 16L  # change permissions
            zip.writestr(info, dump)

            info = zipfile.ZipInfo(save_join(name, "system_Report.txt"),
                                   gmtime())
            info.external_attr = 0644 << 16L
            zip.writestr(info, self.getSystemDump())

            zip.close()

            if not stat(dump_name).st_size:
                raise Exception("Empty Zipfile")

        except Exception, e:
            self.log.debug("Error creating zip file: %s" % e)

            dump_name = dump_name.replace(".zip", ".txt")
            f = open(dump_name, "wb")
            f.write(dump)
            f.close()
Beispiel #18
0
    def checkDownload(self,
                      rules,
                      api_size=0,
                      max_size=50000,
                      delete=True,
                      read_size=0):
        """ checks the content of the last downloaded file, re match is saved to `lastCheck`
        
        :param rules: dict with names and rules to match (compiled regexp or strings)
        :param api_size: expected file size
        :param max_size: if the file is larger then it wont be checked
        :param delete: delete if matched
        :param read_size: amount of bytes to read from files larger then max_size
        :return: dictionary key of the first rule that matched
        """
        lastDownload = fs_encode(self.lastDownload)
        if not exists(lastDownload): return None

        size = stat(lastDownload)
        size = size.st_size

        if api_size and api_size <= size:
            return None
        elif size > max_size and not read_size:
            return None
        self.log.debug("Download Check triggered")
        f = open(lastDownload, "rb")
        content = f.read(read_size if read_size else -1)
        f.close()
        #produces encoding errors, better log to other file in the future?
        #self.log.debug("Content: %s" % content)
        for name, rule in rules.iteritems():
            if type(rule) in (str, unicode):
                if rule in content:
                    if delete:
                        remove(lastDownload)
                    return name
            elif hasattr(rule, "search"):
                m = rule.search(content)
                if m:
                    if delete:
                        remove(lastDownload)
                    self.lastCheck = m
                    return name
Beispiel #19
0
    def writeDebugReport(self, name, pyfile=None, plugin=None):
        """ writes a debug report to disk  """

        dump_name = "debug_%s_%s.zip" % (name, strftime("%d-%m-%Y_%H-%M-%S"))
        if pyfile:
            dump = self.getPluginDump(pyfile.plugin) + "\n"
            dump += self.getFileDump(pyfile)
        else:
            dump = self.getPluginDump(plugin)

        try:
            import zipfile
            zip = zipfile.ZipFile(dump_name, "w")

            if exists(join("tmp", name)):
                for f in listdir(join("tmp", name)):
                    try:
                        # avoid encoding errors
                        zip.write(join("tmp", name, f), save_join(name, f))
                    except:
                        pass

            info = zipfile.ZipInfo(save_join(name, "debug_Report.txt"), gmtime())
            info.external_attr = 0644 << 16L # change permissions
            zip.writestr(info, dump)

            info = zipfile.ZipInfo(save_join(name, "system_Report.txt"), gmtime())
            info.external_attr = 0644 << 16L
            zip.writestr(info, self.getSystemDump())

            zip.close()

            if not stat(dump_name).st_size:
                raise Exception("Empty Zipfile")

        except Exception, e:
            self.log.debug("Error creating zip file: %s" % e)

            dump_name = dump_name.replace(".zip", ".txt")
            f = open(dump_name, "wb")
            f.write(dump)
            f.close()
Beispiel #20
0
    def conf_path(self, trans=False):
        if trans:
            translation = gettext.translation("setup", join(self.path, "locale"),
                languages=[self.config["general"]["language"], "en"], fallback=True)
            translation.install(True)

        print _("Setting new configpath, current configuration will not be transferred!")
        path = self.ask(_("Config path"), abspath(""))
        try:
            path = join(pypath, path)
            if not exists(path):
                makedirs(path)
            f = open(join(pypath, "pyload", "config", "configdir"), "wb")
            f.write(path)
            f.close()
            print _("Config path changed, setup will now close, please restart to go on.")
            print _("Press Enter to exit.")
            raw_input()
            exit()
        except Exception, e:
            print _("Setting config path failed: %s") % str(e)
Beispiel #21
0
    def checkDownload(self, rules, api_size=0, max_size=50000, delete=True, read_size=0):
        """ checks the content of the last downloaded file, re match is saved to `lastCheck`
        
        :param rules: dict with names and rules to match (compiled regexp or strings)
        :param api_size: expected file size
        :param max_size: if the file is larger then it wont be checked
        :param delete: delete if matched
        :param read_size: amount of bytes to read from files larger then max_size
        :return: dictionary key of the first rule that matched
        """
        lastDownload = fs_encode(self.lastDownload)
        if not exists(lastDownload):
            return None

        size = stat(lastDownload)
        size = size.st_size

        if api_size and api_size <= size:
            return None
        elif size > max_size and not read_size:
            return None
        self.log.debug("Download Check triggered")
        f = open(lastDownload, "rb")
        content = f.read(read_size if read_size else -1)
        f.close()
        # produces encoding errors, better log to other file in the future?
        # self.log.debug("Content: %s" % content)
        for name, rule in rules.iteritems():
            if type(rule) in (str, unicode):
                if rule in content:
                    if delete:
                        remove(lastDownload)
                    return name
            elif hasattr(rule, "search"):
                m = rule.search(content)
                if m:
                    if delete:
                        remove(lastDownload)
                    self.lastCheck = m
                    return name