Exemple #1
0
    def download(self,
                 url,
                 get={},
                 post={},
                 ref=True,
                 cookies=True,
                 disposition=False):
        """Downloads the content at url to download folder

        :param url:
        :param get:
        :param post:
        :param ref:
        :param cookies:
        :param disposition: if True and server provides content-disposition header\
        the filename will be changed if needed
        :return: The location where the file was saved
        """
        if self.pyfile.abort:
            self.abort()

        if not url:
            self.fail(_("No url given"))

        url = urllib.unquote(encode(url).strip())

        if self.core.debug:
            self.logDebug(
                "Download url: " + url, *[
                    "%s=%s" % (key, val) for key, val in locals().iteritems()
                    if key not in ("self", "url")
                ])

        self.checkForSameFiles()

        self.pyfile.setStatus("downloading")

        if disposition:
            self.pyfile.name = urlparse.urlparse(url).path.split(
                '/')[-1] or self.pyfile.name

        download_folder = self.core.config.get("general", "download_folder")

        location = fs_join(download_folder, self.pyfile.package().folder)

        if not exists(location):
            try:
                makedirs(location,
                         int(self.core.config.get("permission", "folder"), 8))

                if self.core.config.get("permission",
                                        "change_dl") and os.name != "nt":
                    uid = getpwnam(self.core.config.get("permission",
                                                        "user"))[2]
                    gid = getgrnam(self.core.config.get("permission",
                                                        "group"))[2]
                    chown(location, uid, gid)

            except Exception, e:
                self.fail(e)
Exemple #2
0
    def eval(self, script, engine=None):  #: engine can be a jse name """string""" or an AbstractEngine """class"""
        JSE = self.get(engine)

        if not JSE:
            raise TypeError("engine")

        script = encode(script)

        out, err = JSE.eval(script)

        results = [out]

        if self.core.config.get("general", "debug"):
            if err:
                self.core.log.debug(JSE._name + ":", err)

            engines = self.find()
            engines.remove(JSE)
            for E in engines:
                out, err = E.eval(script)
                res = err or out
                self.core.log.debug(E._name + ":", res)
                results.append(res)

            if len(results) > 1 and len(uniqify(results)) > 1:
                self.core.log.warning("JS output of two or more engines mismatch")

        return results[0]
Exemple #3
0
    def saveConfig(self, config, filename):
        """saves config to filename"""
        with open(filename, "wb") as f:
            chmod(filename, 0600)
            f.write("version: %i \n" % CONF_VERSION)
            for section in config.iterkeys():
                f.write('\n%s - "%s":\n' % (section, config[section]["desc"]))

                for option, data in config[section].iteritems():
                    if option in ("desc", "outline"): continue

                    if isinstance(data["value"], list):
                        value = "[ \n"
                        for x in data["value"]:
                            value += "\t\t" + str(x) + ",\n"
                        value += "\t\t]\n"
                    else:
                        if isinstance(data["value"], basestring):
                            value = data["value"] + "\n"
                        else:
                            value = str(data["value"]) + "\n"
                    try:
                        f.write('\t%s %s : "%s" = %s' %
                                (data["type"], option, data["desc"], value))
                    except UnicodeEncodeError:
                        f.write('\t%s %s : "%s" = %s' %
                                (data["type"], option, data["desc"],
                                 encode(value)))
Exemple #4
0
    def load(self, url, get={}, post={}, ref=True, cookies=True, just_header=False, decode=False, follow_location=True, save_cookies=True):
        """Load content at url and returns it

        :param url:
        :param get:
        :param post:
        :param ref:
        :param cookies:
        :param just_header: If True only the header will be retrieved and returned as dict
        :param decode: Wether to decode the output according to http header, should be True in most cases
        :param follow_location: If True follow location else not
        :param save_cookies: If True saves received cookies else discard them
        :return: Loaded content
        """
        if self.pyfile.abort:
            self.abort()

        if not url:
            self.fail(_("No url given"))

        url = urllib.unquote(encode(url).strip())  #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins

        if self.core.debug:
            self.logDebug("Load url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])

        res = self.req.load(url, get, post, ref, cookies, just_header, decode=decode, follow_location=follow_location, save_cookies=save_cookies)

        if decode:
            res = encode(res)

        if self.core.debug:
            import inspect

            frame = inspect.currentframe()
            framefile = fs_join("tmp", self.getClassName(), "%s_line%s.dump.html" % (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
            try:
                if not os.path.exists(os.path.join("tmp", self.getClassName())):
                    os.makedirs(os.path.join("tmp", self.getClassName()))

                with open(framefile, "wb") as f:
                    del frame  #: delete the frame or it wont be cleaned
                    f.write(res)
            except IOError, e:
                self.logError(e)
Exemple #5
0
    def download(self, url, get={}, post={}, ref=True, cookies=True, disposition=False):
        """Downloads the content at url to download folder

        :param url:
        :param get:
        :param post:
        :param ref:
        :param cookies:
        :param disposition: if True and server provides content-disposition header\
        the filename will be changed if needed
        :return: The location where the file was saved
        """
        if self.pyfile.abort:
            self.abort()

        if not url:
            self.fail(_("No url given"))

        url = urllib.unquote(encode(url).strip())

        if self.core.debug:
            self.logDebug("Download url: " + url, *["%s=%s" % (key, val) for key, val in locals().iteritems() if key not in ("self", "url")])

        self.checkForSameFiles()

        self.pyfile.setStatus("downloading")

        if disposition:
            self.pyfile.name = urlparse.urlparse(url).path.split('/')[-1] or self.pyfile.name

        download_folder = self.core.config.get("general", "download_folder")

        location = fs_join(download_folder, self.pyfile.package().folder)

        if not os.path.exists(location):
            try:
                os.makedirs(location, int(self.core.config.get("permission", "folder"), 8))

                if self.core.config.get("permission", "change_dl") and os.name != "nt":
                    uid = pwd.getpwnam(self.core.config.get("permission", "user"))[2]
                    gid = grp.getgrnam(self.core.config.get("permission", "group"))[2]
                    os.chown(location, uid, gid)

            except Exception, e:
                self.fail(e)
Exemple #6
0
    def cast(self, typ, value):
        """cast value to given format"""
        if not isinstance(value, basestring):
            return value

        elif typ == "int":
            return int(value)
        elif typ == "bool":
            return value.lower() in ("1", "true", "on", "an", "yes")
        elif typ == "time":
            if not value:
                value = "0:00"
            if not ":" in value:
                value += ":00"
            return value
        elif typ in ("str", "file", "folder"):
            return encode(value)
        else:
            return value
Exemple #7
0
    def cast(self, typ, value):
        """cast value to given format"""
        if not isinstance(value, basestring):
            return value

        elif typ == "int":
            return int(value)
        elif typ == "bool":
            return value.lower() in ("1", "true", "on", "an", "yes")
        elif typ == "time":
            if not value:
                value = "0:00"
            if not ":" in value:
                value += ":00"
            return value
        elif typ in ("str", "file", "folder"):
            return encode(value)
        else:
            return value
Exemple #8
0
    def setRequestContext(self,
                          url,
                          get,
                          post,
                          referer,
                          cookies,
                          multipart=False):
        """ sets everything needed for the request """

        url = myquote(url)

        if get:
            get = urlencode(get)
            url = "%s?%s" % (url, get)

        self.c.setopt(pycurl.URL, url)
        self.c.lastUrl = url

        if post:
            self.c.setopt(pycurl.POST, 1)
            if not multipart:
                if type(post) == unicode:
                    post = str(post)  # unicode not allowed
                elif type(post) == str:
                    pass
                else:
                    post = myurlencode(post)

                self.c.setopt(pycurl.POSTFIELDS, post)
            else:
                post = [(x, encode(y)) for x, y in post.iteritems()]
                self.c.setopt(pycurl.HTTPPOST, post)
        else:
            self.c.setopt(pycurl.POST, 0)

        if referer and self.lastURL:
            self.c.setopt(pycurl.REFERER, str(self.lastURL))

        if cookies:
            self.c.setopt(pycurl.COOKIEFILE, "")
            self.c.setopt(pycurl.COOKIEJAR, "")
            self.getCookies()
Exemple #9
0
    def setRequestContext(self, url, get, post, referer, cookies, multipart=False):
        """ sets everything needed for the request """

        url = myquote(url)

        if get:
            get = urlencode(get)
            url = "%s?%s" % (url, get)

        self.c.setopt(pycurl.URL, url)
        self.c.lastUrl = url

        if post:
            self.c.setopt(pycurl.POST, 1)
            if not multipart:
                if type(post) == unicode:
                    post = str(post)  #: unicode not allowed
                elif type(post) == str:
                    pass
                else:
                    post = myurlencode(post)

                self.c.setopt(pycurl.POSTFIELDS, post)
            else:
                post = [(x, encode(y)) for x, y in post.iteritems()]
                self.c.setopt(pycurl.HTTPPOST, post)
        else:
            self.c.setopt(pycurl.POST, 0)

        if referer and self.lastURL:
            self.c.setopt(pycurl.REFERER, str(self.lastURL))

        if cookies:
            self.c.setopt(pycurl.COOKIEFILE, "")
            self.c.setopt(pycurl.COOKIEJAR, "")
            self.getCookies()
Exemple #10
0
 def getPlugin(self, plugin, option):
     """gets a value for a plugin"""
     value = self.plugin[plugin][option]["value"]
     return encode(value)
Exemple #11
0
 def getResult(self):
     return encode(self.result)
Exemple #12
0
def myquote(url):
    return quote(encode(url), safe="%/:=&?~#+!$,;'@()*[]")
Exemple #13
0
def myurlencode(data):
    data = dict(data)
    return urlencode(dict((encode(x), encode(y)) for x, y in data.iteritems()))
Exemple #14
0
    def saveConfig(self, config, filename):
        """saves config to filename"""
        with open(filename, "wb") as f:
            chmod(filename, 0600)
            f.write("version: %i \n" % CONF_VERSION)
            for section in config.iterkeys():
                f.write('\n%s - "%s":\n' % (section, config[section]['desc']))

                for option, data in sorted(config[section].items(), key=lambda i: i[1]['idx'] if i[0] not in ("desc", "outline") else 0):
                    if option in ("desc", "outline"):
                        continue

                    if isinstance(data['value'], list):
                        value = "[ \n"
                        for x in data['value']:
                            value += "\t\t" + str(x) + ",\n"
                        value += "\t\t]\n"
                    else:
                        if isinstance(data['value'], basestring):
                            value = data['value'] + "\n"
                        else:
                            value = str(data['value']) + "\n"
                    try:
                        f.write('\t%s %s : "%s" = %s' % (data['type'], option, data['desc'], value))
                    except UnicodeEncodeError:
                        f.write('\t%s %s : "%s" = %s' % (data['type'], option, data['desc'], encode(value)))
Exemple #15
0
def myurlencode(data):
    data = dict(data)
    return urlencode(dict((encode(x), encode(y)) for x, y in data.iteritems()))
Exemple #16
0
 def getResult(self):
     return encode(self.result)
Exemple #17
0
    def load(self,
             url,
             get={},
             post={},
             ref=True,
             cookies=True,
             just_header=False,
             decode=False,
             follow_location=True,
             save_cookies=True):
        """Load content at url and returns it

        :param url:
        :param get:
        :param post:
        :param ref:
        :param cookies:
        :param just_header: If True only the header will be retrieved and returned as dict
        :param decode: Wether to decode the output according to http header, should be True in most cases
        :param follow_location: If True follow location else not
        :param save_cookies: If True saves received cookies else discard them
        :return: Loaded content
        """
        if self.pyfile.abort:
            self.abort()

        if not url:
            self.fail(_("No url given"))

        url = urllib.unquote(
            encode(url).strip()
        )  #@NOTE: utf8 vs decode -> please use decode attribute in all future plugins

        if self.core.debug:
            self.logDebug(
                "Load url: " + url, *[
                    "%s=%s" % (key, val) for key, val in locals().iteritems()
                    if key not in ("self", "url")
                ])

        res = self.req.load(url,
                            get,
                            post,
                            ref,
                            cookies,
                            just_header,
                            decode=decode,
                            follow_location=follow_location,
                            save_cookies=save_cookies)

        if decode:
            res = encode(res)

        if self.core.debug:
            import inspect

            frame = inspect.currentframe()
            framefile = fs_join(
                "tmp", self.getClassName(), "%s_line%s.dump.html" %
                (frame.f_back.f_code.co_name, frame.f_back.f_lineno))
            try:
                if not os.path.exists(os.path.join("tmp",
                                                   self.getClassName())):
                    os.makedirs(os.path.join("tmp", self.getClassName()))

                with open(framefile, "wb") as f:
                    del frame  #: delete the frame or it wont be cleaned
                    f.write(res)
            except IOError, e:
                self.logError(e)
Exemple #18
0
 def getPlugin(self, plugin, option):
     """gets a value for a plugin"""
     value = self.plugin[plugin][option]['value']
     return encode(value)
Exemple #19
0
 def _log(self, type, args):
     msg = " | ".join([encode(str(a)).strip() for a in args if a])
     logger = getattr(self.core.log, type)
     logger("%s: %s" % (self.getClassName(), msg or _("%s MARK" % type.upper())))
Exemple #20
0
def myquote(url):
    return quote(encode(url), safe="%/:=&?~#+!$,;'@()*[]")
Exemple #21
0
 def _log(self, type, args):
     msg = " | ".join([encode(str(a)).strip() for a in args if a])
     logger = getattr(self.core.log, type)
     logger("%s: %s" %
            (self.getClassName(), msg or _("%s MARK" % type.upper())))
Exemple #22
0
    def saveConfig(self, config, filename):
        """saves config to filename"""
        with open(filename, "wb") as f:
            try:
                os.chmod(filename, 0600)
            except Exception:
                pass

            f.write("version: %i \n" % CONF_VERSION)
            for section in config.iterkeys():
                f.write('\n%s - "%s":\n' % (section, config[section]['desc']))

                for option, data in sorted(config[section].items(), key=lambda i: i[1]['idx'] if i[0] not in ("desc", "outline") else 0):
                    if option in ("desc", "outline"):
                        continue

                    if isinstance(data['value'], list):
                        value = "[ \n"
                        for x in data['value']:
                            value += "\t\t" + str(x) + ",\n"
                        value += "\t\t]\n"
                    else:
                        if isinstance(data['value'], basestring):
                            value = data['value'] + "\n"
                        else:
                            value = str(data['value']) + "\n"
                    try:
                        f.write('\t%s %s : "%s" = %s' % (data['type'], option, data['desc'], value))
                    except UnicodeEncodeError:
                        f.write('\t%s %s : "%s" = %s' % (data['type'], option, data['desc'], encode(value)))