示例#1
0
文件: log.py 项目: heartsg/dscrapy
def _adapt_eventdict(eventDict, log_level=INFO, encoding="utf-8", prepend_level=True):
    """Adapt Twisted log eventDict making it suitable for logging with a Scrapy
    log observer. It may return None to indicate that the event should be
    ignored by a Scrapy log observer.

    `log_level` is the minimum level being logged, and `encoding` is the log
    encoding.
    """
    ev = eventDict.copy()
    if ev["isError"]:
        ev.setdefault("logLevel", ERROR)

    # ignore non-error messages from outside scrapy
    if ev.get("system") != "scrapy" and not ev["isError"]:
        return

    level = ev.get("logLevel")
    if level < log_level:
        return

    spider = ev.get("spider")
    if spider:
        ev["system"] = unicode_to_str(spider.name, encoding)

    lvlname = level_names.get(level, "NOLEVEL")
    message = ev.get("message")
    if message:
        message = [unicode_to_str(x, encoding) for x in message]
        if prepend_level:
            message[0] = "%s: %s" % (lvlname, message[0])
        ev["message"] = message

    why = ev.get("why")
    if why:
        why = unicode_to_str(why, encoding)
        if prepend_level:
            why = "%s: %s" % (lvlname, why)
        ev["why"] = why

    fmt = ev.get("format")
    if fmt:
        fmt = unicode_to_str(fmt, encoding)
        if prepend_level:
            fmt = "%s: %s" % (lvlname, fmt)
        ev["format"] = fmt

    return ev
示例#2
0
文件: url.py 项目: heartsg/dscrapy
def parse_url(url, encoding=None):
    """Return urlparsed url from the given argument (which could be an already
    parsed url)
    """
    return url if isinstance(url, urlparse.ParseResult) else \
        urlparse.urlparse(unicode_to_str(url, encoding))