def _adapt_eventdict(eventDict, log_level=INFO, encoding="utf-8", prepend_level=True):
"""Adapt Twisted log eventDict making it suitable for logging with a Scrapy
log observer. It may return None to indicate that the event should be
ignored by a Scrapy log observer.
`log_level` is the minimum level being logged, and `encoding` is the log
encoding.
"""
ev = eventDict.copy()
if ev["isError"]:
ev.setdefault("logLevel", ERROR)
# ignore non-error messages from outside scrapy
if ev.get("system") != "scrapy" and not ev["isError"]:
return
level = ev.get("logLevel")
if level < log_level:
return
spider = ev.get("spider")
if spider:
ev["system"] = unicode_to_str(spider.name, encoding)
lvlname = level_names.get(level, "NOLEVEL")
message = ev.get("message")
if message:
message = [unicode_to_str(x, encoding) for x in message]
if prepend_level:
message[0] = "%s: %s" % (lvlname, message[0])
ev["message"] = message
why = ev.get("why")
if why:
why = unicode_to_str(why, encoding)
if prepend_level:
why = "%s: %s" % (lvlname, why)
ev["why"] = why
fmt = ev.get("format")
if fmt:
fmt = unicode_to_str(fmt, encoding)
if prepend_level:
fmt = "%s: %s" % (lvlname, fmt)
ev["format"] = fmt
return ev
def parse_url(url, encoding=None):
"""Return urlparsed url from the given argument (which could be an already
parsed url)
"""
return url if isinstance(url, urlparse.ParseResult) else \
urlparse.urlparse(unicode_to_str(url, encoding))