def refererParser(rawReferer):
if isinstance(rawReferer, dict): return rawReferer
import urllib, StringUtil , LumberjackConstants as L
rawReferer = StringUtil.convertTrChars(urllib.parse.unquote_plus(urllib.parse.unquote_plus(rawReferer)))
scheme, authority, path, params, query, frag = urllib.parse.urlparse(rawReferer)
query = [p.split('=') for p in query.split('&')]
query = [p for p in query if len(p) == 2]
referer = {p[0]: p[1] for p in query}
referer['page'] = path[1:]
return referer
def get27Keywords():
keywords = open('keywords27', 'rb').readlines()
import StringUtil
keywords = [StringUtil.convertTrChars(keyword.decode("utf-8")).replace('\n', '').lower() for keyword in keywords]
return keywords
