def get(self, keyword): pages = [] spages = [] words = [] if keyword: import jieba # May fail to load jieba jieba.initialize(usingSmall=True) words = list(jieba.cut(keyword, cut_all=False)) words = [ word for word in words if len(word) > 1 ] # words = list(jieba.cut_for_search(keyword)) keyword = stringutil.parseUnicode(keyword) pages = snapi.getAllPages() pages = globalutil.search(pages, words) globalutil.populateSourceUrl(pages) twitterAccount = globalconfig.getTwitterAccount() spages = bs.search(words[0], twitterAccount) templateValues = { 'keyword': keyword, 'pages': pages, 'spages': spages, 'words': words, } self.render(templateValues, 'search.html')
def getRefererKeyword(referer): if not referer: return None matched = None for robot in config.getRobots(): if re.search(robot.get('referer'), referer, re.IGNORECASE|re.DOTALL): matched = robot break if not matched: return None parameter = matched.get('parameter', 'q') encoding = matched.get('encoding', 'utf8') keyword = _getUnquotedParameter(referer, parameter) if keyword: keyword = stringutil.parseUnicode(keyword, encodings=[encoding]) return keyword