def get(self, keyword):
pages = []
spages = []
words = []
if keyword:
import jieba # May fail to load jieba
jieba.initialize(usingSmall=True)
words = list(jieba.cut(keyword, cut_all=False))
words = [ word for word in words if len(word) > 1 ]
# words = list(jieba.cut_for_search(keyword))
keyword = stringutil.parseUnicode(keyword)
pages = snapi.getAllPages()
pages = globalutil.search(pages, words)
globalutil.populateSourceUrl(pages)
twitterAccount = globalconfig.getTwitterAccount()
spages = bs.search(words[0], twitterAccount)
templateValues = {
'keyword': keyword,
'pages': pages,
'spages': spages,
'words': words,
}
self.render(templateValues, 'search.html')
def getRefererKeyword(referer):
if not referer:
return None
matched = None
for robot in config.getRobots():
if re.search(robot.get('referer'), referer, re.IGNORECASE|re.DOTALL):
matched = robot
break
if not matched:
return None
parameter = matched.get('parameter', 'q')
encoding = matched.get('encoding', 'utf8')
keyword = _getUnquotedParameter(referer, parameter)
if keyword:
keyword = stringutil.parseUnicode(keyword, encodings=[encoding])
return keyword
