def POST(self): global _vm, _session _vm.attachCurrentThread() user_data = json.loads(web.input()['data']) print user_data tgtype = user_data['type'] if tgtype == 'user': tfield = 'author_index' elif tgtype == 'question': tfield = 'question_index' else: return '' with zh_iatd.create_searcher() as searcher: res1 = searcher.searcher.search( zh_iatd.create_query({ 'type': 'answer', tfield: user_data['index'] }), 200, Sort(SortField('likes', SortField.Type.INT, True))) res2 = searcher.searcher.search( zh_iatd.create_query({ 'type': 'answer', tfield: user_data['index'] }), 200, Sort(SortField('date', SortField.Type.INT, True))) res1 = [ zh_pganlz.document_to_obj(searcher.searcher.doc( x.doc)).data.likes for x in res1.scoreDocs ] res2 = [ zh_pganlz.document_to_obj(searcher.searcher.doc(x.doc)) for x in res2.scoreDocs ] res2 = [{'x': x.data.date, 'y': x.data.likes} for x in res2] return json.dumps({'histogram': res1, 'graph': res2})
def main(): _vm = lucene.initVM(vmargs=['-Djava.awt.headless=true']) query = BooleanQuery() query.add(MatchAllDocsQuery(), BooleanClause.Occur.MUST) query.add(TermQuery(Term('type', 'user')), BooleanClause.Occur.MUST) i = 0 with zh_iatd.create_searcher() as searcher: with open('pagerank_data.txt', 'w') as fout: reslst = searcher.searcher.search(query, 100) initval = 1.0 / reslst.totalHits while len(reslst.scoreDocs) > 0: for x in reslst.scoreDocs: realdoc = searcher.searcher.doc(x.doc) obj = document_to_obj(realdoc) if not obj.data.followed_users is None: print '{0:8}'.format(i), ' user', obj.index, len( obj.data.followed_users) fout.write('{0}\t{1}\t{2}\n'.format( obj.index, initval, ' '.join( (x.encode('utf8') for x in obj.data.followed_users)))) else: print '{0:8}'.format(i), 'I user', obj.index i += 1 reslst = searcher.searcher.searchAfter(reslst.scoreDocs[-1], query, 100)
def main(): _vm = lucene.initVM(vmargs=['-Djava.awt.headless=true']) db_writer = zh_iatd.create_index_writer('.newdb') db_reader = zh_iatd.create_searcher(INDEXED_FOLDER) if len(sys.argv) < 2: res = db_reader.searcher.search(MatchAllDocsQuery(), 100) tot = 0 while len(res.scoreDocs) > 0: for x in res.scoreDocs: realdoc = db_reader.searcher.doc(x.doc) obj = document_to_obj(realdoc) newdoc = obj_to_document(obj) db_writer.addDocument(newdoc) tot += 1 sys.stdout.write('\r{0}'.format(tot)) sys.stdout.flush() res = db_reader.searcher.searchAfter(res.scoreDocs[-1], MatchAllDocsQuery(), 100) elif sys.argv[1] == 'mergerank': ranks = {} with open('prrank.txt', 'r') as fin: for x in fin.readlines(): v = x.split() ranks[v[0]] = float(v[1]) res = db_reader.searcher.search(MatchAllDocsQuery(), 100) tot = 0 while len(res.scoreDocs) > 0: for x in res.scoreDocs: realdoc = db_reader.searcher.doc(x.doc) obj = document_to_obj(realdoc) if isinstance(obj, zh_pganlz.user): if obj.index in ranks.keys(): obj.data.rank = ranks[obj.index] newdoc = obj_to_document(obj) db_writer.addDocument(newdoc) tot += 1 sys.stdout.write('\r{0}'.format(tot)) sys.stdout.flush() res = db_reader.searcher.searchAfter(res.scoreDocs[-1], MatchAllDocsQuery(), 100) db_writer.commit()
def main(): _vm = lucene.initVM(vmargs=['-Djava.awt.headless=true']) db_writer = zh_iatd.create_index_writer('.newdb') db_reader = zh_iatd.create_searcher(INDEXED_FOLDER) res = db_reader.searcher.search(MatchAllDocsQuery(), 100) tot = 0 while len(res.scoreDocs) > 0: for x in res.scoreDocs: realdoc = db_reader.searcher.doc(x.doc) obj = document_to_obj(realdoc) if isinstance(obj, zh_pganlz.article): if 'contents' in vars(obj.data).keys(): obj.data.text = obj.data.contents del obj.data.contents newdoc = obj_to_document(obj) db_writer.addDocument(newdoc) tot += 1 sys.stdout.write('\r{0}'.format(tot)) sys.stdout.flush() res = db_reader.searcher.searchAfter(res.scoreDocs[-1], MatchAllDocsQuery(), 100) db_writer.commit()
def POST(self): def build_text_query(k, v): return QueryParser(k, WhitespaceAnalyzer()).parse(' '.join( jieba.lcut(v))) def build_anyterm_query(field, strv): res = BooleanQuery() for i in strv.split(): res.add(TermQuery(Term(field, i)), BooleanClause.Occur.SHOULD) return res def get_query_result(sarc, dct): PAGE_SIZE = 10 PAGE_JUMP = 10 query = BooleanQuery() query.add(TermQuery(Term(zh_pganlz.LTPF_TYPE, '1')), BooleanClause.Occur.MUST) page = 0 sort_lists = [] summ_set = set() exclus_set = None words = [] for k, v in dct.items(): if k in ('index', 'type', 'tag_indices', 'author_index'): query.add(build_anyterm_query(k, dct[k]), BooleanClause.Occur.MUST) elif k in ('text', 'contents', 'title', 'description', 'alias'): words += jieba.lcut(v) query.add( build_text_query(k + zh_pganlz.LTPF_FOR_QUERY, dct[k]), BooleanClause.Occur.MUST) elif k == 'raw': query.add( QueryParser('index', WhitespaceAnalyzer()).parse(dct[k]), BooleanClause.Occur.MUST) elif k == 'enhraw': x = 0 reslst = [] for entry in v: if x == 2: reslst += [ lastdoc + x.encode('utf8') for x in jieba.cut(entry) ] x = 0 else: if x == 0: reslst.append(entry.encode('utf8')) else: lastdoc = entry.encode('utf8') x += 1 query.add( QueryParser('index', WhitespaceAnalyzer()).parse( ' '.join(reslst)), BooleanClause.Occur.MUST) elif k == 'page': page = int(dct[k]) elif k == 'sort': for x in dct['sort']: sort_type = SortField.Type.STRING if 'type' in x.keys(): if x['type'] == 'int': sort_type = SortField.Type.INT elif x['type'] == 'float': sort_type = SortField.Type.FLOAT reverse = False if 'reverse' in x.keys(): reverse = x['reverse'] sort_lists.append( SortField(x['key'], sort_type, reverse)) elif k == 'summarize': summ_set = set(v) elif k == 'exclusive': exclus_set = set(v) ressrt = Sort(*sort_lists) resdocs = sarc.searcher.search(query, PAGE_SIZE, ressrt) if page > 0: if resdocs.totalHits > page * PAGE_SIZE: page -= 1 while page > PAGE_JUMP: resdocs = sarc.searcher.searchAfter( resdocs.scoreDocs[-1], query, PAGE_SIZE * PAGE_JUMP, ressrt) page -= PAGE_JUMP if page > 0: resdocs = sarc.searcher.searchAfter( resdocs.scoreDocs[-1], query, PAGE_SIZE * page, ressrt) resdocs = sarc.searcher.searchAfter( resdocs.scoreDocs[-1], query, PAGE_SIZE, ressrt) else: resdocs.scoreDocs = [] reslst = [] for x in resdocs.scoreDocs: dictobj = zh_pganlz.obj_to_json( zh_pganlz.document_to_obj(sarc.searcher.doc(x.doc))) if 'additional' in dct.keys(): adres = [] for x in dct['additional']: if isinstance(dictobj[x['sourcefield']], list): qlist = dictobj[x['sourcefield']] else: qlist = [dictobj[x['sourcefield']]] cres = [] for qword in qlist: if not isinstance(qword, (unicode, str)): qword = str(qword) searchres = sarc.searcher.search( zh_iatd.create_query({ 'type': x['type'], x['targetfield']: qword }), 1) if searchres.totalHits > 1: print x, 'FOUND', searchres elif searchres.totalHits == 0: cres.append(None) else: cres.append( zh_pganlz.obj_to_json( zh_pganlz.document_to_obj( sarc.searcher.doc( searchres.scoreDocs[0].doc)))) adres.append(cres) for k, v in dictobj.items(): if k in summ_set: dictobj[k + '_summary'] = summarize( hyper_text(v).text, list(set(words))) if not exclus_set is None: for k in dictobj.keys(): if not k in exclus_set: del dictobj[k] if 'additional' in dct.keys(): dictobj['additional'] = adres reslst.append(dictobj) return {'total': resdocs.totalHits, 'data': reslst} global _vm _vm.attachCurrentThread() user_data = web.input() print user_data user_data = json.loads(user_data['data']) print user_data searcher = zh_iatd.create_searcher() print 'querys' in user_data if 'querys' in user_data: reslst = [] for x in user_data['querys']: reslst.append(get_query_result(searcher, x)) print len(reslst) print json.dumps({'results': reslst}) return json.dumps({'results': reslst}) else: print get_query_result(searcher, user_data) return json.dumps(get_query_result(searcher, user_data))
def crawl_until_stop(session): global _vm, _stop, _stopped _vm.attachCurrentThread() db_writer = zh_iatd.create_index_writer() info_logger = external_console_logger('/tmp/zh_c_info') error_logger = external_console_logger('/tmp/zh_c_err') strategy = crawl_strategy() errcount = 0 while not _stop: info_logger.write(' acquiring new tasks... ') task_reader = zh_iatd.create_searcher(TASK_FOLDER) default_query = BooleanQuery() default_query.add(TermQuery(Term('finish_time', '0')), BooleanClause.Occur.MUST) strategy.process_query(default_query) idstart = task_reader.reader.numDocs() searchres = task_reader.searcher.search(default_query, 100) resdocs = [ task_reader.searcher.doc(x.doc) for x in searchres.scoreDocs ] info_logger.write('got:{0} total:{1}\n'.format(searchres.totalHits, idstart)) task_reader.close() task_writer = zh_iatd.create_index_writer(TASK_FOLDER) for doct in resdocs: curt = task() curt.from_document(doct) crlt = curt.to_crawler_task() try: crlt.func(session, crlt) except Exception as e: info_logger.write('FAIL') error_logger.write( '## ERROR ################################\n') zh_pganlz.print_object(crlt, out=error_logger) error_logger.write( '-- stacktrace ---------------------------\n') error_logger.write(traceback.format_exc()) errcount += 1 error_logger.write('[Error count: {0}]\n'.format(errcount)) task_writer.deleteDocuments(Term('docid', str(doct['docid']))) curt.fails += 1 task_writer.addDocument(curt.to_document()) else: if not crlt.result_rep_obj is None: db_writer.deleteDocuments(crlt.result_query) db_writer.addDocument( zh_pganlz.obj_to_document(crlt.result_rep_obj)) for x in crlt.result_new: db_writer.addDocument(zh_pganlz.obj_to_document(x)) db_writer.commit() task_writer.deleteDocuments(Term('docid', str(doct['docid']))) curt.finish_time = int(time.time()) task_writer.addDocument(curt.to_document()) for x in crlt.result_tasks: newt = task() newt.from_crawler_task(x) newt.docid = idstart idstart += 1 task_writer.addDocument(newt.to_document()) if isinstance(crlt.prm_id, unicode): prids = crlt.prm_id.encode('utf8') else: prids = str(crlt.prm_id) info_logger.write( ' ~{0}(+{1}) -{2} {3}({4}, {5}, {6}, {7})\n'.format( task_writer.numDocs(), len(crlt.result_tasks), curt.fails, crlt.func.func_name[14:], prids, crlt.prm_start, crlt.prm_pagesize, crlt.prm_extra)) if _stop: break time.sleep(1) task_writer.close() info_logger.write('stopped\n') _stopped = True
def index_images_until_stop(session, handler, lbound): global _stop, _stopped, _vm _vm.attachCurrentThread() searcher = IndexSearcher( DirectoryReader.open(SimpleFSDirectory(File(TASK_FOLDER)))) query = BooleanQuery() query.add(TermQuery(Term('finish_time', '0')), BooleanClause.Occur.MUST_NOT) query.add(MatchAllDocsQuery(), BooleanClause.Occur.MUST) if not lbound is None: query.add( TermRangeQuery.newStringRange('finish_time', lbound, '9999999999', False, True), BooleanClause.Occur.MUST) sort = Sort(SortField('finish_time', SortField.Type.INT)) tmpbk = None res = searcher.search(query, 100, sort) answer_content_searcher = zh_iatd.create_searcher() logger = external_console_logger('/tmp/zh_imgc_info') while not _stop: print 'got', len(res.scoreDocs), 'docs' for x in res.scoreDocs: try: imgsgot = 0 realdoc = searcher.doc(x.doc) doctype = realdoc['func_name'] objid = realdoc['id'] logger.write(' ft:{0}'.format(realdoc['finish_time'])) if doctype == 'user_data': soup = bs4.BeautifulSoup( session.opener.open( urllib2.Request( url='https://www.zhihu.com/people/{0}'.format( objid))), HTML_PARSER) cover = soup.select( '#ProfileHeader .ProfileHeader-userCover img') if len(cover) > 0: cover_img = cover[0]['src'] imgsgot += 1 handler(cover_img, ZH_IMGTYPE_USERINFO_COVER, objid) avatar_img = soup.select( '#ProfileHeader .ProfileHeader-main .UserAvatar img' )[0]['src'] imgsgot += 1 handler(avatar_img, ZH_IMGTYPE_USER_AVATAR, objid) elif doctype == 'article_data': jsondata = session.get_article_content_raw(objid) if 'titleImage' in jsondata.keys(): cover_img = jsondata['titleImage'] if len(cover_img) > 0: imgsgot += 1 handler(cover_img, ZH_IMGTYPE_ARTICLE_COVER, objid) soup = bs4.BeautifulSoup(jsondata['content'], HTML_PARSER) for x in soup.select('img'): imgsgot += 1 handler(x['src'], ZH_IMGTYPE_IN_ARTICLE, objid) elif doctype == 'topic_data': soup = bs4.BeautifulSoup( session.opener.open( urllib2.Request( url='https://www.zhihu.com/topic/{0}/hot'. format(objid))), HTML_PARSER) topic_img = soup.select( '.zu-main-content .topic-avatar .zm-entry-head-avatar-link img' )[0]['src'] imgsgot += 1 handler(topic_img, ZH_IMGTYPE_TOPIC_ICON, objid) elif doctype == 'answer_comments' and realdoc['start'] == '0': obj, q = zh_iatd.query_object(answer_content_searcher, objid, zh_pganlz.answer) for x in obj.data.text.as_soup().select('img'): imgsgot += 1 handler(x['src'], ZH_IMGTYPE_IN_ANSWER, objid) elif doctype == 'question_data': soup = bs4.BeautifulSoup( session.opener.open( urllib2.Request( url='https://www.zhihu.com/question/{0}'. format(objid))), HTML_PARSER) for x in soup.select('#zh-question-detail img'): imgsgot += 1 handler(x['src'], ZH_IMGTYPE_IN_QUESTION, objid) else: logger.write('\n') continue logger.write(' ({0}, +{1})\n'.format(doctype, imgsgot)) if _stop: break time.sleep(3) except Exception as e: logger.write('\n## ERROR ################################\n') logger.write(traceback.format_exc()) if len(res.scoreDocs) > 0: tmpbk = res.scoreDocs[-1] res = searcher.searchAfter(tmpbk, query, 100, sort) print 'stopped' _stopped = True