def search_keyword(keyword, keyword_id): crawler.info('We are searching keyword "{}"'.format(keyword)) cur_page = 1 encode_keyword = url_parse.quote(keyword) while cur_page < LIMIT: cur_url = URL.format(encode_keyword, cur_page) # current only for login, maybe later crawling page one without login search_page = get_page(cur_url, auth_level=2) if not search_page: crawler.warning('No search result for keyword {}, the source page is {}'.format(keyword, search_page)) return search_list = parse_search.get_search_info(search_page) # Because the search results are sorted by time, if any result has been stored in mysql, # We need not crawl the same keyword in this turn for wb_data in search_list: rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id) KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id) if rs: crawler.info('Weibo {} has been crawled, skip it.'.format(wb_data.weibo_id)) continue else: WbDataOper.add_one(wb_data) app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid,), queue='user_crawler', routing_key='for_user_info') if cur_page == 1: cur_page += 1 elif 'noresult_tit' not in search_page: cur_page += 1 else: crawler.info('Keyword {} has been crawled in this turn'.format(keyword)) return
def search_items_v2(keyword, keyword_id, date_item): search_time_list = [ "{}-{}:{}-{}".format(d, t, d, t + 2) for d, t in itertools.product([date_item], TIME_LIIT) ] for s_time in search_time_list: crawler.info('We are searching keyword "{}", {}'.format( keyword, s_time)) cur_page = 1 encode_keyword = url_parse.quote(keyword) while cur_page < LIMIT: cur_url = MAX_URL.format(encode_keyword, cur_page, s_time) # current only for login, maybe later crawling page one without login search_page = get_page(cur_url, auth_level=1, need_proxy=True) if "您可以尝试更换关键词,再次搜索" in search_page: break if not search_page: crawler.warning( 'No search result for keyword {}, the source page is {}'. format(keyword, search_page)) cur_page += 1 continue # return search_list = parse_search.get_search_info(search_page) if cur_page == 1: cur_page += 1 elif 'noresult_tit' not in search_page: cur_page += 1 else: crawler.info( 'Keyword {} has been crawled in this turn'.format(keyword)) return # Because the search results are sorted by time, if any result has been stored in mysql, # We don't need to crawl the same keyword in this turn for wb_data in search_list: # print(wb_data) rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id) KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id) # todo incremental crawling using time if rs: crawler.info('Weibo {} has been crawled, skip it.'.format( wb_data.weibo_id)) continue else: WbDataOper.add_one(wb_data) # todo: only add seed ids and remove this task app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid, ), queue='user_crawler', routing_key='for_user_info')
def search_keyword(keyword, keyword_id): cur_page = 1 encode_keyword = url_parse.quote(keyword) while cur_page < LIMIT: cur_url = URL.format(encode_keyword, cur_page) if cur_page == 1: search_page = get_page(cur_url, auth_level=1) else: search_page = get_page(cur_url, auth_level=2) if not search_page: crawler.warning( 'No result for keyword {}, the source page is {}'.format( keyword, search_page)) return search_list = parse_search.get_search_info(search_page) # Because the search results are sorted by time, if any result has been stored in mysql, # we need not crawl the same keyword in this turn for wb_data in search_list: rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id) if rs: crawler.info( 'Keyword {} has been crawled in this turn'.format(keyword)) return else: WbDataOper.add_one(wb_data) KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id) # send task for crawling user info app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid, ), queue='user_crawler', routing_key='for_user_info') if cur_page == 1: cur_page += 1 elif 'page next S_txt1 S_line1' in search_page: cur_page += 1 else: crawler.info( 'Keyword {} has been crawled in this turn'.format(keyword)) return
def search_keyword_topic(keyword, keyword_id, start_time='', end_time=''): crawler.info( 'We are crawling weibo topic content with keyword "{}"'.format( keyword)) cur_page = 1 encode_keyword = url_parse.quote(keyword) while cur_page < LIMIT: cur_url = URL.format(encode_keyword, start_time, end_time, cur_page) search_page = get_page(cur_url, auth_level=2) if not search_page: crawler.info( 'No such result for keyword {}, the source page is {}'.format( keyword, search_page)) return search_list = parse_topic.get_search_info(search_page) if cur_page == 1: cur_page += 1 elif '您可以尝试更换关键词' not in search_page: cur_page += 1 else: crawler.info( 'Keyword {} has been crawled in this turn'.format(keyword)) return for wb_data in search_list: rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id) KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id) if rs: crawler.info('Weibo {} has been crawled, skip it.'.format( wb_data.weibo_id)) continue else: WbDataOper.add_one(wb_data) app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid, ), queue='user_crawler', routing_key='for_user_info')
def search_keyword(keyword, keyword_id): crawler.info('We are searching keyword "{}"'.format(keyword)) cur_page = 1 encode_keyword = url_parse.quote(keyword) while cur_page < LIMIT: cur_url = URL.format(encode_keyword, cur_page) # current only for login, maybe later crawling page one without login search_page = get_page(cur_url, auth_level=2) if not search_page: crawler.warning('No search result for keyword {}, the source page is {}'.format(keyword, search_page)) return search_list = parse_search.get_search_info(search_page) if cur_page == 1: cur_page += 1 elif 'noresult_tit' not in search_page: cur_page += 1 else: crawler.info('Keyword {} has been crawled in this turn'.format(keyword)) return # Because the search results are sorted by time, if any result has been stored in mysql, # We don't need to crawl the same keyword in this turn for wb_data in search_list: rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id) KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id) # todo incremental crawling using time if rs: crawler.info('Weibo {} has been crawled, skip it.'.format(wb_data.weibo_id)) continue else: WbDataOper.add_one(wb_data) # todo: only add seed ids and remove this task app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid,), queue='user_crawler', routing_key='for_user_info')