def search_keyword(keyword, keyword_id): crawler.info('We are searching keyword "{}"'.format(keyword)) cur_page = 1 encode_keyword = url_parse.quote(keyword) while cur_page < LIMIT: cur_url = URL.format(encode_keyword, cur_page) # current only for login, maybe later crawling page one without login search_page = get_page(cur_url, auth_level=2) if not search_page: crawler.warning('No search result for keyword {}, the source page is {}'.format(keyword, search_page)) return search_list = parse_search.get_search_info(search_page) # Because the search results are sorted by time, if any result has been stored in mysql, # We need not crawl the same keyword in this turn for wb_data in search_list: rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id) KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id) if rs: crawler.info('Weibo {} has been crawled, skip it.'.format(wb_data.weibo_id)) continue else: WbDataOper.add_one(wb_data) app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid,), queue='user_crawler', routing_key='for_user_info') if cur_page == 1: cur_page += 1 elif 'noresult_tit' not in search_page: cur_page += 1 else: crawler.info('Keyword {} has been crawled in this turn'.format(keyword)) return
def crawl_praise_page(mid): # 这里为了马上拿到返回结果,采用本地调用的方式 cur_time = int(time.time() * 1000) cur_url = BASE_URL.format(mid, cur_time) html = get_page(cur_url, auth_level=2, is_ajax=True) praise_data, ext_param = praise.get_praise_list(html, mid) PraiseOper.add_all(praise_data) WbDataOper.set_weibo_praise_crawled(mid) if not ext_param: crawler.warning( 'fail to get praise page 2 ext_param, mid is {mid}'.format( mid=mid)) return # why no app.send_task and fall back to sequential execution # because weibo praise now require a parameter called max_id # and request without it will return something different from normal browser # should work after 5 # TODO: retry or return depending on ext_param for __ in range(2, 5): # ext_param mainly max_id will be updated each time and be used next time html, praise_data, ext_param = crawl_praise_by_page(mid, ext_param) return
def get_redirect(name, data, post_url, session, proxy): logining_page = session.post(post_url, data=data, headers=headers, proxies=proxy) login_loop = logining_page.content.decode("GBK") # if name or password is wrong, set the value to 2 if 'retcode=101' in login_loop: crawler.error( 'invalid password for {}, please ensure your account and password'. format(name)) LoginInfoOper.freeze_account(name, 2) return '' if 'retcode=2070' in login_loop: crawler.error('invalid verification code') return 'pinerror' if 'retcode=4049' in login_loop: crawler.warning('account {} need verification for login'.format(name)) return 'login_need_pincode' if '正在登录' in login_loop or 'Signing in' in login_loop: pa = r'location\.replace\([\'"](.*?)[\'"]\)' return re.findall(pa, login_loop)[0] else: return ''
def search_items_v2(keyword, keyword_id, date_item): search_time_list = [ "{}-{}:{}-{}".format(d, t, d, t + 2) for d, t in itertools.product([date_item], TIME_LIIT) ] for s_time in search_time_list: crawler.info('We are searching keyword "{}", {}'.format( keyword, s_time)) cur_page = 1 encode_keyword = url_parse.quote(keyword) while cur_page < LIMIT: cur_url = MAX_URL.format(encode_keyword, cur_page, s_time) # current only for login, maybe later crawling page one without login search_page = get_page(cur_url, auth_level=1, need_proxy=True) if "您可以尝试更换关键词,再次搜索" in search_page: break if not search_page: crawler.warning( 'No search result for keyword {}, the source page is {}'. format(keyword, search_page)) cur_page += 1 continue # return search_list = parse_search.get_search_info(search_page) if cur_page == 1: cur_page += 1 elif 'noresult_tit' not in search_page: cur_page += 1 else: crawler.info( 'Keyword {} has been crawled in this turn'.format(keyword)) return # Because the search results are sorted by time, if any result has been stored in mysql, # We don't need to crawl the same keyword in this turn for wb_data in search_list: # print(wb_data) rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id) KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id) # todo incremental crawling using time if rs: crawler.info('Weibo {} has been crawled, skip it.'.format( wb_data.weibo_id)) continue else: WbDataOper.add_one(wb_data) # todo: only add seed ids and remove this task app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid, ), queue='user_crawler', routing_key='for_user_info')
def crawl_weibo_datas(uid): limit = get_max_home_page() cur_page = 1 while cur_page <= limit: url = HOME_URL.format(uid, cur_page) if cur_page == 1: html = get_page(url, auth_level=1) else: html = get_page(url, auth_level=2) weibo_datas = get_data(html) if not weibo_datas: crawler.warning("user {} has no weibo".format(uid)) return # Check whether weibo created after time in spider.yaml timeafter = time.mktime( time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S')) length_weibo_datas = len(weibo_datas) for i in range(0, len(weibo_datas)): weibo_time = time.mktime( time.strptime(weibo_datas[i].create_time, '%Y-%m-%d %H:%M')) if weibo_time < timeafter: weibo_datas = weibo_datas[0:i] break WbDataOper.add_all(weibo_datas) # If the weibo isn't created after the given time, jump out the loop if i != length_weibo_datas - 1: break domain = public.get_userdomain(html) cur_time = int(time.time()*1000) ajax_url_0 = AJAX_URL.format(domain, 0, domain, uid, cur_page, cur_page, cur_time) ajax_url_1 = AJAX_URL.format(domain, 1, domain, uid, cur_page, cur_page, cur_time+100) if cur_page == 1: # here we use local call to get total page number total_page = get_total_page(crawl_ajax_page(ajax_url_1, 2)) auth_level = 1 else: auth_level = 2 if total_page < limit: limit = total_page app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_0, auth_level), queue='ajax_home_crawler', routing_key='ajax_home_info') app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_1, auth_level), queue='ajax_home_crawler', routing_key='ajax_home_info') cur_page += 1 SeedidsOper.set_seed_home_crawled(uid)
def check_cookies_timeout(cls, cookies): if cookies is None: return True if isinstance(cookies, bytes): cookies = cookies.decode('utf-8') cookies = json.loads(cookies) login_time = datetime.datetime.fromtimestamp(cookies['loginTime']) if datetime.datetime.now() - login_time > datetime.timedelta(hours=cookie_expire_time): crawler.warning('The account has been expired') return True return False
def check_cookies_timeout(cls, cookies): if cookies is None: return True if isinstance(cookies, bytes): cookies = cookies.decode('utf-8') cookies = json.loads(cookies) login_time = datetime.datetime.fromtimestamp(cookies['loginTime']) if datetime.datetime.now() - login_time > datetime.timedelta(hours=cookie_expire_time): crawler.warning('The account has been expired') return True return False
def search_keyword(keyword, keyword_id): cur_page = 1 encode_keyword = url_parse.quote(keyword) while cur_page < LIMIT: cur_url = URL.format(encode_keyword, cur_page) if cur_page == 1: search_page = get_page(cur_url, auth_level=1) else: search_page = get_page(cur_url, auth_level=2) if not search_page: crawler.warning( 'No result for keyword {}, the source page is {}'.format( keyword, search_page)) return search_list = parse_search.get_search_info(search_page) # Because the search results are sorted by time, if any result has been stored in mysql, # we need not crawl the same keyword in this turn for wb_data in search_list: rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id) if rs: crawler.info( 'Keyword {} has been crawled in this turn'.format(keyword)) return else: WbDataOper.add_one(wb_data) KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id) # send task for crawling user info app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid, ), queue='user_crawler', routing_key='for_user_info') if cur_page == 1: cur_page += 1 elif 'page next S_txt1 S_line1' in search_page: cur_page += 1 else: crawler.info( 'Keyword {} has been crawled in this turn'.format(keyword)) return
def crawl_weibo_datas(uid): limit = get_max_home_page() cur_page = 1 while cur_page <= limit: url = HOME_URL.format(uid, cur_page) if cur_page == 1: html = get_page(url, auth_level=1) else: html = get_page(url, auth_level=2) weibo_datas = get_data(html) if not weibo_datas: crawler.warning("user {} has no weibo".format(uid)) return WbDataOper.add_all(weibo_datas) domain = public.get_userdomain(html) cur_time = int(time.time()*1000) ajax_url_0 = AJAX_URL.format(domain, 0, domain, uid, cur_page, cur_page, cur_time) ajax_url_1 = AJAX_URL.format(domain, 1, domain, uid, cur_page, cur_page, cur_time+100) if cur_page == 1: # here we use local call to get total page number total_page = get_total_page(crawl_ajax_page(ajax_url_1, 2)) auth_level = 1 else: auth_level = 2 if total_page < limit: limit = total_page app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_0, auth_level), queue='ajax_home_crawler', routing_key='ajax_home_info') app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_1, auth_level), queue='ajax_home_crawler', routing_key='ajax_home_info') cur_page += 1 SeedidsOper.set_seed_home_crawled(uid)
def crawl_ajax_page(url, auth_level): """ :param url: user home ajax url :param auth_level: 1 stands for no login but need fake cookies, 2 stands for login :return: resp.text """ crawler.warning("crawl_ajax_page:{}".format(url)) ajax_html = get_page(url, auth_level, is_ajax=True) ajax_wbdatas = get_ajax_data(ajax_html) if not ajax_wbdatas: return '' # timeafter = time.mktime(time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S')) # for i in range(0,len(ajax_wbdatas)): # weibo_time = time.mktime(time.strptime(ajax_wbdatas[i].create_time, '%Y-%m-%d %H:%M')) # if weibo_time < timeafter: # ajax_wbdatas = ajax_wbdatas[0:i] # break WbDataOper.add_all(ajax_wbdatas) return ajax_html
def get_redirect(name, data, post_url, session, proxy): logining_page = session.post(post_url, data=data, headers=headers, proxies=proxy) login_loop = logining_page.content.decode("GBK") # if name or password is wrong, set the value to 2 if 'retcode=101' in login_loop: crawler.error('invalid password for {}, please ensure your account and password'.format(name)) LoginInfoOper.freeze_account(name, 2) return '' if 'retcode=2070' in login_loop: crawler.error('invalid verification code') return 'pinerror' if 'retcode=4049' in login_loop: crawler.warning('account {} need verification for login'.format(name)) return 'login_need_pincode' if '正在登录' in login_loop or 'Signing in' in login_loop: pa = r'location\.replace\([\'"](.*?)[\'"]\)' return re.findall(pa, login_loop)[0] else: return ''
def search_keyword(keyword, keyword_id): crawler.info('We are searching keyword "{}"'.format(keyword)) cur_page = 1 encode_keyword = url_parse.quote(keyword) while cur_page < LIMIT: cur_url = URL.format(encode_keyword, cur_page) # current only for login, maybe later crawling page one without login search_page = get_page(cur_url, auth_level=2) if not search_page: crawler.warning('No search result for keyword {}, the source page is {}'.format(keyword, search_page)) return search_list = parse_search.get_search_info(search_page) if cur_page == 1: cur_page += 1 elif 'noresult_tit' not in search_page: cur_page += 1 else: crawler.info('Keyword {} has been crawled in this turn'.format(keyword)) return # Because the search results are sorted by time, if any result has been stored in mysql, # We don't need to crawl the same keyword in this turn for wb_data in search_list: rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id) KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id) # todo incremental crawling using time if rs: crawler.info('Weibo {} has been crawled, skip it.'.format(wb_data.weibo_id)) continue else: WbDataOper.add_one(wb_data) # todo: only add seed ids and remove this task app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid,), queue='user_crawler', routing_key='for_user_info')
def get_page(url, auth_level=2, is_ajax=False, need_proxy=False): """ :param url: url to crawl :param auth_level: 0 stands for need nothing,1 stands for no login but need cookies,2 stands for need login. :param is_ajax: whether the request is ajax :param need_proxy: whether the request need a http/https proxy :return: response text, when a exception is raised, return '' """ crawler.info('the crawling url is {url}'.format(url=url)) count = 0 while count < MAX_RETRIES: if auth_level == 2: name_cookies = Cookies.fetch_cookies() if name_cookies is None: crawler.warning( 'No cookie in cookies pool. Maybe all accounts are banned, or all cookies are expired' ) send_email() os.kill(os.getppid(), signal.SIGTERM) # There is no difference between http and https address. proxy = { 'http': name_cookies[2], 'https': name_cookies[2], } else: proxy = getip.getIPWithoutLogin('') # if proxy['http'] is None: # crawler.warning('No available ip in ip pools. Using local ip instead.') try: if auth_level == 2: resp = requests.get(url, headers=headers, cookies=name_cookies[1], timeout=TIME_OUT, verify=False, proxies=proxy) elif auth_level == 1: resp = requests.get(url, headers=headers, cookies=COOKIES, timeout=TIME_OUT, verify=False, proxies=proxy) else: resp = requests.get(url, headers=headers, timeout=TIME_OUT, verify=False, proxies=proxy) except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError, AttributeError) as e: crawler.warning( 'Excepitons are raised when crawling {}.Here are details:{}'. format(url, e)) count += 1 time.sleep(EXCP_INTERAL) continue if resp.status_code == 414: crawler.warning('This ip has been blocked by weibo system') if not need_proxy: send_email() os.kill(os.getppid(), signal.SIGTERM) if resp.text: page = resp.text.encode('utf-8', 'ignore').decode('utf-8') else: count += 1 continue if auth_level == 2: # slow down to aviod being banned time.sleep(INTERAL) if is_banned(resp.url) or is_403(page): crawler.warning('Account {} has been banned'.format( name_cookies[0])) LoginInfoOper.freeze_account(name_cookies[0], 0) Cookies.delete_cookies(name_cookies[0]) count += 1 continue if not is_ajax and not is_complete(page): count += 1 continue if is_404(page): crawler.warning('{} seems to be 404'.format(url)) return '' Urls.store_crawl_url(url, 1) return page Urls.store_crawl_url(url, 0) return ''
def crawl_weibo_datas(uid): limit = get_max_home_page() cur_page = 1 while cur_page <= limit: url = HOME_URL.format(uid, cur_page) if cur_page == 1: html = get_page(url, auth_level=1) else: html = get_page(url, auth_level=2) weibo_datas = get_data(html) if not weibo_datas: crawler.warning("user {} has no weibo".format(uid)) return # Check whether weibo created after time in spider.yaml length_weibo_datas = len(weibo_datas) timeafter = time.mktime( time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S')) weibo_datas = [ weibo_datum for weibo_datum in weibo_datas if determine(weibo_datum, timeafter) ] WbDataOper.add_all(weibo_datas) # If the weibo isn't created after the given time, jump out the loop if len(weibo_datas) != length_weibo_datas: break domain = public.get_userdomain(html) cur_time = int(time.time() * 1000) ajax_url_0 = AJAX_URL.format(domain, 0, domain, uid, cur_page, cur_page, cur_time) ajax_url_1 = AJAX_URL.format(domain, 1, domain, uid, cur_page, cur_page, cur_time + 100) if cur_page == 1: # here we use local call to get total page number total_page = get_total_page(crawl_ajax_page(ajax_url_1, 2)) auth_level = 1 if total_page < limit: limit = total_page # Since the second ajax of page 1 has already been crawled # in the code above and has been stored in databse, # we only have to crawl the first ajax of page 1 crawl_ajax_page(ajax_url_0, auth_level) else: auth_level = 2 # Still the same as before # if total_page != limit: # limit = total_page # crawler.warning("total pagenum is {}".format(total_page)) crawl_ajax_page(ajax_url_0, auth_level) crawl_ajax_page(ajax_url_1, auth_level) cur_page += 1 SeedidsOper.set_seed_home_crawled(uid)
def get_page(url, auth_level=2, is_ajax=False, need_proxy=False): """ :param url: url to crawl :param auth_level: 0 stands for need nothing,1 stands for no login but need cookies,2 stands for need login. :param is_ajax: whether the request is ajax :param need_proxy: whether the request need a http/https proxy :return: response text, when a exception is raised, return '' """ crawler.info('the crawling url is {url}'.format(url=url)) count = 0 while count < MAX_RETRIES: if auth_level == 2: name_cookies = Cookies.fetch_cookies() if name_cookies is None: crawler.warning('No cookie in cookies pool. Maybe all accounts are banned, or all cookies are expired') send_email() os.kill(os.getppid(), signal.SIGTERM) # There is no difference between http and https address. proxy = {'http': name_cookies[2], 'https': name_cookies[2], } else: proxy = getip.getIPWithoutLogin('') # if proxy['http'] is None: # crawler.warning('No available ip in ip pools. Using local ip instead.') try: if auth_level == 2: resp = requests.get(url, headers=headers, cookies=name_cookies[1], timeout=TIME_OUT, verify=False, proxies=proxy) elif auth_level == 1: resp = requests.get(url, headers=headers, cookies=COOKIES, timeout=TIME_OUT, verify=False, proxies=proxy) else: resp = requests.get(url, headers=headers, timeout=TIME_OUT, verify=False, proxies=proxy) except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError, AttributeError) as e: crawler.warning('Excepitons are raised when crawling {}.Here are details:{}'.format(url, e)) count += 1 time.sleep(EXCP_INTERAL) continue if resp.status_code == 414: crawler.warning('This ip has been blocked by weibo system') if not need_proxy: send_email() os.kill(os.getppid(), signal.SIGTERM) if resp.text: page = resp.text.encode('utf-8', 'ignore').decode('utf-8') else: count += 1 continue if auth_level == 2: # slow down to aviod being banned time.sleep(INTERAL) if is_banned(resp.url) or is_403(page): crawler.warning('Account {} has been banned'.format(name_cookies[0])) LoginInfoOper.freeze_account(name_cookies[0], 0) Cookies.delete_cookies(name_cookies[0]) count += 1 continue if not is_ajax and not is_complete(page): count += 1 continue if is_404(page): crawler.warning('{} seems to be 404'.format(url)) return '' Urls.store_crawl_url(url, 1) return page Urls.store_crawl_url(url, 0) return ''
def crawl_weibo_datas(uid): limit = get_max_home_page() cur_page = 1 while cur_page <= limit: url = HOME_URL.format(uid, cur_page) if cur_page == 1: html = get_page(url, auth_level=1) else: html = get_page(url, auth_level=2) weibo_datas = get_data(html) if not weibo_datas: crawler.warning("user {} has no weibo".format(uid)) return # Check whether weibo created after time in spider.yaml length_weibo_datas = len(weibo_datas) timeafter = time.mktime( time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S')) weibo_datas = [ weibo_datum for weibo_datum in weibo_datas if determine(weibo_datum, timeafter) ] WbDataOper.add_all(weibo_datas) # If the weibo isn't created after the given time, jump out the loop if len(weibo_datas) != length_weibo_datas: break domain = public.get_userdomain(html) cur_time = int(time.time()*1000) ajax_url_0 = AJAX_URL.format(domain, 0, domain, uid, cur_page, cur_page, cur_time) ajax_url_1 = AJAX_URL.format(domain, 1, domain, uid, cur_page, cur_page, cur_time+100) if cur_page == 1: # here we use local call to get total page number total_page = get_total_page(crawl_ajax_page(ajax_url_1, 2)) auth_level = 1 if total_page < limit: limit = total_page # Since the second ajax of page 1 has already been crawled # in the code above and has been stored in databse, # we only have to crawl the first ajax of page 1 crawl_ajax_page(ajax_url_0, auth_level) else: auth_level = 2 # Still the same as before # if total_page != limit: # limit = total_page # crawler.warning("total pagenum is {}".format(total_page)) crawl_ajax_page(ajax_url_0, auth_level) crawl_ajax_page(ajax_url_1, auth_level) cur_page += 1 SeedidsOper.set_seed_home_crawled(uid)
def crawl_weibo_datas(uid): limit = get_max_home_page() cur_page = 36 retry_count = 1 while cur_page <= 36: crawler.warning("current page {}".format(cur_page)) url = HOME_URL.format(uid, cur_page) #if cur_page == 1: # html = get_page(url, auth_level=1) #else: html = get_page(url, auth_level=2) weibo_datas = get_data(html) if not weibo_datas: if retry_count < 10: crawler.warning("user {} has no weibo, retry".format(uid)) retry_count = retry_count + 1 #time.sleep(240) continue; else: crawler.warning("user {} has no weibo, return".format(uid)) return # Check whether weibo created after time in spider.yaml # timeafter = time.mktime( # time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S')) # length_weibo_datas = len(weibo_datas) # for i in range(0, len(weibo_datas)): # weibo_time = time.mktime( # time.strptime(weibo_datas[i].create_time, '%Y-%m-%d %H:%M')) # if weibo_time < timeafter: # weibo_datas = weibo_datas[0:i] # break WbDataOper.add_all(weibo_datas) # # If the weibo isn't created after the given time, jump out the loop # if i != length_weibo_datas - 1: # break domain = public.get_userdomain(html) cur_time = int(time.time()*1000) ajax_url_0 = AJAX_URL.format(domain, 0, domain, uid, cur_page, cur_page, cur_time) ajax_url_1 = AJAX_URL.format(domain, 1, domain, uid, cur_page, cur_page, cur_time+100) # if cur_page == 1: # # here we use local call to get total page number # total_page = get_total_page(crawl_ajax_page(ajax_url_1, 2)) # auth_level = 1 # else: auth_level = 2 #if total_page < limit: # limit = total_page crawler.warning("append tasks.home.crawl_ajax_page{}".format(uid)); app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_0, auth_level), queue='ajax_home_crawler', routing_key='ajax_home_info') app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_1, auth_level), queue='ajax_home_crawler', routing_key='ajax_home_info') cur_page += 1 SeedidsOper.set_seed_home_crawled(uid)