def search_keyword(keyword, keyword_id):
crawler.info('We are searching keyword "{}"'.format(keyword))
cur_page = 1
encode_keyword = url_parse.quote(keyword)
while cur_page < LIMIT:
cur_url = URL.format(encode_keyword, cur_page)
# current only for login, maybe later crawling page one without login
search_page = get_page(cur_url, auth_level=2)
if not search_page:
crawler.warning('No search result for keyword {}, the source page is {}'.format(keyword, search_page))
return
search_list = parse_search.get_search_info(search_page)
# Because the search results are sorted by time, if any result has been stored in mysql,
# We need not crawl the same keyword in this turn
for wb_data in search_list:
rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id)
KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id)
if rs:
crawler.info('Weibo {} has been crawled, skip it.'.format(wb_data.weibo_id))
continue
else:
WbDataOper.add_one(wb_data)
app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid,), queue='user_crawler',
routing_key='for_user_info')
if cur_page == 1:
cur_page += 1
elif 'noresult_tit' not in search_page:
cur_page += 1
else:
crawler.info('Keyword {} has been crawled in this turn'.format(keyword))
return
def crawl_praise_page(mid):
# 这里为了马上拿到返回结果，采用本地调用的方式
cur_time = int(time.time() * 1000)
cur_url = BASE_URL.format(mid, cur_time)
html = get_page(cur_url, auth_level=2, is_ajax=True)
praise_data, ext_param = praise.get_praise_list(html, mid)
PraiseOper.add_all(praise_data)
WbDataOper.set_weibo_praise_crawled(mid)
if not ext_param:
crawler.warning(
'fail to get praise page 2 ext_param, mid is {mid}'.format(
mid=mid))
return
# why no app.send_task and fall back to sequential execution
# because weibo praise now require a parameter called max_id
# and request without it will return something different from normal browser
# should work after 5
# TODO: retry or return depending on ext_param
for __ in range(2, 5):
# ext_param mainly max_id will be updated each time and be used next time
html, praise_data, ext_param = crawl_praise_by_page(mid, ext_param)
return
def get_redirect(name, data, post_url, session, proxy):
logining_page = session.post(post_url,
data=data,
headers=headers,
proxies=proxy)
login_loop = logining_page.content.decode("GBK")
# if name or password is wrong, set the value to 2
if 'retcode=101' in login_loop:
crawler.error(
'invalid password for {}, please ensure your account and password'.
format(name))
LoginInfoOper.freeze_account(name, 2)
return ''
if 'retcode=2070' in login_loop:
crawler.error('invalid verification code')
return 'pinerror'
if 'retcode=4049' in login_loop:
crawler.warning('account {} need verification for login'.format(name))
return 'login_need_pincode'
if '正在登录' in login_loop or 'Signing in' in login_loop:
pa = r'location\.replace\([\'"](.*?)[\'"]\)'
return re.findall(pa, login_loop)[0]
else:
return ''
def search_items_v2(keyword, keyword_id, date_item):
search_time_list = [
"{}-{}:{}-{}".format(d, t, d, t + 2)
for d, t in itertools.product([date_item], TIME_LIIT)
]
for s_time in search_time_list:
crawler.info('We are searching keyword "{}", {}'.format(
keyword, s_time))
cur_page = 1
encode_keyword = url_parse.quote(keyword)
while cur_page < LIMIT:
cur_url = MAX_URL.format(encode_keyword, cur_page, s_time)
# current only for login, maybe later crawling page one without login
search_page = get_page(cur_url, auth_level=1, need_proxy=True)
if "您可以尝试更换关键词，再次搜索" in search_page:
break
if not search_page:
crawler.warning(
'No search result for keyword {}, the source page is {}'.
format(keyword, search_page))
cur_page += 1
continue
# return
search_list = parse_search.get_search_info(search_page)
if cur_page == 1:
cur_page += 1
elif 'noresult_tit' not in search_page:
cur_page += 1
else:
crawler.info(
'Keyword {} has been crawled in this turn'.format(keyword))
return
# Because the search results are sorted by time, if any result has been stored in mysql,
# We don't need to crawl the same keyword in this turn
for wb_data in search_list:
# print(wb_data)
rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id)
KeywordsDataOper.insert_keyword_wbid(keyword_id,
wb_data.weibo_id)
# todo incremental crawling using time
if rs:
crawler.info('Weibo {} has been crawled, skip it.'.format(
wb_data.weibo_id))
continue
else:
WbDataOper.add_one(wb_data)
# todo: only add seed ids and remove this task
app.send_task('tasks.user.crawl_person_infos',
args=(wb_data.uid, ),
queue='user_crawler',
routing_key='for_user_info')
def crawl_weibo_datas(uid):
limit = get_max_home_page()
cur_page = 1
while cur_page <= limit:
url = HOME_URL.format(uid, cur_page)
if cur_page == 1:
html = get_page(url, auth_level=1)
else:
html = get_page(url, auth_level=2)
weibo_datas = get_data(html)
if not weibo_datas:
crawler.warning("user {} has no weibo".format(uid))
return
# Check whether weibo created after time in spider.yaml
timeafter = time.mktime(
time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S'))
length_weibo_datas = len(weibo_datas)
for i in range(0, len(weibo_datas)):
weibo_time = time.mktime(
time.strptime(weibo_datas[i].create_time, '%Y-%m-%d %H:%M'))
if weibo_time < timeafter:
weibo_datas = weibo_datas[0:i]
break
WbDataOper.add_all(weibo_datas)
# If the weibo isn't created after the given time, jump out the loop
if i != length_weibo_datas - 1:
break
domain = public.get_userdomain(html)
cur_time = int(time.time()*1000)
ajax_url_0 = AJAX_URL.format(domain, 0, domain, uid, cur_page, cur_page, cur_time)
ajax_url_1 = AJAX_URL.format(domain, 1, domain, uid, cur_page, cur_page, cur_time+100)
if cur_page == 1:
# here we use local call to get total page number
total_page = get_total_page(crawl_ajax_page(ajax_url_1, 2))
auth_level = 1
else:
auth_level = 2
if total_page < limit:
limit = total_page
app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_0, auth_level), queue='ajax_home_crawler',
routing_key='ajax_home_info')
app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_1, auth_level), queue='ajax_home_crawler',
routing_key='ajax_home_info')
cur_page += 1
SeedidsOper.set_seed_home_crawled(uid)
def check_cookies_timeout(cls, cookies):
if cookies is None:
return True
if isinstance(cookies, bytes):
cookies = cookies.decode('utf-8')
cookies = json.loads(cookies)
login_time = datetime.datetime.fromtimestamp(cookies['loginTime'])
if datetime.datetime.now() - login_time > datetime.timedelta(hours=cookie_expire_time):
crawler.warning('The account has been expired')
return True
return False
def check_cookies_timeout(cls, cookies):
if cookies is None:
return True
if isinstance(cookies, bytes):
cookies = cookies.decode('utf-8')
cookies = json.loads(cookies)
login_time = datetime.datetime.fromtimestamp(cookies['loginTime'])
if datetime.datetime.now() - login_time > datetime.timedelta(hours=cookie_expire_time):
crawler.warning('The account has been expired')
return True
return False
def search_keyword(keyword, keyword_id):
cur_page = 1
encode_keyword = url_parse.quote(keyword)
while cur_page < LIMIT:
cur_url = URL.format(encode_keyword, cur_page)
if cur_page == 1:
search_page = get_page(cur_url, auth_level=1)
else:
search_page = get_page(cur_url, auth_level=2)
if not search_page:
crawler.warning(
'No result for keyword {}, the source page is {}'.format(
keyword, search_page))
return
search_list = parse_search.get_search_info(search_page)
# Because the search results are sorted by time, if any result has been stored in mysql,
# we need not crawl the same keyword in this turn
for wb_data in search_list:
rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id)
if rs:
crawler.info(
'Keyword {} has been crawled in this turn'.format(keyword))
return
else:
WbDataOper.add_one(wb_data)
KeywordsDataOper.insert_keyword_wbid(keyword_id,
wb_data.weibo_id)
# send task for crawling user info
app.send_task('tasks.user.crawl_person_infos',
args=(wb_data.uid, ),
queue='user_crawler',
routing_key='for_user_info')
if cur_page == 1:
cur_page += 1
elif 'page next S_txt1 S_line1' in search_page:
cur_page += 1
else:
crawler.info(
'Keyword {} has been crawled in this turn'.format(keyword))
return
def crawl_weibo_datas(uid):
limit = get_max_home_page()
cur_page = 1
while cur_page <= limit:
url = HOME_URL.format(uid, cur_page)
if cur_page == 1:
html = get_page(url, auth_level=1)
else:
html = get_page(url, auth_level=2)
weibo_datas = get_data(html)
if not weibo_datas:
crawler.warning("user {} has no weibo".format(uid))
return
WbDataOper.add_all(weibo_datas)
domain = public.get_userdomain(html)
cur_time = int(time.time()*1000)
ajax_url_0 = AJAX_URL.format(domain, 0, domain, uid, cur_page, cur_page, cur_time)
ajax_url_1 = AJAX_URL.format(domain, 1, domain, uid, cur_page, cur_page, cur_time+100)
if cur_page == 1:
# here we use local call to get total page number
total_page = get_total_page(crawl_ajax_page(ajax_url_1, 2))
auth_level = 1
else:
auth_level = 2
if total_page < limit:
limit = total_page
app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_0, auth_level), queue='ajax_home_crawler',
routing_key='ajax_home_info')
app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_1, auth_level), queue='ajax_home_crawler',
routing_key='ajax_home_info')
cur_page += 1
SeedidsOper.set_seed_home_crawled(uid)
def crawl_ajax_page(url, auth_level):
"""
:param url: user home ajax url
:param auth_level: 1 stands for no login but need fake cookies, 2 stands for login
:return: resp.text
"""
crawler.warning("crawl_ajax_page:{}".format(url))
ajax_html = get_page(url, auth_level, is_ajax=True)
ajax_wbdatas = get_ajax_data(ajax_html)
if not ajax_wbdatas:
return ''
# timeafter = time.mktime(time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S'))
# for i in range(0,len(ajax_wbdatas)):
# weibo_time = time.mktime(time.strptime(ajax_wbdatas[i].create_time, '%Y-%m-%d %H:%M'))
# if weibo_time < timeafter:
# ajax_wbdatas = ajax_wbdatas[0:i]
# break
WbDataOper.add_all(ajax_wbdatas)
return ajax_html
def get_redirect(name, data, post_url, session, proxy):
logining_page = session.post(post_url, data=data, headers=headers, proxies=proxy)
login_loop = logining_page.content.decode("GBK")
# if name or password is wrong, set the value to 2
if 'retcode=101' in login_loop:
crawler.error('invalid password for {}, please ensure your account and password'.format(name))
LoginInfoOper.freeze_account(name, 2)
return ''
if 'retcode=2070' in login_loop:
crawler.error('invalid verification code')
return 'pinerror'
if 'retcode=4049' in login_loop:
crawler.warning('account {} need verification for login'.format(name))
return 'login_need_pincode'
if '正在登录' in login_loop or 'Signing in' in login_loop:
pa = r'location\.replace\([\'"](.*?)[\'"]\)'
return re.findall(pa, login_loop)[0]
else:
return ''
def search_keyword(keyword, keyword_id):
crawler.info('We are searching keyword "{}"'.format(keyword))
cur_page = 1
encode_keyword = url_parse.quote(keyword)
while cur_page < LIMIT:
cur_url = URL.format(encode_keyword, cur_page)
# current only for login, maybe later crawling page one without login
search_page = get_page(cur_url, auth_level=2)
if not search_page:
crawler.warning('No search result for keyword {}, the source page is {}'.format(keyword, search_page))
return
search_list = parse_search.get_search_info(search_page)
if cur_page == 1:
cur_page += 1
elif 'noresult_tit' not in search_page:
cur_page += 1
else:
crawler.info('Keyword {} has been crawled in this turn'.format(keyword))
return
# Because the search results are sorted by time, if any result has been stored in mysql,
# We don't need to crawl the same keyword in this turn
for wb_data in search_list:
rs = WbDataOper.get_wb_by_mid(wb_data.weibo_id)
KeywordsDataOper.insert_keyword_wbid(keyword_id, wb_data.weibo_id)
# todo incremental crawling using time
if rs:
crawler.info('Weibo {} has been crawled, skip it.'.format(wb_data.weibo_id))
continue
else:
WbDataOper.add_one(wb_data)
# todo: only add seed ids and remove this task
app.send_task('tasks.user.crawl_person_infos', args=(wb_data.uid,), queue='user_crawler',
routing_key='for_user_info')
def get_page(url, auth_level=2, is_ajax=False, need_proxy=False):
"""
:param url: url to crawl
:param auth_level: 0 stands for need nothing,1 stands for no login but need cookies,2 stands for need login.
:param is_ajax: whether the request is ajax
:param need_proxy: whether the request need a http/https proxy
:return: response text, when a exception is raised, return ''
"""
crawler.info('the crawling url is {url}'.format(url=url))
count = 0
while count < MAX_RETRIES:
if auth_level == 2:
name_cookies = Cookies.fetch_cookies()
if name_cookies is None:
crawler.warning(
'No cookie in cookies pool. Maybe all accounts are banned, or all cookies are expired'
)
send_email()
os.kill(os.getppid(), signal.SIGTERM)
# There is no difference between http and https address.
proxy = {
'http': name_cookies[2],
'https': name_cookies[2],
}
else:
proxy = getip.getIPWithoutLogin('')
# if proxy['http'] is None:
# crawler.warning('No available ip in ip pools. Using local ip instead.')
try:
if auth_level == 2:
resp = requests.get(url,
headers=headers,
cookies=name_cookies[1],
timeout=TIME_OUT,
verify=False,
proxies=proxy)
elif auth_level == 1:
resp = requests.get(url,
headers=headers,
cookies=COOKIES,
timeout=TIME_OUT,
verify=False,
proxies=proxy)
else:
resp = requests.get(url,
headers=headers,
timeout=TIME_OUT,
verify=False,
proxies=proxy)
except (requests.exceptions.ReadTimeout,
requests.exceptions.ConnectionError, AttributeError) as e:
crawler.warning(
'Excepitons are raised when crawling {}.Here are details:{}'.
format(url, e))
count += 1
time.sleep(EXCP_INTERAL)
continue
if resp.status_code == 414:
crawler.warning('This ip has been blocked by weibo system')
if not need_proxy:
send_email()
os.kill(os.getppid(), signal.SIGTERM)
if resp.text:
page = resp.text.encode('utf-8', 'ignore').decode('utf-8')
else:
count += 1
continue
if auth_level == 2:
# slow down to aviod being banned
time.sleep(INTERAL)
if is_banned(resp.url) or is_403(page):
crawler.warning('Account {} has been banned'.format(
name_cookies[0]))
LoginInfoOper.freeze_account(name_cookies[0], 0)
Cookies.delete_cookies(name_cookies[0])
count += 1
continue
if not is_ajax and not is_complete(page):
count += 1
continue
if is_404(page):
crawler.warning('{} seems to be 404'.format(url))
return ''
Urls.store_crawl_url(url, 1)
return page
Urls.store_crawl_url(url, 0)
return ''
def crawl_weibo_datas(uid):
limit = get_max_home_page()
cur_page = 1
while cur_page <= limit:
url = HOME_URL.format(uid, cur_page)
if cur_page == 1:
html = get_page(url, auth_level=1)
else:
html = get_page(url, auth_level=2)
weibo_datas = get_data(html)
if not weibo_datas:
crawler.warning("user {} has no weibo".format(uid))
return
# Check whether weibo created after time in spider.yaml
length_weibo_datas = len(weibo_datas)
timeafter = time.mktime(
time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S'))
weibo_datas = [
weibo_datum for weibo_datum in weibo_datas
if determine(weibo_datum, timeafter)
]
WbDataOper.add_all(weibo_datas)
# If the weibo isn't created after the given time, jump out the loop
if len(weibo_datas) != length_weibo_datas:
break
domain = public.get_userdomain(html)
cur_time = int(time.time() * 1000)
ajax_url_0 = AJAX_URL.format(domain, 0, domain, uid, cur_page,
cur_page, cur_time)
ajax_url_1 = AJAX_URL.format(domain, 1, domain, uid, cur_page,
cur_page, cur_time + 100)
if cur_page == 1:
# here we use local call to get total page number
total_page = get_total_page(crawl_ajax_page(ajax_url_1, 2))
auth_level = 1
if total_page < limit:
limit = total_page
# Since the second ajax of page 1 has already been crawled
# in the code above and has been stored in databse,
# we only have to crawl the first ajax of page 1
crawl_ajax_page(ajax_url_0, auth_level)
else:
auth_level = 2
# Still the same as before
# if total_page != limit:
# limit = total_page
# crawler.warning("total pagenum is {}".format(total_page))
crawl_ajax_page(ajax_url_0, auth_level)
crawl_ajax_page(ajax_url_1, auth_level)
cur_page += 1
SeedidsOper.set_seed_home_crawled(uid)
def get_page(url, auth_level=2, is_ajax=False, need_proxy=False):
"""
:param url: url to crawl
:param auth_level: 0 stands for need nothing,1 stands for no login but need cookies,2 stands for need login.
:param is_ajax: whether the request is ajax
:param need_proxy: whether the request need a http/https proxy
:return: response text, when a exception is raised, return ''
"""
crawler.info('the crawling url is {url}'.format(url=url))
count = 0
while count < MAX_RETRIES:
if auth_level == 2:
name_cookies = Cookies.fetch_cookies()
if name_cookies is None:
crawler.warning('No cookie in cookies pool. Maybe all accounts are banned, or all cookies are expired')
send_email()
os.kill(os.getppid(), signal.SIGTERM)
# There is no difference between http and https address.
proxy = {'http': name_cookies[2], 'https': name_cookies[2], }
else:
proxy = getip.getIPWithoutLogin('')
# if proxy['http'] is None:
# crawler.warning('No available ip in ip pools. Using local ip instead.')
try:
if auth_level == 2:
resp = requests.get(url, headers=headers, cookies=name_cookies[1], timeout=TIME_OUT, verify=False, proxies=proxy)
elif auth_level == 1:
resp = requests.get(url, headers=headers, cookies=COOKIES, timeout=TIME_OUT, verify=False, proxies=proxy)
else:
resp = requests.get(url, headers=headers, timeout=TIME_OUT, verify=False, proxies=proxy)
except (requests.exceptions.ReadTimeout, requests.exceptions.ConnectionError, AttributeError) as e:
crawler.warning('Excepitons are raised when crawling {}.Here are details:{}'.format(url, e))
count += 1
time.sleep(EXCP_INTERAL)
continue
if resp.status_code == 414:
crawler.warning('This ip has been blocked by weibo system')
if not need_proxy:
send_email()
os.kill(os.getppid(), signal.SIGTERM)
if resp.text:
page = resp.text.encode('utf-8', 'ignore').decode('utf-8')
else:
count += 1
continue
if auth_level == 2:
# slow down to aviod being banned
time.sleep(INTERAL)
if is_banned(resp.url) or is_403(page):
crawler.warning('Account {} has been banned'.format(name_cookies[0]))
LoginInfoOper.freeze_account(name_cookies[0], 0)
Cookies.delete_cookies(name_cookies[0])
count += 1
continue
if not is_ajax and not is_complete(page):
count += 1
continue
if is_404(page):
crawler.warning('{} seems to be 404'.format(url))
return ''
Urls.store_crawl_url(url, 1)
return page
Urls.store_crawl_url(url, 0)
return ''
def crawl_weibo_datas(uid):
limit = get_max_home_page()
cur_page = 1
while cur_page <= limit:
url = HOME_URL.format(uid, cur_page)
if cur_page == 1:
html = get_page(url, auth_level=1)
else:
html = get_page(url, auth_level=2)
weibo_datas = get_data(html)
if not weibo_datas:
crawler.warning("user {} has no weibo".format(uid))
return
# Check whether weibo created after time in spider.yaml
length_weibo_datas = len(weibo_datas)
timeafter = time.mktime(
time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S'))
weibo_datas = [
weibo_datum for weibo_datum in weibo_datas
if determine(weibo_datum, timeafter)
]
WbDataOper.add_all(weibo_datas)
# If the weibo isn't created after the given time, jump out the loop
if len(weibo_datas) != length_weibo_datas:
break
domain = public.get_userdomain(html)
cur_time = int(time.time()*1000)
ajax_url_0 = AJAX_URL.format(domain, 0, domain, uid, cur_page, cur_page, cur_time)
ajax_url_1 = AJAX_URL.format(domain, 1, domain, uid, cur_page, cur_page, cur_time+100)
if cur_page == 1:
# here we use local call to get total page number
total_page = get_total_page(crawl_ajax_page(ajax_url_1, 2))
auth_level = 1
if total_page < limit:
limit = total_page
# Since the second ajax of page 1 has already been crawled
# in the code above and has been stored in databse,
# we only have to crawl the first ajax of page 1
crawl_ajax_page(ajax_url_0, auth_level)
else:
auth_level = 2
# Still the same as before
# if total_page != limit:
# limit = total_page
# crawler.warning("total pagenum is {}".format(total_page))
crawl_ajax_page(ajax_url_0, auth_level)
crawl_ajax_page(ajax_url_1, auth_level)
cur_page += 1
SeedidsOper.set_seed_home_crawled(uid)
def crawl_weibo_datas(uid):
limit = get_max_home_page()
cur_page = 36
retry_count = 1
while cur_page <= 36:
crawler.warning("current page {}".format(cur_page))
url = HOME_URL.format(uid, cur_page)
#if cur_page == 1:
# html = get_page(url, auth_level=1)
#else:
html = get_page(url, auth_level=2)
weibo_datas = get_data(html)
if not weibo_datas:
if retry_count < 10:
crawler.warning("user {} has no weibo, retry".format(uid))
retry_count = retry_count + 1
#time.sleep(240)
continue;
else:
crawler.warning("user {} has no weibo, return".format(uid))
return
# Check whether weibo created after time in spider.yaml
# timeafter = time.mktime(
# time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S'))
# length_weibo_datas = len(weibo_datas)
# for i in range(0, len(weibo_datas)):
# weibo_time = time.mktime(
# time.strptime(weibo_datas[i].create_time, '%Y-%m-%d %H:%M'))
# if weibo_time < timeafter:
# weibo_datas = weibo_datas[0:i]
# break
WbDataOper.add_all(weibo_datas)
# # If the weibo isn't created after the given time, jump out the loop
# if i != length_weibo_datas - 1:
# break
domain = public.get_userdomain(html)
cur_time = int(time.time()*1000)
ajax_url_0 = AJAX_URL.format(domain, 0, domain, uid, cur_page, cur_page, cur_time)
ajax_url_1 = AJAX_URL.format(domain, 1, domain, uid, cur_page, cur_page, cur_time+100)
# if cur_page == 1:
# # here we use local call to get total page number
# total_page = get_total_page(crawl_ajax_page(ajax_url_1, 2))
# auth_level = 1
# else:
auth_level = 2
#if total_page < limit:
# limit = total_page
crawler.warning("append tasks.home.crawl_ajax_page{}".format(uid));
app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_0, auth_level), queue='ajax_home_crawler',
routing_key='ajax_home_info')
app.send_task('tasks.home.crawl_ajax_page', args=(ajax_url_1, auth_level), queue='ajax_home_crawler',
routing_key='ajax_home_info')
cur_page += 1
SeedidsOper.set_seed_home_crawled(uid)
