def crawl_ajax_page(url):
ajax_html_0 = get_page(url)
ajax_wbdatas_0 = get_home_wbdata_byajax(ajax_html_0)
if not ajax_wbdatas_0:
return
insert_weibo_datas(ajax_wbdatas_0)
def crawl_ajax_page(url):
ajax_html = get_page(url, user_verify=False)
ajax_wbdatas = get_home_wbdata_byajax(ajax_html)
if not ajax_wbdatas:
return ''
insert_weibo_datas(ajax_wbdatas)
return ajax_html
def test_ajax_home_page_data(self):
"""
测试通过ajax返回的主页数据是否可以正常解析
:return:
"""
from page_parse import home
with open('tests/asyncpersonhome.html', encoding='utf-8') as f:
html = f.read()
datas = home.get_home_wbdata_byajax(html)
self.assertEqual(len(datas), 15)
def crawl_ajax_page(url):
"""
:param url: user home ajax url
:return: resp.text
"""
ajax_html = get_page(url, user_verify=False)
ajax_wbdatas = get_home_wbdata_byajax(ajax_html)
if not ajax_wbdatas:
return ''
insert_weibo_datas(ajax_wbdatas)
return ajax_html
def crawl_ajax_page(url):
"""
:param url: user home ajax url
:return: resp.text
"""
ajax_html = get_page(url, user_verify=False)
ajax_wbdatas = get_home_wbdata_byajax(ajax_html)
if not ajax_wbdatas:
return ''
insert_weibo_datas(ajax_wbdatas)
return ajax_html
def test_ajax_home_page_data(self):
"""
测试通过ajax返回的主页数据是否可以正常解析
:return:
"""
from page_parse import home
url = TEST_SERVER + 'asyncpersonhome.html'
resp = requests.get(url)
resp.encoding = 'utf-8'
html = resp.text
datas = home.get_home_wbdata_byajax(html)
self.assertEqual(len(datas), 15)
def crawl_ajax_page(url):
"""
返回值主要供第一次本地调用使用（获取总页数），网络调用忽略返回值
:param url:
:return:
"""
ajax_html = get_page(url, user_verify=False)
ajax_wbdatas = get_home_wbdata_byajax(ajax_html)
if not ajax_wbdatas:
return ''
insert_weibo_datas(ajax_wbdatas)
return ajax_html
def test_ajax_home_page_data(self):
"""
测试通过ajax返回的主页数据是否可以正常解析
:return:
"""
from page_parse import home
url = TEST_SERVER + 'asyncpersonhome.html'
resp = requests.get(url)
resp.encoding = 'utf-8'
html = resp.text
datas = home.get_home_wbdata_byajax(html)
self.assertEqual(len(datas), 15)
