def crawl_ajax_page(url): ajax_html_0 = get_page(url) ajax_wbdatas_0 = get_home_wbdata_byajax(ajax_html_0) if not ajax_wbdatas_0: return insert_weibo_datas(ajax_wbdatas_0)
def crawl_ajax_page(url): ajax_html = get_page(url, user_verify=False) ajax_wbdatas = get_home_wbdata_byajax(ajax_html) if not ajax_wbdatas: return '' insert_weibo_datas(ajax_wbdatas) return ajax_html
def test_ajax_home_page_data(self): """ 测试通过ajax返回的主页数据是否可以正常解析 :return: """ from page_parse import home with open('tests/asyncpersonhome.html', encoding='utf-8') as f: html = f.read() datas = home.get_home_wbdata_byajax(html) self.assertEqual(len(datas), 15)
def crawl_ajax_page(url): """ :param url: user home ajax url :return: resp.text """ ajax_html = get_page(url, user_verify=False) ajax_wbdatas = get_home_wbdata_byajax(ajax_html) if not ajax_wbdatas: return '' insert_weibo_datas(ajax_wbdatas) return ajax_html
def crawl_ajax_page(url): """ :param url: user home ajax url :return: resp.text """ ajax_html = get_page(url, user_verify=False) ajax_wbdatas = get_home_wbdata_byajax(ajax_html) if not ajax_wbdatas: return '' insert_weibo_datas(ajax_wbdatas) return ajax_html
def test_ajax_home_page_data(self): """ 测试通过ajax返回的主页数据是否可以正常解析 :return: """ from page_parse import home url = TEST_SERVER + 'asyncpersonhome.html' resp = requests.get(url) resp.encoding = 'utf-8' html = resp.text datas = home.get_home_wbdata_byajax(html) self.assertEqual(len(datas), 15)
def crawl_ajax_page(url): """ 返回值主要供第一次本地调用使用(获取总页数),网络调用忽略返回值 :param url: :return: """ ajax_html = get_page(url, user_verify=False) ajax_wbdatas = get_home_wbdata_byajax(ajax_html) if not ajax_wbdatas: return '' insert_weibo_datas(ajax_wbdatas) return ajax_html
def test_ajax_home_page_data(self): """ 测试通过ajax返回的主页数据是否可以正常解析 :return: """ from page_parse import home url = TEST_SERVER + 'asyncpersonhome.html' resp = requests.get(url) resp.encoding = 'utf-8' html = resp.text datas = home.get_home_wbdata_byajax(html) self.assertEqual(len(datas), 15)