def test_parse_home_info(url, is_login, is_ajax, cookies, session): if is_login == 1: content = session.get(url).text if not is_ajax: assert len(home.get_data(content)) > 0 else: assert len(home.get_ajax_data(content)) > 0 else: content = requests.get(url, cookies=cookies).text if not is_ajax: assert len(home.get_data(content)) > 0 else: assert len(home.get_ajax_data(content)) > 0 time.sleep(REQUEST_INTERNAL)
def test_crawl_first_home_page(): from page_parse.home import get_ajax_data url = 'http://weibo.com/u/1800822823?is_ori=1&is_tag=0&profile_ftype=1&page=1' content = get_page(url, auth_level=1) assert "['islogin']" in content time.sleep(REQUEST_INTERNAL) cur_time = int(time.time() * 1000) ajax_url_0 = HOME_AJAX_URL.format('100505', 0, '100505', '1800822823', 1, 1, cur_time) ajax_url_1 = HOME_AJAX_URL.format('100505', 0, '100505', '1800822823', 1, 1, cur_time + 100) content = get_page(ajax_url_0, auth_level=1, is_ajax=True) assert 'Sina Visitor System' not in content assert len(get_ajax_data(content)) > 0 time.sleep(REQUEST_INTERNAL) content = get_page(ajax_url_1, auth_level=1, is_ajax=True) assert 'Sina Visitor System' not in content assert len(get_ajax_data(content)) > 0 time.sleep(REQUEST_INTERNAL)
def crawl_ajax_page(url, auth_level): """ :param url: user home ajax url :param auth_level: 1 stands for no login but need fake cookies, 2 stands for login :return: resp.text """ ajax_html = get_page(url, auth_level, is_ajax=True) ajax_wbdatas = get_ajax_data(ajax_html) if not ajax_wbdatas: return '' WbDataOper.add_all(ajax_wbdatas) return ajax_html
def crawl_ajax_page(url, auth_level): """ :param url: user home ajax url :param auth_level: 1 stands for no login but need fake cookies, 2 stands for login :return: resp.text """ ajax_html = get_page(url, auth_level, is_ajax=True) ajax_wbdatas = get_ajax_data(ajax_html) if not ajax_wbdatas: return '' timeafter = time.mktime(time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S')) for i in range(0,len(ajax_wbdatas)): weibo_time = time.mktime(time.strptime(ajax_wbdatas[i].create_time, '%Y-%m-%d %H:%M')) if weibo_time < timeafter: ajax_wbdatas = ajax_wbdatas[0:i] break WbDataOper.add_all(ajax_wbdatas) return ajax_html
def crawl_ajax_page(url, auth_level): """ :param url: user home ajax url :param auth_level: 1 stands for no login but need fake cookies, 2 stands for login :return: resp.text """ ajax_html = get_page(url, auth_level, is_ajax=True) ajax_wbdata = get_ajax_data(ajax_html) if not ajax_wbdata: return '' timeafter = time.mktime( time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S')) ajax_wbdata = [ ajax_wbdatum for ajax_wbdatum in ajax_wbdata if determine(ajax_wbdatum, timeafter) ] WbDataOper.add_all(ajax_wbdata) return ajax_html
def crawl_ajax_page(url, auth_level): """ :param url: user home ajax url :param auth_level: 1 stands for no login but need fake cookies, 2 stands for login :return: resp.text """ ajax_html = get_page(url, auth_level, is_ajax=True) ajax_wbdata = get_ajax_data(ajax_html) if not ajax_wbdata: return '' timeafter = time.mktime( time.strptime(get_time_after(), '%Y-%m-%d %H:%M:%S')) ajax_wbdata = [ ajax_wbdatum for ajax_wbdatum in ajax_wbdata if determine(ajax_wbdatum, timeafter) ] WbDataOper.add_all(ajax_wbdata) return ajax_html