def test_selector():
request = Request("https://www.baidu.com?a=1&b=2", data={}, params=None)
response = request.get_response()
print(response)
print(response.xpath("//a/@href"))
print(response.css("a::attr(href)"))
print(response.css("a::attr(href)").extract_first())
content = response.re("<a.*?href='(.*?)'")
print(content)
# -*- coding: utf-8 -*- """ Created on 2021/3/4 11:26 下午 --------- @summary: --------- @author: Boris @email: [email protected] """ from feapder import Request request = Request("https://www.baidu.com", data={}, params=None) response = request.get_response() print(response) print(response.xpath("//a/@href")) print(response.css("a::attr(href)")) print(response.css("a::attr(href)").extract_first()) content = response.re("<a.*?href='(.*?)'") print(content)
def debug_spider_test():
# debug爬虫
spider = test_spider.TestSpider.to_DebugSpider(
redis_key="feapder:test_spider",
request=Request("http://www.baidu.com"))
spider.start()
def request(**kwargs):
kwargs.setdefault("proxies", None)
response = Request(**kwargs).get_response()
print(response)
IPython.embed(header="now you can use response")
# -*- coding: utf-8 -*- """ Created on 2021/3/4 11:26 下午 --------- @summary: --------- @author: Boris @email: [email protected] """ from feapder import Request request = Request("https://www.baidu.com?a=1&b=2", data={}, params=None) print(request.fingerprint) request = Request("https://www.baidu.com?b=2&a=1", data={}, params=None) print(request.fingerprint) response = request.get_response() print(response) print(response.xpath("//a/@href")) print(response.css("a::attr(href)")) print(response.css("a::attr(href)").extract_first()) content = response.re("<a.*?href='(.*?)'") print(content)