def html_WITH_proxy():
"""test random proxy mechanism"""
url = "http://docs.python-requests.org/"
spider = Crawler()
spider.enable_proxy()
for i in range(10):
html = spider.html(url)
print(i, spider.pm.current_proxy)
if html:
print("\tSUCCESS")
spider.pm.update_health(1)
else:
print("\tFAILED")
print(spider)
def enable_proxy():
print("{:=^100}".format("enable_proxy"))
spider = Crawler()
spider.enable_proxy()
print(spider)
