def freeProxySecond(proxy_number=100):
"""
crawl 66 proxy http://www.66ip.cn/
:param proxy_number: number of the proxies
:return:
"""
url = "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&" \
"ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=".format(proxy_number)
request = WebRequest()
html = request.get(url).text # decoded string
for proxy in re.findall(r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}', html):
yield proxy
def freeProxySixth():
"""
crawl xun proxy http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10
:return:
"""
url = 'http://www.xdaili.cn/ipagent/freeip/getFreeIps?page=1&rows=10'
request = WebRequest()
try:
res = request.get(url).json()
for row in res['RESULT']['rows']:
yield '{}:{}'.format(row['ip'], row['port'])
except:
pass
def get_response(self, company_name, qyxx_id=None, pageNum = 1):
self.post_data['trdDataRequest']['name'] = company_name
self.post_data['trdDataRequest']['pageNum'] = pageNum
self.post_data['companyName'] = company_name
self.post_data['companyId'] = qyxx_id
time.sleep(0.3)
resp_text = WebRequest().post_data_json(self.request_url, data=json.dumps(self.post_data)).text
resp_json = json.loads(resp_text)
# print(resp_json)
return resp_json
def getHtmlTree(url):
"""
get html tree
:param url:
:return:
"""
header = {'Connection': 'keep-alive',
'Cache-Control': 'max-age=0',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko)',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'zh-CN,zh;q=0.8',
}
wr = WebRequest()
time.sleep(2)
html = wr.get(url=url, header=header).content
return etree.HTML(html)
def test_execute_with_success_response(self, mock_urlopen):
mock_urlopen.return_value.status = 200
wr = WebRequest("http://www.google.com")
self.assertEqual(wr.execute(), "SUCCESS")
def test_execute_with_failure_response(self, mock_urlopen):
mock_urlopen.return_value.status = 404
wr = WebRequest("http://www.google.com")
self.assertEqual(wr.execute(), "FAILURE")