def store_proxies(proxies):
conn = RedisClient(name='certificate_proxies')
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
conn_check = connect_check(proxies)
if conn_check:
proxies = json.dumps(proxies)
duplicate_check = conn.exist(proxies)
if not duplicate_check:
conn.set(proxies, 1)
info_logger.info(str(now) + ' New proxies: ' + str(proxies))
print(str(now) + ' New proxies: ' + str(proxies))
else:
info_logger.info(
str(now) + ' Already exist proxies: ' + str(proxies))
else:
err_logger.error(
str(now) + ' Can not connect baidu.com -- proxies: ' +
str(proxies))
def get_uniform_proxy(name, url):
proxy_url = 'http://166.188.20.55:3000/api/proxy/web'
headers = {
'Content-Type':
'application/json',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/'
'537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
}
data = {'name': name, 'url': url}
while True:
resp = requests.post(url=proxy_url,
headers=headers,
data=json.dumps(data)).json()
region = resp['proxy']['region']
if region == 'R1' or region == 'R2':
# print(region, 'err')
continue
else:
conn = RedisClient(name='certificate_proxies')
ip = resp['proxy']['ip']
port = '30000'
proxies = {
'http': 'http://%s:%s' % (ip, port),
'https': 'http://%s:%s' % (ip, port),
}
ping_url = 'http://www.baidu.com'
status_code = requests.get(ping_url,
proxies=proxies,
timeout=(3.1, 15)).status_code
if status_code == 200:
p = json.dumps(proxies)
check = conn.exist(p)
if not check:
conn.set(p, 1)
# conn.lpush(p)
print('New proxies: ', proxies)
break
else:
print('already exist proxies: ', p)
else:
print('connection error...')
def download_proxies():
# url = 'http://www.xdaili.cn/ipagent/privateProxy/applyStaticProxy?' \
# 'count=1&spiderId=fd4708592c97444c9f42060c500649ac&returnType=2'
url = 'http://www.xdaili.cn/ipagent/privateProxy/applyStaticProxy' \
'?count=1&spiderId=8407fdf0311d4eb7b30f9e39699795e5&returnType=2'
second = datetime.datetime.now().second
if second == 0 or second == 24 or second == 48:
#print(second)
time.sleep(1)
conn = RedisClient(name='certificate_proxies')
content = requests.get(url, timeout=(3.1, 15)).json()
result = content['RESULT']
if result == '提取太频繁,请按规定频率提取!':
err_logger.error(result)
else:
for proxy in content['RESULT']:
ip = proxy['ip']
port = proxy['port']
proxies = {
'http': 'http://%s:%s' % (ip, port),
'https': 'http://%s:%s' % (ip, port),
}
ping_url = 'http://www.baidu.com'
status_code = requests.get(ping_url,
timeout=(3.1, 15),
proxies=proxies).status_code
if status_code == 200:
p = json.dumps(proxies)
now = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
check = conn.exist(p)
if not check:
conn.set(p, 1)
conn.lpush(p)
now = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
print(now, ' New proxies: ', p)
else:
print(now, ' already exist proxies: ', p)
