def get_task():
conn = RedisClient(name='certificate', host='localhost', port='6379')
if conn.llen() == 0:
return None
info = conn.rpop()
try:
task = info.decode()
return task
except Exception as e:
print(e)
return None
def get_proxies():
while True:
conn = RedisClient(name='certificate_proxies', host='localhost', port='6379')
proxy = conn.lpop()
if not proxy:
print('无可用IP')
time.sleep(6)
else:
proxies = json.loads(proxy.decode())
ping_url = 'http://www.baidu.com'
try:
status_code = requests.get(ping_url, proxies=proxies, timeout=5).status_code
if status_code == 200:
return proxies
except Exception as e:
print(e)
continue
def download_proxies():
conn = RedisClient(name='certificate_proxies')
url = 'http://svip.kuaidaili.com/api/getproxy'
params = {
'orderid': '979397309945634',
'num': 20,
'quality': 2,
'format': 'json'
}
content = requests.get(url, params=params).json()
for proxy in content['data']['proxy_list']:
proxies = {
'http': 'http://%s' % proxy,
'https': 'http://%s' % proxy,
}
ping_url = 'http://www.baidu.com'
status_code = requests.get(ping_url).status_code
if status_code == 200:
p = json.dumps(proxies)
now = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
check = conn.exist(p)
if not check:
conn.set(p, 1)
conn.lpush(p)
now = time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
print(now, ' New proxies: ', p)
else:
print(now, ' already exist proxies: ', p)
def save(proxy):
conn = RedisClient(name='certificate_proxies')
proxies = {
'http': 'http://%s' % proxy,
'https': 'http://%s' % proxy,
}
# ping_url = 'http://www.baidu.com'
ping_url = 'http://zscx.osta.org.cn/'
status_code = requests.get(ping_url, timeout=(3.01, 6.01)).status_code
if status_code == 200:
p = json.dumps(proxies)
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
check = conn.exist(p)
if not check:
conn.set(p, 1)
conn.lpush(p)
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
print(now, ' New proxies: ', p)
else:
print(now, ' already exist proxies: ', p)
def get_uniform_proxy(name, url):
proxy_url = 'http://166.188.20.55:3000/api/proxy/web'
headers = {
'Content-Type':
'application/json',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/'
'537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
}
data = {'name': name, 'url': url}
while True:
resp = requests.post(url=proxy_url,
headers=headers,
data=json.dumps(data)).json()
region = resp['proxy']['region']
if region == 'R1' or region == 'R2':
# print(region, 'err')
continue
else:
conn = RedisClient(name='certificate_proxies')
ip = resp['proxy']['ip']
port = '30000'
proxies = {
'http': 'http://%s:%s' % (ip, port),
'https': 'http://%s:%s' % (ip, port),
}
ping_url = 'http://www.baidu.com'
status_code = requests.get(ping_url, proxies=proxies,
timeout=6.1).status_code
if status_code == 200:
p = json.dumps(proxies)
check = conn.exist(p)
if not check:
conn.set(p, 1)
conn.lpush(p)
print('New proxies: ', proxies)
break
else:
print('already exist proxies: ', p)
else:
print('connection error...')
def download_proxies():
conn = RedisClient(name='certificate_proxies')
url = 'http://www.xdaili.cn/ipagent/privateProxy/applyStaticProxy?count=1&spiderId=fd4708592c97444c9f42060c500649ac&returnType=2'
content = requests.get(url).json()
for proxy in content['RESULT']:
ip = proxy['ip']
port = proxy['port']
proxies = {
'http': 'http://%s:%s' % (ip, port),
'https': 'http://%s:%s' % (ip, port),
}
ping_url = 'http://www.baidu.com'
status_code = requests.get(ping_url).status_code
if status_code == 200:
p = json.dumps(proxies)
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
check = conn.exist(p)
if not check:
conn.set(p, 1)
conn.lpush(p)
now = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
print(now, ' New proxies: ', p)
else:
print(now, ' already exist proxies: ', p)
def push_task(task):
conn = RedisClient(name='certificate', host='localhost', port='6379')
conn.lpush(task)