def func_get_proxy_to_redis():
# chance = random.random()
auth = kdl.Auth("997803479675913", "er2siw6i58c61s387sqljvovoz8zybaq")
client = kdl.Client(auth)
# 获取订单到期时间, 返回时间字符串
# expire_time = client.get_order_expire_time()
# print("expire time", expire_time)
# 获取ip白名单, 返回ip列表
# ip_whitelist = client.get_ip_whitelist()
# print("ip whitelist", ip_whitelist)
# 设置ip白名单，参数类型为字符串或列表或元组
# 成功则返回True, 否则抛出异常
# client.set_ip_whitelist([])
# client.set_ip_whitelist("127.0.0.1, 192.168.0.139")
# print(client.get_ip_whitelist())
# client.set_ip_whitelist(tuple())
# 提取私密代理ip, 第一个参数为提取的数量, 其他参数以关键字参数的形式传入(不需要传入signature和timestamp)
# 具体有哪些参数请参考帮助中心: "https://help.kuaidaili.com/api/getdps/"
# 返回ip列表
# 注意：若您使用的是python2, 且在终端调用，或在文件中调用且没有加 "# -*- coding: utf-8 -*-" 的话
# 传入area参数时，请传入unicode类型，如 area=u'北京,上海'
# ips = client.get_dps(1, sign_type='simple', format='json', pt=2, area='北京,上海,广东')
# print("dps proxy: ", ips)
# 检测私密代理有效性： 返回 ip: true/false 组成的dict
#ips = client.get_dps(1, sign_type='simple', format='json')
# valids = client.check_dps_valid(ips)
# print("valids: ", valids)
# 获取私密代理剩余时间: 返回 ip: seconds(剩余秒数) 组成的dict
ips = client.get_dps(1, format='json',dedup=1)
seconds = client.get_dps_valid_time(ips)
# print("seconds: ", seconds)
for key in seconds:
rds.set(key, key, ex=int(seconds[key]) - 3)
# -*- coding: utf-8 -*-
"""私密代理使用示例
接口鉴权说明：
目前支持的鉴权方式有 "simple" 和 "hmacsha1" 两种，默认使用 "simple"鉴权。
所有方法均可添加关键字参数sign_type修改鉴权方式。
"""
import kdl
auth = kdl.Auth("test_order_id", "test_api_key")
client = kdl.Client(auth)
# 获取订单到期时间, 返回时间字符串
expire_time = client.get_order_expire_time()
print("expire time", expire_time)
# 获取ip白名单, 返回ip列表
ip_whitelist = client.get_ip_whitelist()
print("ip whitelist", ip_whitelist)
# 设置ip白名单，参数类型为字符串或列表或元组
# 成功则返回True, 否则抛出异常
client.set_ip_whitelist([])
client.set_ip_whitelist("171.113.244.40,171.113.244.41")
print(client.get_ip_whitelist())
class RotateProxyManager(object):
orderid = '977309259542981' # 订单号
get_proxy_url = "https://dps.kdlapi.com/api/getdps/?orderid={}&num={}&pt=1&format=json&sep=1"
signature = 'exw3mf50r22o5v79w6a04b1kmey0r40f'
# fetch_url = self.check_proxy_url.format(self.orderid, proxy,signature)
# reserved_proxy_count = 900;
# max_proxies = 10;
# switches = 0;
# proxy_pool = [];
# cloud_times = 0;
# cur_proxy = {"proxy":"202.183.32.182:80", "good":0, "bad":0};
cur_url = ''
POOL = []
cursor = 0
auth = kdl.Auth("977309259542981", "exw3mf50r22o5v79w6a04b1kmey0r40f")
client = kdl.Client(auth)
def __init__(self):
self.get_proxy_from_cloud(1)
self.cur_url = self.nextProxy()
# self.check_proxy_from_cloud(self.cur_url)
def prettyList(self):
for p in self.proxy_pool:
logger.info(p)
def nextProxy(self):
self.cursor += 1
self.cursor = self.cursor % len(self.POOL)
return self.POOL[self.cursor]
def get_proxy_from_cloud(self, count):
# fetch_url = self.get_proxy_url.format(self.orderid, count)
# r = requests.get(fetch_url)
r = self.client.get_dps(count, sign_type='hmacsha1', format='json')
# print("dps proxy: ", r)
# if r.status_code != 200:
# logger.error("fail to fetch proxy")
# return False
# content = json.loads(r.content.decode('utf-8'))
# logger.info(content);
# ips = content['data']['proxy_list']
# left = content['data']['order_left_count']
ips = r
if len(ips) == 0:
logger.info("should stop the spider")
logger.info(ips)
from scrapy.exceptions import CloseSpider
raise CloseSpider('no more proxies')
for proxy in ips:
ascproxy = proxy.encode('ascii')
if ascproxy not in self.POOL:
self.POOL.append(ascproxy)
# return ips;
def check_proxy_from_cloud(self, proxy):
# check_proxy_url = "https://dps.kdlapi.com/api/checkdpsvalid/?orderid={}&proxy={}&signature
r = self.client.check_dps_valid(proxy)
# logger.info('check from result {}'.format(r))
# r = requests.get(fetch_url)
# if r.status_code != 200:
# logger.error("fail to fetch proxy")
# return False
# content = json.loads(r.content.decode('utf-8'))
# logger.info(content);
# statusDict = content['data']
# valid = statusDict[proxy]
valid = r[proxy]
return valid
def invalidProxy(self, proxy):
#1\ if proxy is in the pool, find the index
logger.info(self.POOL)
ipport = proxy.replace("http://", "")
logger.warn("gonna check it in the pool {}".format(ipport))
if ipport in self.POOL:
# logger.warn("gonna revalid it {}".format(ipport))
valid = self.check_proxy_from_cloud(ipport)
if valid == False:
logger.warn("gonna remove it {}".format(ipport))
self.POOL.remove(ipport)
self.get_proxy_from_cloud(1)
if len(self.POOL) == 0:
self.get_proxy_from_cloud(1)
#2\ validate the proxy from the cloud
#3\ remove it from the pool if invalid
#4\ get a new proxy from the api and append it to the pool
return
