def func_get_proxy_to_redis(): # chance = random.random() auth = kdl.Auth("997803479675913", "er2siw6i58c61s387sqljvovoz8zybaq") client = kdl.Client(auth) # 获取订单到期时间, 返回时间字符串 # expire_time = client.get_order_expire_time() # print("expire time", expire_time) # 获取ip白名单, 返回ip列表 # ip_whitelist = client.get_ip_whitelist() # print("ip whitelist", ip_whitelist) # 设置ip白名单,参数类型为字符串或列表或元组 # 成功则返回True, 否则抛出异常 # client.set_ip_whitelist([]) # client.set_ip_whitelist("127.0.0.1, 192.168.0.139") # print(client.get_ip_whitelist()) # client.set_ip_whitelist(tuple()) # 提取私密代理ip, 第一个参数为提取的数量, 其他参数以关键字参数的形式传入(不需要传入signature和timestamp) # 具体有哪些参数请参考帮助中心: "https://help.kuaidaili.com/api/getdps/" # 返回ip列表 # 注意:若您使用的是python2, 且在终端调用,或在文件中调用且没有加 "# -*- coding: utf-8 -*-" 的话 # 传入area参数时,请传入unicode类型,如 area=u'北京,上海' # ips = client.get_dps(1, sign_type='simple', format='json', pt=2, area='北京,上海,广东') # print("dps proxy: ", ips) # 检测私密代理有效性: 返回 ip: true/false 组成的dict #ips = client.get_dps(1, sign_type='simple', format='json') # valids = client.check_dps_valid(ips) # print("valids: ", valids) # 获取私密代理剩余时间: 返回 ip: seconds(剩余秒数) 组成的dict ips = client.get_dps(1, format='json',dedup=1) seconds = client.get_dps_valid_time(ips) # print("seconds: ", seconds) for key in seconds: rds.set(key, key, ex=int(seconds[key]) - 3)
# -*- coding: utf-8 -*- """私密代理使用示例 接口鉴权说明: 目前支持的鉴权方式有 "simple" 和 "hmacsha1" 两种,默认使用 "simple"鉴权。 所有方法均可添加关键字参数sign_type修改鉴权方式。 """ import kdl auth = kdl.Auth("test_order_id", "test_api_key") client = kdl.Client(auth) # 获取订单到期时间, 返回时间字符串 expire_time = client.get_order_expire_time() print("expire time", expire_time) # 获取ip白名单, 返回ip列表 ip_whitelist = client.get_ip_whitelist() print("ip whitelist", ip_whitelist) # 设置ip白名单,参数类型为字符串或列表或元组 # 成功则返回True, 否则抛出异常 client.set_ip_whitelist([]) client.set_ip_whitelist("171.113.244.40,171.113.244.41") print(client.get_ip_whitelist())
class RotateProxyManager(object): orderid = '977309259542981' # 订单号 get_proxy_url = "https://dps.kdlapi.com/api/getdps/?orderid={}&num={}&pt=1&format=json&sep=1" signature = 'exw3mf50r22o5v79w6a04b1kmey0r40f' # fetch_url = self.check_proxy_url.format(self.orderid, proxy,signature) # reserved_proxy_count = 900; # max_proxies = 10; # switches = 0; # proxy_pool = []; # cloud_times = 0; # cur_proxy = {"proxy":"202.183.32.182:80", "good":0, "bad":0}; cur_url = '' POOL = [] cursor = 0 auth = kdl.Auth("977309259542981", "exw3mf50r22o5v79w6a04b1kmey0r40f") client = kdl.Client(auth) def __init__(self): self.get_proxy_from_cloud(1) self.cur_url = self.nextProxy() # self.check_proxy_from_cloud(self.cur_url) def prettyList(self): for p in self.proxy_pool: logger.info(p) def nextProxy(self): self.cursor += 1 self.cursor = self.cursor % len(self.POOL) return self.POOL[self.cursor] def get_proxy_from_cloud(self, count): # fetch_url = self.get_proxy_url.format(self.orderid, count) # r = requests.get(fetch_url) r = self.client.get_dps(count, sign_type='hmacsha1', format='json') # print("dps proxy: ", r) # if r.status_code != 200: # logger.error("fail to fetch proxy") # return False # content = json.loads(r.content.decode('utf-8')) # logger.info(content); # ips = content['data']['proxy_list'] # left = content['data']['order_left_count'] ips = r if len(ips) == 0: logger.info("should stop the spider") logger.info(ips) from scrapy.exceptions import CloseSpider raise CloseSpider('no more proxies') for proxy in ips: ascproxy = proxy.encode('ascii') if ascproxy not in self.POOL: self.POOL.append(ascproxy) # return ips; def check_proxy_from_cloud(self, proxy): # check_proxy_url = "https://dps.kdlapi.com/api/checkdpsvalid/?orderid={}&proxy={}&signature r = self.client.check_dps_valid(proxy) # logger.info('check from result {}'.format(r)) # r = requests.get(fetch_url) # if r.status_code != 200: # logger.error("fail to fetch proxy") # return False # content = json.loads(r.content.decode('utf-8')) # logger.info(content); # statusDict = content['data'] # valid = statusDict[proxy] valid = r[proxy] return valid def invalidProxy(self, proxy): #1\ if proxy is in the pool, find the index logger.info(self.POOL) ipport = proxy.replace("http://", "") logger.warn("gonna check it in the pool {}".format(ipport)) if ipport in self.POOL: # logger.warn("gonna revalid it {}".format(ipport)) valid = self.check_proxy_from_cloud(ipport) if valid == False: logger.warn("gonna remove it {}".format(ipport)) self.POOL.remove(ipport) self.get_proxy_from_cloud(1) if len(self.POOL) == 0: self.get_proxy_from_cloud(1) #2\ validate the proxy from the cloud #3\ remove it from the pool if invalid #4\ get a new proxy from the api and append it to the pool return