def generator_proxies(self):
for name in self.list_name:
if name in self.dic:
url,params = chambering(origin_proxies[name],strike=False)
result = requester(url,params,GET=True,timeout=None)
response = regex.Espace_eliminate.sub("",result.text)
ips, ports, types = self.dic[name]['ip'].finditer(response),\
self.dic[name]['port'].finditer(response),\
self.dic[name]['type'].finditer(response)
for i, j, k in zip(ips,ports,types):
ip = self.dic[name]['sub'].sub(" ", i.group())
port = self.dic[name]['sub'].sub(" ", j.group())
type = self.dic[name]['sub'].sub(" ", k.group())
# self.count = self.count+1
# print((ip, port, type))
if Filter.filter(ip,self.filter_proxy):
proxy = eval(regex.Espace_eliminate.sub("", str((ip, port, type.lower()))))
self.logger.info(f"ip : {proxy[0]} port : {proxy[1]} type : {proxy[2]}")
self.container.put(proxy)
def generator_proxies(self):
''' 从默认代理池(list_name中定义的)获取可用代理 '''
for name in self.list_name:
if name in self.dic:
url, params = chambering(origin_proxies[name], strike=False)
result = requester(url, params, GET=True, timeout=None)
if result == None:
continue
response = regex.Espace_eliminate.sub("", result.text)
ips, ports, types = self.dic[name]['ip'].finditer(response),\
self.dic[name]['port'].finditer(response),\
self.dic[name]['type'].finditer(response)
for i, j, k in zip(ips, ports, types):
ip = self.dic[name]['sub'].sub(" ", i.group())
port = self.dic[name]['sub'].sub(" ", j.group())
type = self.dic[name]['sub'].sub(" ", k.group())
# self.count = self.count+1
# print((ip, port, type))
if Filter.filter(ip,
self.filter_proxy): # 不是特殊文件链接 就加入到代理集合中
proxy = eval(
regex.Espace_eliminate.sub(
"", str((ip, port, type.lower()))))
# temp=r.replace('(','').replace(')','') # 字符串转元组安全方法
# a=tuple([int(i) for i in temp.split(',')])
self.logger.info(
f"ip : {proxy[0]} port : {proxy[1]} type : {proxy[2]}"
)
self.container.put(proxy)
