def generator_proxies(self): for name in self.list_name: if name in self.dic: url,params = chambering(origin_proxies[name],strike=False) result = requester(url,params,GET=True,timeout=None) response = regex.Espace_eliminate.sub("",result.text) ips, ports, types = self.dic[name]['ip'].finditer(response),\ self.dic[name]['port'].finditer(response),\ self.dic[name]['type'].finditer(response) for i, j, k in zip(ips,ports,types): ip = self.dic[name]['sub'].sub(" ", i.group()) port = self.dic[name]['sub'].sub(" ", j.group()) type = self.dic[name]['sub'].sub(" ", k.group()) # self.count = self.count+1 # print((ip, port, type)) if Filter.filter(ip,self.filter_proxy): proxy = eval(regex.Espace_eliminate.sub("", str((ip, port, type.lower())))) self.logger.info(f"ip : {proxy[0]} port : {proxy[1]} type : {proxy[2]}") self.container.put(proxy)
def generator_proxies(self): ''' 从默认代理池(list_name中定义的)获取可用代理 ''' for name in self.list_name: if name in self.dic: url, params = chambering(origin_proxies[name], strike=False) result = requester(url, params, GET=True, timeout=None) if result == None: continue response = regex.Espace_eliminate.sub("", result.text) ips, ports, types = self.dic[name]['ip'].finditer(response),\ self.dic[name]['port'].finditer(response),\ self.dic[name]['type'].finditer(response) for i, j, k in zip(ips, ports, types): ip = self.dic[name]['sub'].sub(" ", i.group()) port = self.dic[name]['sub'].sub(" ", j.group()) type = self.dic[name]['sub'].sub(" ", k.group()) # self.count = self.count+1 # print((ip, port, type)) if Filter.filter(ip, self.filter_proxy): # 不是特殊文件链接 就加入到代理集合中 proxy = eval( regex.Espace_eliminate.sub( "", str((ip, port, type.lower())))) # temp=r.replace('(','').replace(')','') # 字符串转元组安全方法 # a=tuple([int(i) for i in temp.split(',')]) self.logger.info( f"ip : {proxy[0]} port : {proxy[1]} type : {proxy[2]}" ) self.container.put(proxy)