class Process_Proxies(RetryMiddleware): logger = logging.getLogger(__name__) def dele_proxy(self, proxy, res=None): print('删除代理') if proxy: argents.remove(proxy) def process_response(self, request, response, spider): # if request.meta.get('dont_retry',False): # return response # if response.status in self.retry_http_codes: if response.status != 200: print('状态码异常') reason = response_status_message(response.status) print("将要删除代理::", request.meta['proxy']) self.dele_proxy(request.meta['proxy'], False) time.sleep(random.randint(3, 5)) return self._retry(request, reason, spider) or response return response def process_exception(self, request, exception, spider): if isinstance(exception, self.EXCEPTIONS_TO_RETRY) and not request.meta.get( 'dont_retry', False): print("将要删除代理::", request.meta['proxy']) self.dele_proxy(request.meta.get('proxy', False)) time.sleep(random.randint(3, 5)) self.logger.warning('连接异常,进行重试......') return self._retry(request, exception, spider)
class ProxyMiddleware(object): # 输出log日志 log = logging.getLogger(__name__) # def process_request(self, request, spider): # self.log.debug('Using Proxy') # request.meta['proxy'] = 'http://127.0.0.1:1080' def process_exception(self, request, exception, spider): self.log.debug('Get Exception') request.meta['proxy'] = 'http://127.0.0.1:1080' return request
import cgi import ftplib import os from base64 import b64decode from io import BytesIO from conda.gateways.logging import getLogger from requests import Response from requests.adapters import BaseAdapter from requests.hooks import dispatch_hook from ...common.compat import StringIO from ...common.url import urlparse from ...exceptions import AuthenticationError log = getLogger(__name__) class FTPAdapter(BaseAdapter): """A Requests Transport Adapter that handles FTP urls.""" def __init__(self): super(FTPAdapter, self).__init__() # Build a dictionary keyed off the methods we support in upper case. # The values of this dictionary should be the functions we use to # send the specific queries. self.func_table = { 'LIST': self.list, 'RETR': self.retr, 'STOR': self.stor, 'NLST': self.nlst,
import cgi import ftplib import os from base64 import b64decode from io import BytesIO from conda.gateways.logging import getLogger from requests import Response from requests.adapters import BaseAdapter from requests.hooks import dispatch_hook from ...common.compat import StringIO from ...common.url import urlparse from ...exceptions import AuthenticationError log = getLogger(__name__) class FTPAdapter(BaseAdapter): """A Requests Transport Adapter that handles FTP urls.""" def __init__(self): super(FTPAdapter, self).__init__() # Build a dictionary keyed off the methods we support in upper case. # The values of this dictionary should be the functions we use to # send the specific queries. self.func_table = {'LIST': self.list, 'RETR': self.retr, 'STOR': self.stor, 'NLST': self.nlst, 'GET': self.retr}