class Process_Proxies(RetryMiddleware):
logger = logging.getLogger(__name__)
def dele_proxy(self, proxy, res=None):
print('删除代理')
if proxy:
argents.remove(proxy)
def process_response(self, request, response, spider):
# if request.meta.get('dont_retry',False):
# return response
# if response.status in self.retry_http_codes:
if response.status != 200:
print('状态码异常')
reason = response_status_message(response.status)
print("将要删除代理：：", request.meta['proxy'])
self.dele_proxy(request.meta['proxy'], False)
time.sleep(random.randint(3, 5))
return self._retry(request, reason, spider) or response
return response
def process_exception(self, request, exception, spider):
if isinstance(exception,
self.EXCEPTIONS_TO_RETRY) and not request.meta.get(
'dont_retry', False):
print("将要删除代理：：", request.meta['proxy'])
self.dele_proxy(request.meta.get('proxy', False))
time.sleep(random.randint(3, 5))
self.logger.warning('连接异常,进行重试......')
return self._retry(request, exception, spider)
class ProxyMiddleware(object):
# 输出log日志
log = logging.getLogger(__name__)
# def process_request(self, request, spider):
# self.log.debug('Using Proxy')
# request.meta['proxy'] = 'http://127.0.0.1:1080'
def process_exception(self, request, exception, spider):
self.log.debug('Get Exception')
request.meta['proxy'] = 'http://127.0.0.1:1080'
return request
import cgi
import ftplib
import os
from base64 import b64decode
from io import BytesIO
from conda.gateways.logging import getLogger
from requests import Response
from requests.adapters import BaseAdapter
from requests.hooks import dispatch_hook
from ...common.compat import StringIO
from ...common.url import urlparse
from ...exceptions import AuthenticationError
log = getLogger(__name__)
class FTPAdapter(BaseAdapter):
"""A Requests Transport Adapter that handles FTP urls."""
def __init__(self):
super(FTPAdapter, self).__init__()
# Build a dictionary keyed off the methods we support in upper case.
# The values of this dictionary should be the functions we use to
# send the specific queries.
self.func_table = {
'LIST': self.list,
'RETR': self.retr,
'STOR': self.stor,
'NLST': self.nlst,
import cgi
import ftplib
import os
from base64 import b64decode
from io import BytesIO
from conda.gateways.logging import getLogger
from requests import Response
from requests.adapters import BaseAdapter
from requests.hooks import dispatch_hook
from ...common.compat import StringIO
from ...common.url import urlparse
from ...exceptions import AuthenticationError
log = getLogger(__name__)
class FTPAdapter(BaseAdapter):
"""A Requests Transport Adapter that handles FTP urls."""
def __init__(self):
super(FTPAdapter, self).__init__()
# Build a dictionary keyed off the methods we support in upper case.
# The values of this dictionary should be the functions we use to
# send the specific queries.
self.func_table = {'LIST': self.list,
'RETR': self.retr,
'STOR': self.stor,
'NLST': self.nlst,
'GET': self.retr}