def proxy_open(self, req, type):
orig_type = req.get_type()
try:
proxy = self.proxies[type]
except KeyError:
return None
proxy_type, user, password, hostport = urllib2._parse_proxy(proxy)
if proxy_type is None:
proxy_type = orig_type
if user and password:
user_pass = '******' % (urllib2.unquote(user), urllib2.unquote(password))
creds = base64.b64encode(user_pass).strip()
req.add_header('Proxy-Authorization', 'Basic ' + creds)
hostport = urllib2.unquote(hostport)
req.set_proxy(hostport, proxy_type)
if orig_type == proxy_type:
# let other handlers take care of it
return None
else:
# need to start over, because the other handlers don't
# grok the proxy's URL type
# e.g. if we have a constructor arg proxies like so:
# {'http': 'ftp://proxy.example.com'}, we may end up turning
# a request for http://acme.example.com/a into one for
# ftp://proxy.example.com/a
return self.parent.open(req)
def connect(self):
#- Parse proxies
pd = urllib.getproxies().get('http', None)
if pd is None:
sockstype = ''
else:
sockstype, user, password, hostport = urllib2._parse_proxy(pd)
if 'socks' not in sockstype:
return httplib.HTTPConnection.connect(self)
assert ':' in hostport # if we don't have a port we're screwed
host, port = hostport.rsplit(':', 1)
port = int(port)
for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM):
af, socktype, proto, canonname, sa = res
try:
self.sock = self._sockettype(af, socktype, proto)
self.sock.setproxy(proxytype=getattr(socks, 'PROXY_TYPE_%s' % sockstype.upper()), addr=host, port=port, rdns=False, username=user, password=password)
#- The rest is the same as superclass
if self.debuglevel > 0:
print "connect: (%s, %s)" % (self.host, self.port)
self.sock.connect(sa)
except socket.error, msg:
if self.debuglevel > 0:
print 'connect fail:', (self.host, self.port)
if self.sock:
self.sock.close()
self.sock = None
continue
break
def _get_proxy(self, url):
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type
or "http", hostport, '', '', '', ''))
creds = None
return creds, proxy_url
def proxy_open(self, req, type):
orig_type = req.get_type()
try:
proxy = self.proxies[type]
except KeyError:
return None
proxy_type, user, password, hostport = urllib2._parse_proxy(proxy)
if proxy_type is None:
proxy_type = orig_type
if user and password:
user_pass = '******' % (urllib2.unquote(user),
urllib2.unquote(password))
creds = base64.b64encode(user_pass).strip()
req.add_header('Proxy-Authorization', 'Basic ' + creds)
hostport = urllib2.unquote(hostport)
req.set_proxy(hostport, proxy_type)
if orig_type == proxy_type:
# let other handlers take care of it
return None
else:
# need to start over, because the other handlers don't
# grok the proxy's URL type
# e.g. if we have a constructor arg proxies like so:
# {'http': 'ftp://proxy.example.com'}, we may end up turning
# a request for http://acme.example.com/a into one for
# ftp://proxy.example.com/a
return self.parent.open(req)
def http_request(self, request):
# Luckily we only need to worry about http proxies/urls.
try:
proxy = self.proxies['http']
except KeyError:
return None
proxy_type, user, password, hostport = urllib2._parse_proxy(proxy)
if proxy_type != 'http':
log.warning(
'Got unexpected proxy type %r in asynchttp. not modifying request',
proxy_type)
return
if proxy_type is None:
proxy_type = 'http'
if user and password:
user_pass = '******' % (urllib2.unquote(user),
urllib2.unquote(password))
creds = user_pass.encode('base64').strip()
request.headers['Proxy-Authorization'] = 'Basic ' + creds
hostport = urllib.unquote(hostport)
request.set_proxy(hostport, proxy_type)
def create_from_url(cls, url: str) -> 'Proxy':
proxy_type, login, password, hostport = _parse_proxy(url)
if ':' in hostport:
host, port = hostport.split(':')
kw = {'host': host, "port": port}
else:
kw = {'host': hostport}
self = cls(scheme=proxy_type, login=login, password=password, **kw)
return self
def _get_proxy(self, url, orig_type):
proxy_type, user, password, hostport = _parse_proxy(url)
# proxy_url = urlparse.urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
if user:
user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password)),
encoding=self.auth_encoding)
creds = base64.b64encode(user_pass).strip()
else:
creds = None
def _get_proxy(self, url, orig_type=''):
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type
or orig_type, hostport, '', '', '', ''))
creds = util._basic_auth_header(user, password, self.auth_encoding) \
if user else None
return creds, proxy_url
def connect(self):
# Call the connect() method of httplib.HTTPConnection
httplib.HTTPConnection.connect(self)
# At this point I am connected to the proxy server so I can send the CONNECT request
connectCommand = "CONNECT " + str(self._real_host) + ":" + str(
self._real_port) + " HTTP/1.0\r\n\r\n"
self.send(connectCommand)
# Expect a HTTP/1.0 200 Connection established
response = self.response_class(self.sock,
strict=self.strict,
method=self._method)
(version, code, message) = response._read_status()
# Probably here we can handle auth requests...
# 407 Proxy Authentication Required
if (code == 407):
# Obtain the list of proxies using a call to urllib.getproxies()
proxyDictionary = urllib.getproxies()
# Base the proxy string on what protocol was requested
desiredProxy = proxyDictionary[self.protocol]
# Parse out the proxy string for the username and password
proxy_type, user, password, hostport = urllib2._parse_proxy(
desiredProxy)
proxyAuthorizationString = 'Proxy-Authorization: Basic %s\r\n\r\n' % base64.b64encode(
'%s:%s' % (user, password))
connectCommand = "CONNECT " + str(self._real_host) + ":" + str(
self._real_port) + " HTTP/1.0\r\n"
httplib.HTTPConnection.connect(self)
self.send(connectCommand)
self.send(proxyAuthorizationString)
# Proxy returned something other than 407 or 200
elif (code != 200):
self.close()
raise socket.error, "Proxy connection failed: %d %s" % (
code, message.strip())
# Eat up header block from proxy....
while True:
# Note to investigate using "fp" directly and performing a readline().
line = response.fp.readline()
if line == '\r\n': break
# Determine if we are trying to do an SSL connection. If we are
# we need to make sure we do the SSL handshake
if (self.protocol == 'https'):
newSSLSocket = ssl.wrap_socket(self.sock,
keyfile=None,
certfile=None)
self.sock = newSSLSocket
def getOpener(self, values):
opener_handlers = [urllib2.HTTPHandler(debuglevel = values.debug)]
if hasattr(httplib, 'HTTPS'):
opener_handlers.append(urllib2.HTTPSHandler(debuglevel = values.debug))
include = None if values.include else self.log
pSwitches = [values.post301, values.post302, values.post303]
# opener_handlers = [LogHandler(values.url)]
# opener_handlers.append(netHTTPRedirectHandler(location = values.location, include = include, postSwitches = pSwitches))
opener_handlers.append(netHTTPRedirectHandler(location = values.location, include = include, postSwitches = pSwitches))
cookie_val = None
if values.cookie_jar: cookie_val = values.cookie_jar
if values.cookie: cookie_val = values.cookie
if cookie_val != None:
cj = cookielib.LWPCookieJar()
if os.path.isfile(cookie_val): cj.load(cookie_val)
opener_handlers.append(urllib2.HTTPCookieProcessor(cj))
passwordManager = urllib2.HTTPPasswordMgrWithDefaultRealm()
if values.user:
user, password = values.user.partition(':')[0:3:2]
if not password:
try:
password = tkSimpleDialog.askstring("Enter password for " + user, "Enter password:"******"Enter password for " + user)
passwordManager.add_password(None, values.url, user, password)
opener_handlers.append(urllib2.HTTPBasicAuthHandler(passwordManager))
if values.auth_method == 'digest':
opener_handlers.append(urllib2.HTTPDigestAuthHandler(passwordManager))
pass
if values.proxy:
proxyStr = values.proxy
protocol, user, password, proxyhost = urllib2._parse_proxy(proxyStr)
protocol = protocol or 'http'
proxy_handler = urllib2.ProxyHandler({protocol:proxyhost})
opener_handlers.append(proxy_handler)
if not user:
if values.proxy_user:
user, password = values.proxy_user.partition(':')[0:3:2]
if user and not password:
try:
password = tkSimpleDialog.askstring("Enter proxy password for " + user, "Enter password:"******"Enter proxy password for " + user)
passwordManager.add_password(None, proxyhost, user, password)
opener_handlers.append(urllib2.ProxyBasicAuthHandler(passwordManager))
if values.proxy_auth == 'digest':
opener_handlers.append(urllib2.ProxyDigestAuthHandler(passwordManager))
pass
opener = urllib2.build_opener(*opener_handlers)
return opener
def get_proxy(url, orig_type, auth_encoding):
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
if user:
creds = basic_auth_header(user, password, auth_encoding)
else:
creds = None
return creds, proxy_url
def _get_proxy(self, url, orig_type):
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
if user:
creds = self._basic_auth_header(user, password)
else:
creds = None
return creds, proxy_url
def pre_requests(self):
if self.proxy:
self._proxy_config = {}
proxy_type, user, password, hostport = _parse_proxy(self.proxy)
if user and password:
user_pass = '******' % (unquote(user), unquote(password))
creds = base64.b64encode(user_pass).strip()
self._proxy_config['creds'] = creds
hostport = unquote(hostport)
self._proxy_config['hostport'] = hostport
self._proxy_config['proxy_type'] = proxy_type
def _make_request(self, url, h={}, d=None, p=None):
header = copy.deepcopy(sz_conf['header'])
header.update(h)
data = urllib.make_data(d) if d else None
request = urllib2.Request( url, data, header)
if p:
t,_,_,h = urllib2._parse_proxy(p)
request.set_proxy(h,t)
return request
def make_request(url, header={}, data=None, proxy=None):
#header = copy.deepcopy(sz_conf['header'])
#header.update(h)
data = _make_data(data) if data else None
request = urllib2.Request( url, data, header)
if not proxy: return request
t,_,_,h = urllib2._parse_proxy(p)
request.set_proxy(h,t)
return request
def _get_proxy(self, url, orig_type):
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
if user and password:
user_pass = '******' % (unquote(user), unquote(password))
creds = base64.b64encode(user_pass).strip()
else:
creds = None
return creds, proxy_url
def _request(self, sock, method, path, protocol_version, headers, payload, bufsize=__bufsize__, crlf=None, return_sock=None):
skip_headers = self.skip_headers
need_crlf = self.crlf
if crlf:
need_crlf = 1
if need_crlf:
request_data = 'GET / HTTP/1.1\r\n\r\n\r\n'
else:
request_data = ''
request_data += '%s %s %s\r\n' % (method, path, protocol_version)
request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.iteritems() if k not in skip_headers)
if self.proxy:
_, username, password, _ = urllib2._parse_proxy(self.proxy)
if username and password:
request_data += 'Proxy-Authorization: Basic %s\r\n' % base64.b64encode('%s:%s' % (username, password))
request_data += '\r\n'
# Send the request
if not payload:
sock.sendall(request_data)
else:
if isinstance(payload, basestring):
request_data += payload
sock.sendall(request_data)
elif hasattr(payload, 'read'):
sock.sendall(request_data)
while 1:
data = payload.read(bufsize)
if not data:
break
sock.sendall(data)
else:
raise TypeError('http.request(payload) must be a string or buffer, not %r' % type(payload))
if need_crlf:
try:
response = httplib.HTTPResponse(sock)
response.begin()
response.read()
except Exception:
self.logger.exception('crlf skip read')
return None
if return_sock:
return sock
response = httplib.HTTPResponse(sock, buffering=True) if sys.hexversion > 0x02070000 else httplib.HTTPResponse(sock)
try:
response.begin()
except httplib.BadStatusLine:
response = None
return response
def _get_proxy(self, url):
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type, hostport, '', '', '', ''))
if user and password:
user_pass = '******' % (unquote(user), unquote(password))
creds = base64.b64encode(user_pass).strip()
else:
creds = None
return creds, proxy_url
def _make_request(self, url, h={}, d=None, p=None):
header = copy.deepcopy(sz_conf['header'])
header.update(h)
data = urllib.make_data(d) if d else None
request = urllib2.Request(url, data, header)
if p:
t, _, _, h = urllib2._parse_proxy(p)
request.set_proxy(h, t)
return request
def _get_proxy(self, url):
#>>> _parse_proxy('http://*****:*****@proxy.example.com/')
#('http', 'joe', 'password', 'proxy.example.com')
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type
or "http", hostport, '', '', '', ''))
#如果有用户生成证书用于连接
if user:
user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password)),
encoding="utf-8")
creds = base64.b64encode(user_pass).strip()
else:
creds = None
return creds, proxy_url
def __init__(self, proxy: str):
scheme, user, password, host_port = _parse_proxy(proxy)
self.scheme = scheme
self.user = user
self.password = password
host, port = host_port.split(':')
self.host = host
self.port = port
self.user_pass_base64 = None
if user and password:
self.user_pass_base64 = base64.b64encode(
f'{user}:{password}'.encode())
else:
self.user_pass_base64 = None
def _get_proxy(self, url):
#>>> _parse_proxy('http://*****:*****@proxy.example.com/')
#('http', 'joe', 'password', 'proxy.example.com')
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or "http", hostport, '', '', '', ''))
#如果有用户生成证书用于连接
if user:
user_pass = to_bytes(
'%s:%s' % (unquote(user), unquote(password)),
encoding="utf-8")
creds = base64.b64encode(user_pass).strip()
else:
creds = None
return creds, proxy_url
def socks_open(self, req, sockstype):
orig_url_type, __, __, orighostport = urllib2._parse_proxy(req.get_full_url())
req.set_proxy(orighostport, orig_url_type)
endpoint = req.get_host()
if ':' in endpoint:
host, port = endpoint.rsplit(':', 1)
port = int(port)
else:
host, port = endpoint, 80
req._proxied = True
return self.parent.open(req)
def connect(self):
# Call the connect() method of httplib.HTTPConnection
httplib.HTTPConnection.connect(self)
# At this point I am connected to the proxy server so I can send the CONNECT request
connectCommand = "CONNECT " + str(self._real_host) + ":" + str(self._real_port) + " HTTP/1.0\r\n\r\n"
self.send(connectCommand)
# Expect a HTTP/1.0 200 Connection established
response = self.response_class(self.sock, strict=self.strict, method=self._method)
(version, code, message) = response._read_status()
# Probably here we can handle auth requests...
# 407 Proxy Authentication Required
if (code == 407):
# Obtain the list of proxies using a call to urllib.getproxies()
proxyDictionary = urllib.getproxies()
# Base the proxy string on what protocol was requested
desiredProxy = proxyDictionary[self.protocol]
# Parse out the proxy string for the username and password
proxy_type, user, password, hostport = urllib2._parse_proxy(desiredProxy)
proxyAuthorizationString = 'Proxy-Authorization: Basic %s\r\n\r\n' % base64.b64encode('%s:%s' % (user, password))
connectCommand = "CONNECT " + str(self._real_host) + ":" + str(self._real_port) + " HTTP/1.0\r\n"
httplib.HTTPConnection.connect(self)
self.send(connectCommand)
self.send(proxyAuthorizationString)
# Proxy returned something other than 407 or 200
elif (code != 200):
self.close()
raise socket.error, "Proxy connection failed: %d %s" % (code, message.strip())
# Eat up header block from proxy....
while True:
# Note to investigate using "fp" directly and performing a readline().
line = response.fp.readline()
if line == '\r\n': break
# Determine if we are trying to do an SSL connection. If we are
# we need to make sure we do the SSL handshake
if (self.protocol == 'https'):
newSSLSocket = ssl.wrap_socket(self.sock, keyfile=None, certfile=None)
self.sock = newSSLSocket
def _get_proxy(url: str) -> (str, str):
"""
Transform proxy url into a tuple of credentials and proxy url without credentials
:param str url:
:return str, str: credentials, proxy url
"""
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type, hostport, '', '', '', ''))
if user and password:
user_pass = '******' % (unquote(user), unquote(password))
creds = base64.b64encode(user_pass.encode("utf-8")).strip()
else:
creds = None
return creds, proxy_url
def get_proxy_info(proxystr=None):
"""
Get proxy config from string or environment variables.
If a proxy string is passed in, it overrides whatever might be in the
environment variables.
Returns dictionary of identified proxy information.
Raises ValueError on any configuration error.
"""
default_port = 80
# Only check for env variables if no explicit proxy string was provided.
if proxystr is None or len(proxystr) < 1:
# FIXME: We should be supporting http_proxy, HTTP_PROXY variables.
proxy_info = {
'host' : os.environ.get('PROXY_HOST', None),
'port' : os.environ.get('PROXY_PORT', default_port),
'user' : os.environ.get('PROXY_USER', None),
'pass' : os.environ.get('PROXY_PASS', None)
}
# Parse the passed proxy string
else:
# XXX Using proxy parsing function from urllib2 to parse proxystr
_, user, passwd, host_port = urllib2._parse_proxy(proxystr)
host, port = urllib2.splitport(host_port)
proxy_info = {
'host' : host,
'port' : port or default_port,
'user' : user,
'pass' : passwd,
}
# If a user was specified, but no password was, prompt for it now.
user = proxy_info.get('user', None)
if user is not None and len(user) > 0:
passwd = proxy_info.get('pass', None)
if passwd is None or len(passwd) < 1:
import getpass
proxy_info['pass'] = getpass.getpass()
return proxy_info
def connect(self):
"Connect to a host on a given (SSL) port."
pd = urllib.getproxies().get('https', None)
if pd is None:
sockstype = ''
else:
sockstype, user, password, hostport = urllib2._parse_proxy(pd)
assert ':' in hostport # if we don't have a port we're screwed
host, port = hostport.rsplit(':', 1)
port = int(port)
sock = self._sockettype(socket.AF_INET, socket.SOCK_STREAM)
sock.setproxy(proxytype=getattr(socks, 'PROXY_TYPE_%s' % sockstype.upper()), addr=host, port=port, rdns=True, username=user, password=password)
sock.connect((self.host, self.port))
self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file)
def extract_proxy_hostport(proxy):
"""
Return the hostport component from a given proxy:
>>> extract_proxy_hostport('example.com')
'example.com'
>>> extract_proxy_hostport('http://www.example.com')
'www.example.com'
>>> extract_proxy_hostport('127.0.0.1:8000')
'127.0.0.1:8000'
>>> extract_proxy_hostport('127.0.0.1')
'127.0.0.1'
>>> extract_proxy_hostport('localhost')
'localhost'
>>> extract_proxy_hostport('zot:4321')
'zot:4321'
>>> extract_proxy_hostport('http://*****:*****@baz:1234')
'baz:1234'
"""
return _parse_proxy(proxy)[3]
def proxy_open(self, req, type):
try:
req._proxied
except AttributeError:
proxyinfo = self.proxies.get(type, '')
proxytype = urllib2._parse_proxy(proxyinfo)[0]
if proxytype is None:
req._proxied = False
return urllib2.ProxyHandler.proxy_open(self, req, type)
else:
req._proxytype = proxytype
req._proxied = True
if proxytype == 'http' and type != 'https': # Http proxy
return urllib2.ProxyHandler.proxy_open(self, req, type)
else:
return None
else:
# Already proxied. skip it.
return None
def process_request(self, request, spider):
PROXIES = [
{
'ip_port': '111.11.228.75:80',
'user_pass': ''
},
{
'ip_port': '111.11.228.75:80',
'user_pass': ''
},
{
'ip_port': '111.11.228.75:80',
'user_pass': ''
},
{
'ip_port': '111.11.228.75:80',
'user_pass': ''
},
{
'ip_port': '111.11.228.75:80',
'user_pass': ''
},
]
url = random.choice(PROXIES)
#url = "http://*****:*****@192.168.11.11:9999/"
orig_type = ""
# from httpproxy._get_proxy
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type
or orig_type, hostport, '', '', '', ''))
if user:
creds = self._basic_auth_header(user, password)
else:
creds = None
request.meta['proxy'] = proxy_url
if creds:
request.headers['Proxy-Authorization'] = b'Basic ' + creds
def extract_proxy_hostport(proxy):
"""
提取代理连接串中的host和port:
>>> extract_proxy_hostport('heismart.cn')
'heismart.cn'
>>> extract_proxy_hostport('http://www.heismart.cn')
'www.heismart.cn'
>>> extract_proxy_hostport('127.0.0.1:8000')
'127.0.0.1:8000'
>>> extract_proxy_hostport('127.0.0.1')
'127.0.0.1'
>>> extract_proxy_hostport('localhost')
'localhost'
>>> extract_proxy_hostport('zot:4321')
'zot:4321'
>>> extract_proxy_hostport('http://*****:*****@heismart.cn:8080')
'baz:1234'
"""
return _parse_proxy(proxy)[3]
def process_request(self, request, spider):
PROXIES = [
# "http://root:@[email protected]:8000/",
"http://113.194.28.88:9999",
"http://27.43.185.186:9999",
"http://113.128.122.131:9999",
"http://123.169.115.180:9999",
"http://163.204.245.204:9999",
"http://110.243.26.245:9999",
"http://218.58.193.98:8060",
"http://123.169.125.114:9999",
"http://1.198.72.73:9999 ",
"http://115.218.214.75:9000",
"http://171.13.137.32:9999",
"http://183.166.103.19:9999",
"http://220.249.149.137:9999",
"http://113.194.137.166:9999",
"http://110.243.12.85:9999",
"http://115.221.244.4:9999",
]
url = random.choice(PROXIES)
orig_type = ""
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
if user:
creds = self._basic_auth_header(user, password)
else:
creds = None
request.meta['proxy'] = proxy_url
if creds:
request.headers['Proxy-Authorization'] = b'Basic ' + creds
# ModuleNotFoundError: No module named 'scrapy.contrib'
# 原因是scrapy-1.6.0已删除scrapy.contrib
# pip uninstall Scrapy
# pip install Scrapy==1.5.2
def http_request(self, request):
# Luckily we only need to worry about http proxies/urls.
try:
proxy = self.proxies["http"]
except KeyError:
return None
proxy_type, user, password, hostport = urllib2._parse_proxy(proxy)
if proxy_type != "http":
log.warning("Got unexpected proxy type %r in asynchttp. not modifying request", proxy_type)
return
if proxy_type is None:
proxy_type = "http"
if user and password:
user_pass = "******" % (urllib2.unquote(user), urllib2.unquote(password))
creds = user_pass.encode("base64").strip()
request.headers["Proxy-Authorization"] = "Basic " + creds
hostport = urllib.unquote(hostport)
request.set_proxy(hostport, proxy_type)
def process_request(self, request, spider):
PROXIES = [
"http://*****:*****@192.168.11.11:9999/",
"http://*****:*****@192.168.11.12:9999/",
"http://*****:*****@192.168.11.13:9999/",
"http://*****:*****@192.168.11.14:9999/",
"http://*****:*****@192.168.11.15:9999/",
"http://*****:*****@192.168.11.16:9999/",
]
url = random.choice(PROXIES)
orig_type = ""
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
if user:
creds = self._basic_auth_header(user, password)
else:
creds = None
request.meta['proxy'] = proxy_url
if creds:
request.headers['Proxy-Authorization'] = b'Basic ' + creds
def process_request(self, request, spider):
if 'proxy' in request.meta and not request.meta.get('_rotating_proxy'):
return
proxy = self.proxies.get_random()
if not proxy:
if self.stop_if_no_proxies:
raise CloseSpider("no_proxies")
else:
logger.warn("No proxies available; marking all proxies "
"as unchecked")
self.proxies.reset()
proxy = self.proxies.get_random()
if proxy is None:
logger.error("No proxies available even after a reset.")
raise CloseSpider("no_proxies_after_reset")
username, password = _parse_proxy(proxy)[1:3]
if username and password:
proxy_auth = basic_auth_header(username, password)
request.headers['Proxy-Authorization'] = proxy_auth
request.meta['proxy'] = proxy
request.meta['download_slot'] = self.get_proxy_slot(proxy)
request.meta['_rotating_proxy'] = True
def __init__(self, url):
self.url = url
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or 'http', hostport, '', '', '', ''))
if user and password:
user_pass = b'%s:%s' % (unquote(user), unquote(password))
creds = base64.b64encode(user_pass).strip()
else:
creds = None
self.proxy_type = proxy_type
self.user = user
self.password = password
split_result = hostport.split(':')
self.host = split_result[1]
if len(split_result) == 2:
self.port = split_result[-1]
self.proxy_url = proxy_url
self.creds = creds
self.last_use_time = 0
self.last_success_time = 0
self.last_fail_time = 0
self.use_count = 0
self.success_count = 0
self.fail_count = 0
header = copy.deepcopy(Data.base_header)
header.update({
'Cookie':'ASP.NET_SessionId=%s'%sid['pid'],
})
data = self._make_data({
'__VIEWSTATE': user['__VIEWSTATE'],
'__EVENTVALIDATION': user['__EVENTVALIDATION'],
'userCode': user['pid'],
'userPassword': user['pwd'],
'txtVail': sid['code'],
'Button1': '',
})
request = urllib2.Request( self._urls['s0'], data, header)
proxy = random.choice(self.proxies)
t,_,_,h = urllib2._parse_proxy(proxy)
request.set_proxy(h,t)
request._user = user
rs.append( (rs,self._login_cb ))
for body in Pool.imap(fetch2, rs):
if body is None: continue
def parse_uri(cls, uri): """returns tuple (method, username, password, hostname)""" return urllib2._parse_proxy(uri)
queue = gevent.queue.Queue()
start_time = time.time()
for ip in ips:
gevent.spawn(_create_ssl_connection, (ip, port), timeout, queue)
for i in xrange(len(ips)):
ssl_sock = queue.get()
if ssl_sock:
gevent.spawn(_close_ssl_connection, len(ips)-i-1, queue)
return ssl_sock
else:
self.logger.warning('Http.create_ssl_connection to %s, port=%r return None, try again.', ips, port)
def create_connection_withproxy(self, (host, port), timeout=None, source_address=None, proxy=None):
assert isinstance(proxy, (str, unicode))
self.logger.debug('Http.create_connection_withproxy connect (%r, %r)', host, port)
scheme, username, password, address = urllib2._parse_proxy(proxy or self.proxy)
try:
try:
self.dns_resolve(host)
except socket.error:
pass
proxyhost, _, proxyport = address.rpartition(':')
sock = socket.create_connection((proxyhost, int(proxyport)))
hostname = random.choice(list(self.dns.get(host)) or [host if not host.endswith('.appspot.com') else 'www.google.com'])
request_data = 'CONNECT %s:%s HTTP/1.1\r\n' % (hostname, port)
if username and password:
request_data += 'Proxy-authorization: Basic %s\r\n' % base64.b64encode('%s:%s' % (username, password)).strip()
request_data += '\r\n'
sock.sendall(request_data)
response = httplib.HTTPResponse(sock)
response.begin()
