def extract_cookiejar(self):
jar = CookieJar()
# self._respose could be None
# if this method is called from custom preapre response
# function called from spider cache backend
if self._response and self._request:
jar.extract_cookies(
MockResponse(self._response._original_response.msg),
MockRequest(self._request),
)
return jar
def extract_cookiejar(self):
jar = CookieJar()
# self._respose could be None
# if this method is called from custom preapre response
if self._response and self._request:
jar.extract_cookies(
# pylint: disable=protected-access
MockResponse(self._response._original_response.msg),
# pylint: enable=protected-access
MockRequest(self._request),
)
return jar
def extract_cookiejar(self):
jar = CookieJar()
# self._respose could be None
# if this method is called from custom preapre response
# function called from spider cache backend
if self._response and self._request:
jar.extract_cookies(
# pylint: disable=protected-access
MockResponse(self._response._original_response.msg),
# pylint: enable=protected-access
MockRequest(self._request),
)
return jar
def extract_cookiejar(self, resp, req):
jar = CookieJar()
jar.extract_cookies(MockResponse(resp._original_response.msg),
MockRequest(req))
return jar
class CookieTransport(xmlrpclib.Transport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist=list()
for h,v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h,v])
# ...and put them over the connection
for h,v in cookielist:
connection.putheader(h,v)
# This is just python 2.7's xmlrpclib.Transport.single_request, with
# send additions noted below to send cookies along with the request
def single_request_with_cookies(self, host, handler, request_body, verbose=0):
# ADDED: construct the URL and Request object for proper cookie handling
request_url = "%s://%s%s" % (self.scheme,host,handler)
#log.debug("request_url is %s" % request_url)
cookie_request = urllib2.Request(request_url)
try:
if six.PY2:
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(h, cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
else:
# Python 3 xmlrpc.client.Transport makes its own connection
h = self.send_request(host, handler, request_body, verbose)
response = h.getresponse()
# ADDED: parse headers and get cookies here
cookie_response = CookieResponse(response.msg)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response,cookie_request)
#log.debug("cookiejar now contains: %s" % self.cookiejar._cookies)
# And write back any changes
if hasattr(self.cookiejar,'save'):
try:
self.cookiejar.save(self.cookiejar.filename)
except Exception as e:
raise
#log.error("Couldn't write cookiefile %s: %s" % \
# (self.cookiejar.filename,str(e)))
if response.status == 200:
self.verbose = verbose
return self.parse_response(response)
if (response.getheader("content-length", 0)):
response.read()
raise xmlrpclib.ProtocolError(
host + handler,
response.status, response.reason,
response.msg,
)
except xmlrpclib.Fault:
raise
finally:
try:
h.close()
except NameError: # h not initialized yet
pass
# Override the appropriate request method
single_request = single_request_with_cookies # python 2.7+
def extract_cookiejar(self, resp, req):
jar = CookieJar()
jar.extract_cookies(MockResponse(resp._original_response.msg),
MockRequest(req))
return jar
class HttpTransport(Transport):
"""
HTTP transport using urllib2. Provided basic http transport
that provides for cookies, proxies but no authentication.
"""
def __init__(self, **kwargs):
"""
@param kwargs: Keyword arguments.
- B{proxy} - An http proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{timeout} - Set the url open timeout (seconds).
- type: I{float}
- default: 90
"""
Transport.__init__(self)
Unskin(self.options).update(kwargs)
self.cookiejar = CookieJar()
self.proxy = {}
self.urlopener = None
def open(self, request):
try:
url = request.url
log.debug('opening (%s)', url)
u2request = urllib.request.Request(url)
self.proxy = self.options.proxy
return self.u2open(u2request)
except urllib.error.HTTPError as e:
raise TransportError(str(e), e.code, e.fp)
def send(self, request):
result = None
url = request.url
msg = request.message
headers = request.headers
try:
u2request = urllib.request.Request(url, msg, headers)
self.addcookies(u2request)
self.proxy = self.options.proxy
request.headers.update(u2request.headers)
log.debug('sending:\n%s', request)
fp = self.u2open(u2request)
self.getcookies(fp, u2request)
result = Reply(200, fp.headers.__dict__, fp.read())
log.debug('received:\n%s', result)
except urllib.error.HTTPError as e:
if e.code in (202, 204):
result = None
else:
raise TransportError(e.msg, e.code, e.fp)
return result
def addcookies(self, u2request):
"""
Add cookies in the cookiejar to the request.
@param u2request: A urllib2 request.
@rtype: u2request: urllib2.Requet.
"""
self.cookiejar.add_cookie_header(u2request)
def getcookies(self, fp, u2request):
"""
Add cookies in the request to the cookiejar.
@param u2request: A urllib2 request.
@rtype: u2request: urllib2.Requet.
"""
self.cookiejar.extract_cookies(fp, u2request)
def u2open(self, u2request):
"""
Open a connection.
@param u2request: A urllib2 request.
@type u2request: urllib2.Requet.
@return: The opened file-like urllib2 object.
@rtype: fp
"""
tm = self.options.timeout
url = self.u2opener()
return url.open(u2request, timeout=tm)
def u2opener(self):
"""
Create a urllib opener.
@return: An opener.
@rtype: I{OpenerDirector}
"""
if self.urlopener is None:
return urllib.request.build_opener(*self.u2handlers())
else:
return self.urlopener
def u2handlers(self):
"""
Get a collection of urllib handlers.
@return: A list of handlers to be installed in the opener.
@rtype: [Handler,...]
"""
handlers = []
handlers.append(urllib.request.ProxyHandler(self.proxy))
return handlers
def __deepcopy__(self, memo={}):
clone = self.__class__()
p = Unskin(self.options)
cp = Unskin(clone.options)
cp.update(p)
return clone
