def extract_cookiejar(self): jar = CookieJar() # self._respose could be None # if this method is called from custom preapre response # function called from spider cache backend if self._response and self._request: jar.extract_cookies( MockResponse(self._response._original_response.msg), MockRequest(self._request), ) return jar
def extract_cookiejar(self): jar = CookieJar() # self._respose could be None # if this method is called from custom preapre response if self._response and self._request: jar.extract_cookies( # pylint: disable=protected-access MockResponse(self._response._original_response.msg), # pylint: enable=protected-access MockRequest(self._request), ) return jar
def extract_cookiejar(self): jar = CookieJar() # self._respose could be None # if this method is called from custom preapre response # function called from spider cache backend if self._response and self._request: jar.extract_cookies( # pylint: disable=protected-access MockResponse(self._response._original_response.msg), # pylint: enable=protected-access MockRequest(self._request), ) return jar
def extract_cookiejar(self, resp, req): jar = CookieJar() jar.extract_cookies(MockResponse(resp._original_response.msg), MockRequest(req)) return jar
class CookieTransport(xmlrpclib.Transport): '''A subclass of xmlrpclib.Transport that supports cookies.''' cookiejar = None scheme = 'http' # Cribbed from xmlrpclib.Transport.send_user_agent def send_cookies(self, connection, cookie_request): if self.cookiejar is None: self.cookiejar = CookieJar() elif self.cookiejar: # Let the cookiejar figure out what cookies are appropriate self.cookiejar.add_cookie_header(cookie_request) # Pull the cookie headers out of the request object... cookielist=list() for h,v in cookie_request.header_items(): if h.startswith('Cookie'): cookielist.append([h,v]) # ...and put them over the connection for h,v in cookielist: connection.putheader(h,v) # This is just python 2.7's xmlrpclib.Transport.single_request, with # send additions noted below to send cookies along with the request def single_request_with_cookies(self, host, handler, request_body, verbose=0): # ADDED: construct the URL and Request object for proper cookie handling request_url = "%s://%s%s" % (self.scheme,host,handler) #log.debug("request_url is %s" % request_url) cookie_request = urllib2.Request(request_url) try: if six.PY2: h = self.make_connection(host) if verbose: h.set_debuglevel(1) self.send_request(h, handler, request_body) self.send_host(h, host) self.send_cookies(h, cookie_request) # ADDED. creates cookiejar if None. self.send_user_agent(h) self.send_content(h, request_body) else: # Python 3 xmlrpc.client.Transport makes its own connection h = self.send_request(host, handler, request_body, verbose) response = h.getresponse() # ADDED: parse headers and get cookies here cookie_response = CookieResponse(response.msg) # Okay, extract the cookies from the headers self.cookiejar.extract_cookies(cookie_response,cookie_request) #log.debug("cookiejar now contains: %s" % self.cookiejar._cookies) # And write back any changes if hasattr(self.cookiejar,'save'): try: self.cookiejar.save(self.cookiejar.filename) except Exception as e: raise #log.error("Couldn't write cookiefile %s: %s" % \ # (self.cookiejar.filename,str(e))) if response.status == 200: self.verbose = verbose return self.parse_response(response) if (response.getheader("content-length", 0)): response.read() raise xmlrpclib.ProtocolError( host + handler, response.status, response.reason, response.msg, ) except xmlrpclib.Fault: raise finally: try: h.close() except NameError: # h not initialized yet pass # Override the appropriate request method single_request = single_request_with_cookies # python 2.7+
class HttpTransport(Transport): """ HTTP transport using urllib2. Provided basic http transport that provides for cookies, proxies but no authentication. """ def __init__(self, **kwargs): """ @param kwargs: Keyword arguments. - B{proxy} - An http proxy to be specified on requests. The proxy is defined as {protocol:proxy,} - type: I{dict} - default: {} - B{timeout} - Set the url open timeout (seconds). - type: I{float} - default: 90 """ Transport.__init__(self) Unskin(self.options).update(kwargs) self.cookiejar = CookieJar() self.proxy = {} self.urlopener = None def open(self, request): try: url = request.url log.debug('opening (%s)', url) u2request = urllib.request.Request(url) self.proxy = self.options.proxy return self.u2open(u2request) except urllib.error.HTTPError as e: raise TransportError(str(e), e.code, e.fp) def send(self, request): result = None url = request.url msg = request.message headers = request.headers try: u2request = urllib.request.Request(url, msg, headers) self.addcookies(u2request) self.proxy = self.options.proxy request.headers.update(u2request.headers) log.debug('sending:\n%s', request) fp = self.u2open(u2request) self.getcookies(fp, u2request) result = Reply(200, fp.headers.__dict__, fp.read()) log.debug('received:\n%s', result) except urllib.error.HTTPError as e: if e.code in (202, 204): result = None else: raise TransportError(e.msg, e.code, e.fp) return result def addcookies(self, u2request): """ Add cookies in the cookiejar to the request. @param u2request: A urllib2 request. @rtype: u2request: urllib2.Requet. """ self.cookiejar.add_cookie_header(u2request) def getcookies(self, fp, u2request): """ Add cookies in the request to the cookiejar. @param u2request: A urllib2 request. @rtype: u2request: urllib2.Requet. """ self.cookiejar.extract_cookies(fp, u2request) def u2open(self, u2request): """ Open a connection. @param u2request: A urllib2 request. @type u2request: urllib2.Requet. @return: The opened file-like urllib2 object. @rtype: fp """ tm = self.options.timeout url = self.u2opener() return url.open(u2request, timeout=tm) def u2opener(self): """ Create a urllib opener. @return: An opener. @rtype: I{OpenerDirector} """ if self.urlopener is None: return urllib.request.build_opener(*self.u2handlers()) else: return self.urlopener def u2handlers(self): """ Get a collection of urllib handlers. @return: A list of handlers to be installed in the opener. @rtype: [Handler,...] """ handlers = [] handlers.append(urllib.request.ProxyHandler(self.proxy)) return handlers def __deepcopy__(self, memo={}): clone = self.__class__() p = Unskin(self.options) cp = Unskin(clone.options) cp.update(p) return clone