Ejemplo n.º 1
0
 def extract_cookiejar(self):
     jar = CookieJar()
     # self._respose could be None
     # if this method is called from custom preapre response
     # function called from spider cache backend
     if self._response and self._request:
         jar.extract_cookies(
             MockResponse(self._response._original_response.msg),
             MockRequest(self._request),
         )
     return jar
Ejemplo n.º 2
0
    def test_cookiejar(self):
        cookie1 = create_cookie('foo', 'bar', self.server.address)
        cookie2 = create_cookie('foo', 'bar', self.server.address)
        self.assertFalse(cookie1 == cookie2)

        cookie0 = create_cookie('foo', 'bar', domain='.dumpz.org')
        self.assertEqual(cookie0.domain, '.dumpz.org')

        jar = CookieJar()
        jar.set_cookie(create_cookie('foo', 'bar', domain='foo.com'))
        jar.set_cookie(create_cookie('foo', 'bar', domain='bar.com'))
        self.assertEqual(len(jar), 2)
Ejemplo n.º 3
0
    def test_cookiejar(self):
        c1 = create_cookie('foo', 'bar')
        c2 = create_cookie('foo', 'bar')
        self.assertFalse(c1 == c2)

        c = create_cookie('foo', 'bar', domain='.dumpz.org')
        self.assertEquals(c.domain, '.dumpz.org')

        cj = CookieJar()
        cj.set_cookie(create_cookie('foo', 'bar', domain='foo.com'))
        cj.set_cookie(create_cookie('foo', 'bar', domain='bar.com'))
        self.assertEqual(len(cj), 2)
Ejemplo n.º 4
0
 def extract_cookiejar(self):
     jar = CookieJar()
     # self._respose could be None
     # if this method is called from custom preapre response
     if self._response and self._request:
         jar.extract_cookies(
             # pylint: disable=protected-access
             MockResponse(self._response._original_response.msg),
             # pylint: enable=protected-access
             MockRequest(self._request),
         )
     return jar
Ejemplo n.º 5
0
    def test_cookiejar(self):
        c1 = create_cookie('foo', 'bar', self.server.address)
        c2 = create_cookie('foo', 'bar', self.server.address)
        self.assertFalse(c1 == c2)

        c = create_cookie('foo', 'bar', domain='.dumpz.org')
        self.assertEquals(c.domain, '.dumpz.org')

        cj = CookieJar()
        cj.set_cookie(create_cookie('foo', 'bar', domain='foo.com'))
        cj.set_cookie(create_cookie('foo', 'bar', domain='bar.com'))
        self.assertEqual(len(cj), 2)
Ejemplo n.º 6
0
    def test_cookiejar(self):
        cookie1 = create_cookie('foo', 'bar', self.server.address)
        cookie2 = create_cookie('foo', 'bar', self.server.address)
        self.assertFalse(cookie1 == cookie2)

        cookie0 = create_cookie('foo', 'bar', domain='.dumpz.org')
        self.assertEqual(cookie0.domain, '.dumpz.org')

        jar = CookieJar()
        jar.set_cookie(create_cookie('foo', 'bar', domain='foo.com'))
        jar.set_cookie(create_cookie('foo', 'bar', domain='bar.com'))
        self.assertEqual(len(jar), 2)
Ejemplo n.º 7
0
 def extract_cookiejar(self):
     jar = CookieJar()
     # self._respose could be None
     # if this method is called from custom preapre response
     # function called from spider cache backend
     if self._response and self._request:
         jar.extract_cookies(
             # pylint: disable=protected-access
             MockResponse(self._response._original_response.msg),
             # pylint: enable=protected-access
             MockRequest(self._request),
         )
     return jar
Ejemplo n.º 8
0
 def send_cookies(self, connection, cookie_request):
     if self.cookiejar is None:
         self.cookiejar = CookieJar()
     elif self.cookiejar:
         # Let the cookiejar figure out what cookies are appropriate
         self.cookiejar.add_cookie_header(cookie_request)
         # Pull the cookie headers out of the request object...
         cookielist=list()
         for h,v in cookie_request.header_items():
             if h.startswith('Cookie'):
                 cookielist.append([h,v])
         # ...and put them over the connection
         for h,v in cookielist:
             connection.putheader(h,v)
Ejemplo n.º 9
0
def srtm_login_or_skip(monkeypatch):
    import os
    try:
        srtm_username = os.environ['SRTM_USERNAME']
    except KeyError:
        pytest.skip('SRTM_USERNAME environment variable is unset.')
    try:
        srtm_password = os.environ['SRTM_PASSWORD']
    except KeyError:
        pytest.skip('SRTM_PASSWORD environment variable is unset.')

    from six.moves.urllib.request import (HTTPBasicAuthHandler,
                                          HTTPCookieProcessor,
                                          HTTPPasswordMgrWithDefaultRealm,
                                          build_opener)
    from six.moves.http_cookiejar import CookieJar

    password_manager = HTTPPasswordMgrWithDefaultRealm()
    password_manager.add_password(
        None,
        "https://urs.earthdata.nasa.gov",
        srtm_username,
        srtm_password)
    cookie_jar = CookieJar()
    opener = build_opener(HTTPBasicAuthHandler(password_manager),
                          HTTPCookieProcessor(cookie_jar))

    monkeypatch.setattr(cartopy.io, 'urlopen', opener.open)
Ejemplo n.º 10
0
 def __init__(self, **kwargs):
     """
     @param kwargs: Keyword arguments.
         - B{proxy} - An http proxy to be specified on requests.
              The proxy is defined as {protocol:proxy,}
                 - type: I{dict}
                 - default: {}
         - B{timeout} - Set the url open timeout (seconds).
                 - type: I{float}
                 - default: 90
     """
     Transport.__init__(self)
     Unskin(self.options).update(kwargs)
     self.cookiejar = CookieJar()
     self.proxy = {}
     self.urlopener = None
Ejemplo n.º 11
0
    def __setstate__(self, state):
        state['cookiejar'] = CookieJar()
        for cookie in state['_cookiejar_cookies']:
            state['cookiejar'].set_cookie(cookie)
        del state['_cookiejar_cookies']

        for slot, value in state.items():
            setattr(self, slot, value)
Ejemplo n.º 12
0
Archivo: curl.py Proyecto: Gwill/grab
    def extract_cookiejar(self):
        """
        Extract cookies that pycurl instance knows.

        Returns `CookieJar` object.
        """

        # Example of line:
        # www.google.com\tFALSE\t/accounts/\tFALSE\t0'
        # \tGoogleAccountsLocale_session\ten
        # Fields:
        # * domain
        # * whether or not all machines under that domain can
        # read the cookie's information.
        # * path
        # * Secure Flag: whether or not a secure connection (HTTPS)
        # is required to read the cookie.
        # * exp. timestamp
        # * name
        # * value
        cookiejar = CookieJar()
        for line in self.curl.getinfo(pycurl.INFO_COOKIELIST):
            values = line.split("\t")
            domain = values[0].lower()
            if domain.startswith("#httponly_"):
                domain = domain.replace("#httponly_", "")
                httponly = True
            else:
                httponly = False
            # old
            # cookies[values[-2]] = values[-1]
            # new
            cookie = create_cookie(
                name=values[5],
                value=values[6],
                domain=domain,
                path=values[2],
                secure=values[3] == "TRUE",
                expires=int(values[4]) if values[4] else None,
                httponly=httponly,
            )
            cookiejar.set_cookie(cookie)
        return cookiejar
Ejemplo n.º 13
0
Archivo: curl.py Proyecto: lyicy/grab
    def extract_cookiejar(self):
        """
        Extract cookies that pycurl instance knows.

        Returns `CookieJar` object.
        """

        # Example of line:
        # www.google.com\tFALSE\t/accounts/\tFALSE\t0'
        # \tGoogleAccountsLocale_session\ten
        # Fields:
        # * domain
        # * whether or not all machines under that domain can
        # read the cookie's information.
        # * path
        # * Secure Flag: whether or not a secure connection (HTTPS)
        # is required to read the cookie.
        # * exp. timestamp
        # * name
        # * value
        cookiejar = CookieJar()
        for line in self.curl.getinfo(pycurl.INFO_COOKIELIST):
            values = line.split('\t')
            domain = values[0].lower()
            if domain.startswith('#httponly_'):
                domain = domain.replace('#httponly_', '')
                httponly = True
            else:
                httponly = False
            # old
            # cookies[values[-2]] = values[-1]
            # new
            cookie = create_cookie(
                name=values[5],
                value=values[6],
                domain=domain,
                path=values[2],
                secure=values[3] == "TRUE",
                expires=int(values[4]) if values[4] else None,
                httponly=httponly,
            )
            cookiejar.set_cookie(cookie)
        return cookiejar
Ejemplo n.º 14
0
 def __init__(self, client, method):
     """
     @param client: A suds client.
     @type client: L{Client}
     @param method: A target method.
     @type method: L{Method}
     """
     self.client = client
     self.method = method
     self.options = client.options
     self.cookiejar = CookieJar()
Ejemplo n.º 15
0
Archivo: http.py Proyecto: qtacore/QT4S
 def create_session(self):
     if self.use_requests:
         session = requests.Session()
         session.verify = self._ssl_verify
         session.cert = self._ssl_cert
         session.trust_env = False  # let qt4s handle proxying
         cookie_jar = session.cookies
     else:
         from six.moves.urllib import request
         cookie_jar = CookieJar()
         cookie_processor = request.HTTPCookieProcessor(cookie_jar)
         session = request.build_opener(request.ProxyHandler({}),
                                        cookie_processor)
     return session, cookie_jar
    def __init__(self, server, receiver, oauth_header=None):
        """Initialize the message sender.

        Args:
            server: The bayeux server to send messages to
            receiver: The message receiver to pass the responses to
        """
        self.cookie_jar = CookieJar()
        self.agent = CookieAgent(
            Agent(reactor, pool=HTTPConnectionPool(reactor)), self.cookie_jar)
        self.client_id = -1  #Will be set upon receipt of the handshake response
        self.msg_id = 0
        self.server = server
        self.receiver = receiver
        self.oauth_header = oauth_header
Ejemplo n.º 17
0
    def _make_opener(self, realm, base_url, username, password):
        """HTTP Basic Auth and cookie support for token verification."""
        auth_handler = HTTPBasicAuthHandler()
        auth_handler.add_password(realm=realm,
                                  uri=base_url,
                                  user=username,
                                  passwd=password)
        opener = build_opener(auth_handler)
        install_opener(opener)

        cookie_jar = CookieJar()
        cookie_handler = HTTPCookieProcessor(cookie_jar)

        handlers = [auth_handler, cookie_handler]
        opener = build_opener(*handlers)
        return opener
Ejemplo n.º 18
0
    def __init__(self, url):
        if url.startswith('https://'):
            self._transport = GSSAPITransport()
        elif url.startswith('http://'):
            raise NitrateError(
                "Encrypted https communication required for "
                "GSSAPI authentication.\nURL provided: {0}".format(url))
        else:
            raise NitrateError("Unrecognized URL scheme: {0}".format(url))

        self._transport.cookiejar = CookieJar()
        # print("COOKIES:", self._transport.cookiejar._cookies)
        self.server = xmlrpclib.ServerProxy(url,
                                            transport=self._transport,
                                            verbose=VERBOSE,
                                            allow_none=1)

        # Login, get a cookie into our cookie jar:
        login_dict = self.do_command("Auth.login_krbv", [])
Ejemplo n.º 19
0
    def __init__(self, username, password, url, use_mod_auth_kerb=False):
        if url.startswith('https://'):
            self._transport = SafeCookieTransport()
        elif url.startswith('http://'):
            self._transport = CookieTransport()
        else:
            raise NitrateError("Unrecognized URL scheme")

        self._transport.cookiejar = CookieJar()
        # print("COOKIES:", self._transport.cookiejar._cookies)
        self.server = xmlrpclib.ServerProxy(url,
                                            transport=self._transport,
                                            verbose=VERBOSE,
                                            allow_none=1)

        # Login, get a cookie into our cookie jar:
        login_dict = self.do_command(
            "Auth.login", [dict(
                username=username,
                password=password,
            )])
Ejemplo n.º 20
0
def _get_initial_token(url):
    """
    Create initial connection to get authentication token for future
    requests.

    Returns a string to be used in subsequent connections with the
    X-CSRFToken header or the empty string if we didn't find any token in
    the cookies.
    """
    logging.info('Getting initial CSRF token.')

    cookiejar = CookieJar()
    opener = build_opener(HTTPCookieProcessor(cookiejar))
    install_opener(opener)
    opener.open(url)

    for cookie in cookiejar:
        if cookie.name == 'csrftoken':
            logging.info('Found CSRF token.')
            return cookie.value

    logging.warn('Did not find the CSRF token.')
    return ''
Ejemplo n.º 21
0
    def __init__(self, name, host=None, username=None, password=None):
        """Constructor.

        :param name:
        :type name: string
        :param host:
        :type host: string
        :param username:
        :type username: string
        :param password:
        :type password: string
        """
        self.name = name
        self.username = app.TORRENT_USERNAME if username is None else username
        self.password = app.TORRENT_PASSWORD if password is None else password
        self.host = app.TORRENT_HOST if host is None else host
        self.rpcurl = app.TORRENT_RPCURL
        self.url = None
        self.response = None
        self.auth = None
        self.last_time = time.time()
        self.session = helpers.make_session()
        self.session.auth = (self.username, self.password)
        self.session.cookies = CookieJar()
Ejemplo n.º 22
0
def url_downloader(url, data=None, path=None, cookie=None,
                   timeout=5, retry=1, retry_ivl=5, agent=None, proxy=None):
    """Download URL link
    url:    url to download
    data:   post data
    path:   download to local file
    timeout:    socket timeout
    retry:  retry times to download url
    retry_ivl:  interval time when retry
    agent:  http user agent
    proxy:  socks5://127.0.0.1:1080
    """
    while True:
        try:
            if isinstance(data, dict):
                data = urlencode(data)
            request = Request(url, data=data)
            request.add_header('User-Agent', agent or get_user_agent())
            if data:
                request.add_header(
                    'Content-Type',
                    'application/x-www-form-urlencoded;charset=utf-8')
            response = None
            handlers = []
            if proxy:
                scheme, host, port = proxy.split(':')
                host = host.strip('/')
                proxy_handler = SocksiPyHandler(
                    socks.PROXY_TYPES[scheme.upper()], host, int(port)
                )
                handlers.append(proxy_handler)
            if cookie is None:
                cookie = CookieJar()
            cookie_handler = HTTPCookieProcessor(cookie)
            handlers.append(cookie_handler)

            opener = build_opener(*handlers)
            response = opener.open(request, timeout=timeout)
            content_encoding = response.info().get('content-encoding')
            if content_encoding:
                r_data = gzip.decompress(response.read())
            else:
                r_data = response.read()
            if path:
                with open(path, 'wb') as f:
                    f.write(r_data)
                r_data = None
            response.close()
            mime = response.info().get('content-type')
            real_url = response.geturl()
            err_msg = 'Ok'
            break
        except (URLError, socket.error, Exception) as err:
            response and response.close()
            retry -= 1
            err_msg = str(err)
            if retry > 0:
                time.sleep(retry_ivl)
                retry_ivl += retry_ivl
                timeout += timeout
            else:
                mime = r_data = real_url = None
                break
    return {
        'mime': mime,
        'path': path,
        'data': r_data,
        'url': real_url,
        'cookie': cookie,
        'error': err_msg,
    }
Ejemplo n.º 23
0
RATING_NUKED = 2

CODEC_UNKNOWN = 0
CODEC_XVID = 1
CODEC_H264 = 2
CODEC_H265 = 3
CODEC_MP3 = 4
CODEC_AAC = 5
CODEC_AC3 = 6
CODEC_DTS = 7
CODEC_DTSHD = 8
CODEC_DTSHDMA = 9

USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.66 Safari/537.36"

COOKIE_JAR = CookieJar()
urllib_request.install_opener(urllib_request.build_opener(urllib_request.HTTPCookieProcessor(COOKIE_JAR)))

class closing(object):
    def __init__(self, thing):
        self.thing = thing

    def __enter__(self):
        return self.thing

    def __exit__(self, *exc_info):
        self.thing.close()


def parse_json(data):
    try:
Ejemplo n.º 24
0
 def from_cookie_list(cls, clist):
     jar = CookieJar()
     for cookie in clist:
         jar.set_cookie(cookie)
     return cls(jar)
Ejemplo n.º 25
0
 def __init__(self, cookiejar=None):
     if cookiejar is not None:
         self.cookiejar = cookiejar
     else:
         self.cookiejar = CookieJar()
Ejemplo n.º 26
0
class CookieManager(object):
    """
    Each Grab instance has `cookies` attribute that is instance of
    `CookieManager` class.

    That class contains helpful methods to create, load, save cookies from/to
    different places.
    """

    __slots__ = ('cookiejar',)

    def __init__(self, cookiejar=None):
        if cookiejar is not None:
            self.cookiejar = cookiejar
        else:
            self.cookiejar = CookieJar()
        # self.disable_cookiejar_lock(self.cookiejar)

    # def disable_cookiejar_lock(self, cj):
        # cj._cookies_lock = dummy_threading.RLock()

    def set(self, name, value, domain, **kwargs):
        """Add new cookie or replace existing cookie with same parameters.

        :param name: name of cookie
        :param value: value of cookie
        :param kwargs: extra attributes of cookie
        """

        if domain == 'localhost':
            domain = ''

        self.cookiejar.set_cookie(create_cookie(name, value, domain, **kwargs))

    def update(self, cookies):
        if isinstance(cookies, CookieJar):
            for cookie in cookies:
                self.cookiejar.set_cookie(cookie)
        elif isinstance(cookies, CookieManager):
            for cookie in cookies.cookiejar:
                self.cookiejar.set_cookie(cookie)
        else:
            raise GrabMisuseError('Unknown type of cookies argument: %s'
                                  % type(cookies))

    @classmethod
    def from_cookie_list(cls, clist):
        cj = CookieJar()
        for cookie in clist:
            cj.set_cookie(cookie)
        return cls(cj)

    def clear(self):
        self.cookiejar = CookieJar()

    def __getstate__(self):
        state = {}
        for cls in type(self).mro():
            cls_slots = getattr(cls, '__slots__', ())
            for slot in cls_slots:
                if slot != '__weakref__':
                    if hasattr(self, slot):
                        state[slot] = getattr(self, slot)

        state['_cookiejar_cookies'] = list(self.cookiejar)
        del state['cookiejar']

        return state

    def __setstate__(self, state):
        state['cookiejar'] = CookieJar()
        for cookie in state['_cookiejar_cookies']:
            state['cookiejar'].set_cookie(cookie)
        del state['_cookiejar_cookies']

        for slot, value in state.items():
            setattr(self, slot, value)

    def __getitem__(self, key):
        for cookie in self.cookiejar:
            if cookie.name == key:
                return cookie.value
        raise KeyError

    def items(self):
        res = []
        for cookie in self.cookiejar:
            res.append((cookie.name, cookie.value))
        return res

    def load_from_file(self, path):
        """
        Load cookies from the file.

        Content of file should be a JSON-serialized list of dicts.
        """

        with open(path) as inf:
            data = inf.read()
            if data:
                items = json.loads(data)
            else:
                items = {}
        for item in items:
            extra = dict((x, y) for x, y in item.items()
                         if x not in ['name', 'value', 'domain'])
            self.set(item['name'], item['value'], item['domain'], **extra)

    def get_dict(self):
        res = []
        for cookie in self.cookiejar:
            res.append(dict((x, getattr(cookie, x)) for x in COOKIE_ATTRS))
        return res

    def save_to_file(self, path):
        """
        Dump all cookies to file.

        Cookies are dumped as JSON-serialized dict of keys and values.
        """

        with open(path, 'w') as out:
            out.write(json.dumps(self.get_dict()))
Ejemplo n.º 27
0
 def clear(self):
     self.cookiejar = CookieJar()
Ejemplo n.º 28
0
class HttpTransport(Transport):
    """
    HTTP transport using urllib2.  Provided basic http transport
    that provides for cookies, proxies but no authentication.
    """

    def __init__(self, **kwargs):
        """
        @param kwargs: Keyword arguments.
            - B{proxy} - An http proxy to be specified on requests.
                 The proxy is defined as {protocol:proxy,}
                    - type: I{dict}
                    - default: {}
            - B{timeout} - Set the url open timeout (seconds).
                    - type: I{float}
                    - default: 90
        """
        Transport.__init__(self)
        Unskin(self.options).update(kwargs)
        self.cookiejar = CookieJar()
        self.proxy = {}
        self.urlopener = None

    def open(self, request):
        try:
            url = request.url
            log.debug('opening (%s)', url)
            u2request = urllib.request.Request(url)
            self.proxy = self.options.proxy
            return self.u2open(u2request)
        except urllib.error.HTTPError as e:
            raise TransportError(str(e), e.code, e.fp)

    def send(self, request):
        result = None
        url = request.url
        msg = request.message
        headers = request.headers
        try:
            u2request = urllib.request.Request(url, msg, headers)
            self.addcookies(u2request)
            self.proxy = self.options.proxy
            request.headers.update(u2request.headers)
            log.debug('sending:\n%s', request)
            fp = self.u2open(u2request)
            self.getcookies(fp, u2request)
            result = Reply(200, fp.headers.__dict__, fp.read())
            log.debug('received:\n%s', result)
        except urllib.error.HTTPError as e:
            if e.code in (202, 204):
                result = None
            else:
                raise TransportError(e.msg, e.code, e.fp)
        return result

    def addcookies(self, u2request):
        """
        Add cookies in the cookiejar to the request.
        @param u2request: A urllib2 request.
        @rtype: u2request: urllib2.Requet.
        """
        self.cookiejar.add_cookie_header(u2request)

    def getcookies(self, fp, u2request):
        """
        Add cookies in the request to the cookiejar.
        @param u2request: A urllib2 request.
        @rtype: u2request: urllib2.Requet.
        """
        self.cookiejar.extract_cookies(fp, u2request)

    def u2open(self, u2request):
        """
        Open a connection.
        @param u2request: A urllib2 request.
        @type u2request: urllib2.Requet.
        @return: The opened file-like urllib2 object.
        @rtype: fp
        """
        tm = self.options.timeout
        url = self.u2opener()
        return url.open(u2request, timeout=tm)

    def u2opener(self):
        """
        Create a urllib opener.
        @return: An opener.
        @rtype: I{OpenerDirector}
        """
        if self.urlopener is None:
            return urllib.request.build_opener(*self.u2handlers())
        else:
            return self.urlopener

    def u2handlers(self):
        """
        Get a collection of urllib handlers.
        @return: A list of handlers to be installed in the opener.
        @rtype: [Handler,...]
        """
        handlers = []
        handlers.append(urllib.request.ProxyHandler(self.proxy))
        return handlers

    def __deepcopy__(self, memo={}):
        clone = self.__class__()
        p = Unskin(self.options)
        cp = Unskin(clone.options)
        cp.update(p)
        return clone
Ejemplo n.º 29
0
    def __init__(self,
                 url,
                 cookie_file=None,
                 username=None,
                 password=None,
                 api_token=None,
                 agent=None,
                 session=None,
                 disable_proxy=False,
                 auth_callback=None,
                 otp_token_callback=None,
                 verify_ssl=True,
                 save_cookies=True,
                 ext_auth_cookies=None):
        if not url.endswith('/'):
            url += '/'

        self.url = url + 'api/'

        self.save_cookies = save_cookies
        self.ext_auth_cookies = ext_auth_cookies

        if self.save_cookies:
            self.cookie_jar, self.cookie_file = create_cookie_jar(
                cookie_file=cookie_file)

            try:
                self.cookie_jar.load(ignore_expires=True)
            except IOError:
                pass
        else:
            self.cookie_jar = CookieJar()
            self.cookie_file = None

        if self.ext_auth_cookies:
            try:
                self.cookie_jar.load(ext_auth_cookies, ignore_expires=True)
            except IOError as e:
                logging.critical(
                    'There was an error while loading a '
                    'cookie file: %s', e)
                pass

        # Get the cookie domain from the url. If the domain
        # does not contain a '.' (e.g. 'localhost'), we assume
        # it is a local domain and suffix it (See RFC 2109).
        parsed_url = urlparse(url)
        self.domain = parsed_url[1].partition(':')[0]  # Remove Port.

        if self.domain.count('.') < 1:
            self.domain = '%s.local' % self.domain

        if session:
            cookie = Cookie(version=0,
                            name=RB_COOKIE_NAME,
                            value=session,
                            port=None,
                            port_specified=False,
                            domain=self.domain,
                            domain_specified=True,
                            domain_initial_dot=True,
                            path=parsed_url[2],
                            path_specified=True,
                            secure=False,
                            expires=None,
                            discard=False,
                            comment=None,
                            comment_url=None,
                            rest={'HttpOnly': None})
            self.cookie_jar.set_cookie(cookie)

            if self.save_cookies:
                self.cookie_jar.save()

        if username:
            # If the username parameter is given, we have to clear the session
            # cookie manually or it will override the username:password
            # combination retrieved from the authentication callback.
            try:
                self.cookie_jar.clear(self.domain, parsed_url[2],
                                      RB_COOKIE_NAME)
            except KeyError:
                pass

        # Set up the HTTP libraries to support all of the features we need.
        password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password,
                                                  api_token, auth_callback,
                                                  otp_token_callback)
        self.preset_auth_handler = PresetHTTPAuthHandler(
            self.url, password_mgr)

        handlers = []

        if not verify_ssl:
            context = ssl._create_unverified_context()
            handlers.append(HTTPSHandler(context=context))

        if disable_proxy:
            handlers.append(ProxyHandler({}))

        handlers += [
            HTTPCookieProcessor(self.cookie_jar),
            ReviewBoardHTTPBasicAuthHandler(password_mgr),
            HTTPDigestAuthHandler(password_mgr),
            self.preset_auth_handler,
            ReviewBoardHTTPErrorProcessor(),
        ]

        if agent:
            self.agent = agent
        else:
            self.agent = ('RBTools/' + get_package_version()).encode('utf-8')

        opener = build_opener(*handlers)
        opener.addheaders = [
            (str('User-agent'), str(self.agent)),
        ]
        install_opener(opener)

        self._cache = None
        self._urlopen = urlopen
Ejemplo n.º 30
0
class ReviewBoardServer(object):
    """Represents a Review Board server we are communicating with.

    Provides methods for executing HTTP requests on a Review Board
    server's Web API.

    The ``auth_callback`` parameter can be used to specify a callable
    which will be called when authentication fails. This callable will
    be passed the realm, and url of the Review Board server and should
    return a 2-tuple of username, password. The user can be prompted
    for their credentials using this mechanism.
    """
    def __init__(self,
                 url,
                 cookie_file=None,
                 username=None,
                 password=None,
                 api_token=None,
                 agent=None,
                 session=None,
                 disable_proxy=False,
                 auth_callback=None,
                 otp_token_callback=None,
                 verify_ssl=True,
                 save_cookies=True,
                 ext_auth_cookies=None):
        if not url.endswith('/'):
            url += '/'

        self.url = url + 'api/'

        self.save_cookies = save_cookies
        self.ext_auth_cookies = ext_auth_cookies

        if self.save_cookies:
            self.cookie_jar, self.cookie_file = create_cookie_jar(
                cookie_file=cookie_file)

            try:
                self.cookie_jar.load(ignore_expires=True)
            except IOError:
                pass
        else:
            self.cookie_jar = CookieJar()
            self.cookie_file = None

        if self.ext_auth_cookies:
            try:
                self.cookie_jar.load(ext_auth_cookies, ignore_expires=True)
            except IOError as e:
                logging.critical(
                    'There was an error while loading a '
                    'cookie file: %s', e)
                pass

        # Get the cookie domain from the url. If the domain
        # does not contain a '.' (e.g. 'localhost'), we assume
        # it is a local domain and suffix it (See RFC 2109).
        parsed_url = urlparse(url)
        self.domain = parsed_url[1].partition(':')[0]  # Remove Port.

        if self.domain.count('.') < 1:
            self.domain = '%s.local' % self.domain

        if session:
            cookie = Cookie(version=0,
                            name=RB_COOKIE_NAME,
                            value=session,
                            port=None,
                            port_specified=False,
                            domain=self.domain,
                            domain_specified=True,
                            domain_initial_dot=True,
                            path=parsed_url[2],
                            path_specified=True,
                            secure=False,
                            expires=None,
                            discard=False,
                            comment=None,
                            comment_url=None,
                            rest={'HttpOnly': None})
            self.cookie_jar.set_cookie(cookie)

            if self.save_cookies:
                self.cookie_jar.save()

        if username:
            # If the username parameter is given, we have to clear the session
            # cookie manually or it will override the username:password
            # combination retrieved from the authentication callback.
            try:
                self.cookie_jar.clear(self.domain, parsed_url[2],
                                      RB_COOKIE_NAME)
            except KeyError:
                pass

        # Set up the HTTP libraries to support all of the features we need.
        password_mgr = ReviewBoardHTTPPasswordMgr(self.url, username, password,
                                                  api_token, auth_callback,
                                                  otp_token_callback)
        self.preset_auth_handler = PresetHTTPAuthHandler(
            self.url, password_mgr)

        handlers = []

        if not verify_ssl:
            context = ssl._create_unverified_context()
            handlers.append(HTTPSHandler(context=context))

        if disable_proxy:
            handlers.append(ProxyHandler({}))

        handlers += [
            HTTPCookieProcessor(self.cookie_jar),
            ReviewBoardHTTPBasicAuthHandler(password_mgr),
            HTTPDigestAuthHandler(password_mgr),
            self.preset_auth_handler,
            ReviewBoardHTTPErrorProcessor(),
        ]

        if agent:
            self.agent = agent
        else:
            self.agent = ('RBTools/' + get_package_version()).encode('utf-8')

        opener = build_opener(*handlers)
        opener.addheaders = [
            (str('User-agent'), str(self.agent)),
        ]
        install_opener(opener)

        self._cache = None
        self._urlopen = urlopen

    def enable_cache(self, cache_location=None, in_memory=False):
        """Enable caching for all future HTTP requests.

        The cache will be created at the default location if none is provided.

        If the in_memory parameter is True, the cache will be created in memory
        instead of on disk. This overrides the cache_location parameter.
        """
        if not self._cache:
            self._cache = APICache(create_db_in_memory=in_memory,
                                   db_location=cache_location)

            self._urlopen = self._cache.make_request

    def login(self, username, password):
        """Reset the user information"""
        self.preset_auth_handler.reset(username, password)

    def logout(self):
        """Logs the user out of the session."""
        self.preset_auth_handler.reset(None, None)
        self.make_request(HttpRequest('%ssession/' % self.url,
                                      method='DELETE'))
        self.cookie_jar.clear(self.domain)

        if self.save_cookies:
            self.cookie_jar.save()

    def process_error(self, http_status, data):
        """Processes an error, raising an APIError with the information."""
        # In Python 3, the data can be bytes, not str, and json.loads
        # explicitly requires decoded strings.
        data = force_unicode(data)

        try:
            rsp = json_loads(data)

            assert rsp['stat'] == 'fail'

            logging.debug('Got API Error %d (HTTP code %d): %s',
                          rsp['err']['code'], http_status, rsp['err']['msg'])
            logging.debug('Error data: %r', rsp)

            raise create_api_error(http_status, rsp['err']['code'], rsp,
                                   rsp['err']['msg'])
        except ValueError:
            logging.debug('Got HTTP error: %s: %s', http_status, data)
            raise APIError(http_status, None, None, data)

    def make_request(self, request):
        """Perform an http request.

        The request argument should be an instance of
        'rbtools.api.request.HttpRequest'.
        """
        try:
            content_type, body = request.encode_multipart_formdata()
            headers = request.headers

            if body:
                headers.update({
                    'Content-Type': content_type,
                    'Content-Length': str(len(body)),
                })
            else:
                headers['Content-Length'] = '0'

            rsp = self._urlopen(
                Request(request.url, body, headers, request.method))
        except HTTPError as e:
            self.process_error(e.code, e.read())
        except URLError as e:
            raise ServerInterfaceError('%s' % e.reason)

        if self.save_cookies:
            try:
                self.cookie_jar.save()
            except IOError:
                pass

        return rsp
Ejemplo n.º 31
0
class CookieManager(object):
    """
    Each Grab instance has `cookies` attribute that is instance of
    `CookieManager` class.

    That class contains helpful methods to create, load, save cookies from/to
    different places.
    """

    __slots__ = ('cookiejar', )

    def __init__(self, cookiejar=None):
        if cookiejar is not None:
            self.cookiejar = cookiejar
        else:
            self.cookiejar = CookieJar()
        # self.disable_cookiejar_lock(self.cookiejar)

    # def disable_cookiejar_lock(self, cj):
    # cj._cookies_lock = dummy_threading.RLock()

    def set(self, name, value, domain, **kwargs):
        """Add new cookie or replace existing cookie with same parameters.

        :param name: name of cookie
        :param value: value of cookie
        :param kwargs: extra attributes of cookie
        """

        if domain == 'localhost':
            domain = ''

        self.cookiejar.set_cookie(create_cookie(name, value, domain, **kwargs))

    def update(self, cookies):
        if isinstance(cookies, CookieJar):
            for cookie in cookies:
                self.cookiejar.set_cookie(cookie)
        elif isinstance(cookies, CookieManager):
            for cookie in cookies.cookiejar:
                self.cookiejar.set_cookie(cookie)
        else:
            raise GrabMisuseError('Unknown type of cookies argument: %s' %
                                  type(cookies))

    @classmethod
    def from_cookie_list(cls, clist):
        jar = CookieJar()
        for cookie in clist:
            jar.set_cookie(cookie)
        return cls(jar)

    def clear(self):
        self.cookiejar = CookieJar()

    def __getstate__(self):
        state = {}
        for cls in type(self).mro():
            cls_slots = getattr(cls, '__slots__', ())
            for slot in cls_slots:
                if slot != '__weakref__':
                    if hasattr(self, slot):
                        state[slot] = getattr(self, slot)

        state['_cookiejar_cookies'] = list(self.cookiejar)
        del state['cookiejar']

        return state

    def __setstate__(self, state):
        state['cookiejar'] = CookieJar()
        for cookie in state['_cookiejar_cookies']:
            state['cookiejar'].set_cookie(cookie)
        del state['_cookiejar_cookies']

        for slot, value in state.items():
            setattr(self, slot, value)

    def __getitem__(self, key):
        for cookie in self.cookiejar:
            if cookie.name == key:
                return cookie.value
        raise KeyError

    def items(self):
        res = []
        for cookie in self.cookiejar:
            res.append((cookie.name, cookie.value))
        return res

    def load_from_file(self, path):
        """
        Load cookies from the file.

        Content of file should be a JSON-serialized list of dicts.
        """

        with open(path) as inf:
            data = inf.read()
            if data:
                items = json.loads(data)
            else:
                items = {}
        for item in items:
            extra = dict((x, y) for x, y in item.items()
                         if x not in ['name', 'value', 'domain'])
            self.set(item['name'], item['value'], item['domain'], **extra)

    def get_dict(self):
        res = []
        for cookie in self.cookiejar:
            res.append(dict((x, getattr(cookie, x)) for x in COOKIE_ATTRS))
        return res

    def save_to_file(self, path):
        """
        Dump all cookies to file.

        Cookies are dumped as JSON-serialized dict of keys and values.
        """

        with open(path, 'w') as out:
            out.write(json.dumps(self.get_dict()))

    def get_cookie_header(self, req):
        """
        :param req: object with httplib.Request interface
            Actually, it have to have `url` and `headers` attributes
        """
        mocked_req = MockRequest(req)
        self.cookiejar.add_cookie_header(mocked_req)
        return mocked_req.get_new_headers().get('Cookie')
Ejemplo n.º 32
0
    def __init__(self, url, cookie_file=None, username=None, password=None,
                 api_token=None, agent=None, session=None, disable_proxy=False,
                 auth_callback=None, otp_token_callback=None,
                 verify_ssl=True, save_cookies=True):
        if not url.endswith('/'):
            url += '/'

        self.url = url + 'api/'

        self.save_cookies = save_cookies

        if self.save_cookies:
            self.cookie_jar, self.cookie_file = create_cookie_jar(
                cookie_file=cookie_file)

            try:
                self.cookie_jar.load(ignore_expires=True)
            except IOError:
                pass
        else:
            self.cookie_jar = CookieJar()
            self.cookie_file = None

        # Get the cookie domain from the url. If the domain
        # does not contain a '.' (e.g. 'localhost'), we assume
        # it is a local domain and suffix it (See RFC 2109).
        parsed_url = urlparse(url)
        self.domain = parsed_url[1].partition(':')[0]  # Remove Port.

        if self.domain.count('.') < 1:
            self.domain = '%s.local' % self.domain

        if session:
            cookie = Cookie(
                version=0,
                name=RB_COOKIE_NAME,
                value=session,
                port=None,
                port_specified=False,
                domain=self.domain,
                domain_specified=True,
                domain_initial_dot=True,
                path=parsed_url[2],
                path_specified=True,
                secure=False,
                expires=None,
                discard=False,
                comment=None,
                comment_url=None,
                rest={'HttpOnly': None})
            self.cookie_jar.set_cookie(cookie)

            if self.save_cookies:
                self.cookie_jar.save()

        if username:
            # If the username parameter is given, we have to clear the session
            # cookie manually or it will override the username:password
            # combination retrieved from the authentication callback.
            try:
                self.cookie_jar.clear(self.domain, parsed_url[2],
                                      RB_COOKIE_NAME)
            except KeyError:
                pass

        # Set up the HTTP libraries to support all of the features we need.
        password_mgr = ReviewBoardHTTPPasswordMgr(self.url,
                                                  username,
                                                  password,
                                                  api_token,
                                                  auth_callback,
                                                  otp_token_callback)
        self.preset_auth_handler = PresetHTTPAuthHandler(self.url,
                                                         password_mgr)

        handlers = []

        if not verify_ssl:
            context = ssl._create_unverified_context()
            handlers.append(HTTPSHandler(context=context))

        if disable_proxy:
            handlers.append(ProxyHandler({}))

        handlers += [
            HTTPCookieProcessor(self.cookie_jar),
            ReviewBoardHTTPBasicAuthHandler(password_mgr),
            HTTPDigestAuthHandler(password_mgr),
            self.preset_auth_handler,
            ReviewBoardHTTPErrorProcessor(),
        ]

        if agent:
            self.agent = agent
        else:
            self.agent = ('RBTools/' + get_package_version()).encode('utf-8')

        opener = build_opener(*handlers)
        opener.addheaders = [
            (b'User-agent', self.agent),
        ]
        install_opener(opener)

        self._cache = None
        self._urlopen = urlopen
Ejemplo n.º 33
0
class ReviewBoardServer(object):
    """Represents a Review Board server we are communicating with.

    Provides methods for executing HTTP requests on a Review Board
    server's Web API.

    The ``auth_callback`` parameter can be used to specify a callable
    which will be called when authentication fails. This callable will
    be passed the realm, and url of the Review Board server and should
    return a 2-tuple of username, password. The user can be prompted
    for their credentials using this mechanism.
    """
    def __init__(self, url, cookie_file=None, username=None, password=None,
                 api_token=None, agent=None, session=None, disable_proxy=False,
                 auth_callback=None, otp_token_callback=None,
                 verify_ssl=True, save_cookies=True):
        if not url.endswith('/'):
            url += '/'

        self.url = url + 'api/'

        self.save_cookies = save_cookies

        if self.save_cookies:
            self.cookie_jar, self.cookie_file = create_cookie_jar(
                cookie_file=cookie_file)

            try:
                self.cookie_jar.load(ignore_expires=True)
            except IOError:
                pass
        else:
            self.cookie_jar = CookieJar()
            self.cookie_file = None

        # Get the cookie domain from the url. If the domain
        # does not contain a '.' (e.g. 'localhost'), we assume
        # it is a local domain and suffix it (See RFC 2109).
        parsed_url = urlparse(url)
        self.domain = parsed_url[1].partition(':')[0]  # Remove Port.

        if self.domain.count('.') < 1:
            self.domain = '%s.local' % self.domain

        if session:
            cookie = Cookie(
                version=0,
                name=RB_COOKIE_NAME,
                value=session,
                port=None,
                port_specified=False,
                domain=self.domain,
                domain_specified=True,
                domain_initial_dot=True,
                path=parsed_url[2],
                path_specified=True,
                secure=False,
                expires=None,
                discard=False,
                comment=None,
                comment_url=None,
                rest={'HttpOnly': None})
            self.cookie_jar.set_cookie(cookie)

            if self.save_cookies:
                self.cookie_jar.save()

        if username:
            # If the username parameter is given, we have to clear the session
            # cookie manually or it will override the username:password
            # combination retrieved from the authentication callback.
            try:
                self.cookie_jar.clear(self.domain, parsed_url[2],
                                      RB_COOKIE_NAME)
            except KeyError:
                pass

        # Set up the HTTP libraries to support all of the features we need.
        password_mgr = ReviewBoardHTTPPasswordMgr(self.url,
                                                  username,
                                                  password,
                                                  api_token,
                                                  auth_callback,
                                                  otp_token_callback)
        self.preset_auth_handler = PresetHTTPAuthHandler(self.url,
                                                         password_mgr)

        handlers = []

        if not verify_ssl:
            context = ssl._create_unverified_context()
            handlers.append(HTTPSHandler(context=context))

        if disable_proxy:
            handlers.append(ProxyHandler({}))

        handlers += [
            HTTPCookieProcessor(self.cookie_jar),
            ReviewBoardHTTPBasicAuthHandler(password_mgr),
            HTTPDigestAuthHandler(password_mgr),
            self.preset_auth_handler,
            ReviewBoardHTTPErrorProcessor(),
        ]

        if agent:
            self.agent = agent
        else:
            self.agent = ('RBTools/' + get_package_version()).encode('utf-8')

        opener = build_opener(*handlers)
        opener.addheaders = [
            (b'User-agent', self.agent),
        ]
        install_opener(opener)

        self._cache = None
        self._urlopen = urlopen

    def enable_cache(self, cache_location=None, in_memory=False):
        """Enable caching for all future HTTP requests.

        The cache will be created at the default location if none is provided.

        If the in_memory parameter is True, the cache will be created in memory
        instead of on disk. This overrides the cache_location parameter.
        """
        if not self._cache:
            self._cache = APICache(create_db_in_memory=in_memory,
                                   db_location=cache_location)

            self._urlopen = self._cache.make_request

    def login(self, username, password):
        """Reset the user information"""
        self.preset_auth_handler.reset(username, password)

    def logout(self):
        """Logs the user out of the session."""
        self.preset_auth_handler.reset(None, None)
        self.make_request(HttpRequest('%ssession/' % self.url,
                                      method='DELETE'))
        self.cookie_jar.clear(self.domain)

        if self.save_cookies:
            self.cookie_jar.save()

    def process_error(self, http_status, data):
        """Processes an error, raising an APIError with the information."""
        try:
            rsp = json_loads(data)

            assert rsp['stat'] == 'fail'

            logging.debug('Got API Error %d (HTTP code %d): %s' %
                          (rsp['err']['code'], http_status, rsp['err']['msg']))
            logging.debug('Error data: %r' % rsp)

            raise create_api_error(http_status, rsp['err']['code'], rsp,
                                   rsp['err']['msg'])
        except ValueError:
            logging.debug('Got HTTP error: %s: %s' % (http_status, data))
            raise APIError(http_status, None, None, data)

    def make_request(self, request):
        """Perform an http request.

        The request argument should be an instance of
        'rbtools.api.request.HttpRequest'.
        """
        try:
            content_type, body = request.encode_multipart_formdata()
            headers = request.headers

            if body:
                headers.update({
                    b'Content-Type': content_type,
                    b'Content-Length': str(len(body)),
                })
            else:
                headers[b'Content-Length'] = '0'

            r = Request(request.url.encode('utf-8'), body, headers,
                        request.method.encode('utf-8'))
            rsp = self._urlopen(r)
        except HTTPError as e:
            self.process_error(e.code, e.read())
        except URLError as e:
            raise ServerInterfaceError('%s' % e.reason)

        if self.save_cookies:
            try:
                self.cookie_jar.save()
            except IOError:
                pass

        return rsp
Ejemplo n.º 34
0
 def __init__(self, cookiejar=None):
     if cookiejar is not None:
         self.cookiejar = cookiejar
     else:
         self.cookiejar = CookieJar()
Ejemplo n.º 35
0
 def from_cookie_list(cls, clist):
     jar = CookieJar()
     for cookie in clist:
         jar.set_cookie(cookie)
     return cls(jar)
Ejemplo n.º 36
0
    def setUp(self):
        MongoTestCase.setUp(self)
        datadir = pkg_resources.resource_filename(__name__, 'data')
        staticdir = pkg_resources.resource_filename(__name__, 'static/en')
        self.redis_instance = RedisTemporaryInstance.get_instance()
        # load the IdP configuration
        _test_config = {
            'debug': True,
            'insecure_cookies': False,
            'mongo_uri': self.tmp_db.uri,
            'environment': 'test_suite',
            'pysaml2_config': os.path.join(datadir, 'test_SSO_conf.py'),
            'static_dir': staticdir,
            'tou_version': 'mock-version',
            'shared_session_secret_key': 'shared-session-secret-key',
            'shared_session_ttl': 30,
            'redis_host': 'localhost',
            'redis_port': str(self.redis_instance.port),
            'redis_db': '0',
            'listen_addr': 'unittest-idp.example.edu',
            'listen_port': 443,
            'base_url': 'https://unittest-idp.example.edu/',
            'content_packages': [('eduid_idp', 'tests/static')],
            'action_plugins': ['tou', 'mfa'],
            'default_eppn_scope': 'example.edu',
        }

        cherry_conf = {
            'tools.sessions.on': True,
            'tools.sessions.storage_class': EduidSession,
            'tools.sessions.name': 'sessid',
            'tools.sessions.domain': 'unittest-idp.example.edu',
            'tools.sessions.secure': True,
            'tools.sessions.persistent': True,
            'tools.sessions.httponly': False,
        }

        self.config = IdPConfig.init_config(test_config=_test_config,
                                            debug=False)
        cherry_conf.update(self.config.to_dict())
        cherrypy.config.update(cherry_conf)
        cherrypy.config.logger = logger

        if hasattr(cherrypy.request, '_session_init_flag'):
            del cherrypy.request._session_init_flag
        init(storage_class=EduidSession,
             path='/',
             name='sessid',
             domain="unittest-idp.example.edu")
        # Create the IdP app
        self.idp_app = IdPApplication(logger, self.config)

        self.actions = self.idp_app.context.actions_db

        # setup some test data
        _email = '*****@*****.**'
        self.test_user = self.amdb.get_user_by_mail(_email)
        self.test_action = self.actions.add_action(self.test_user.eppn,
                                                   action_type='dummy',
                                                   preference=100,
                                                   params={})

        # prevent the HTTP server from ever starting
        cherrypy.server.unsubscribe()
        # mount the IdP app in the cherrypy app server
        cherrypy.tree.mount(self.idp_app, '', {'/': cherry_conf})

        # create a webtest testing environment
        from six.moves.http_cookiejar import CookieJar

        self.http = webtest.TestApp(cherrypy.tree,
                                    extra_environ={'wsgi.url_scheme': 'https'},
                                    cookiejar=CookieJar())
Ejemplo n.º 37
0
 def from_cookie_list(cls, clist):
     cj = CookieJar()
     for cookie in clist:
         cj.set_cookie(cookie)
     return cls(cj)
Ejemplo n.º 38
0
 def from_cookie_list(cls, clist):
     cj = CookieJar()
     for cookie in clist:
         cj.set_cookie(cookie)
     return cls(cj)
Ejemplo n.º 39
0
 def clear(self):
     self.cookiejar = CookieJar()
Ejemplo n.º 40
0
 def extract_cookiejar(self, resp, req):
     jar = CookieJar()
     jar.extract_cookies(MockResponse(resp._original_response.msg),
                         MockRequest(req))
     return jar
Ejemplo n.º 41
0
Archivo: urllib3.py Proyecto: ad-m/grab
 def extract_cookiejar(self, resp, req):
     jar = CookieJar()
     jar.extract_cookies(MockResponse(resp._original_response.msg),
                         MockRequest(req))
     return jar
Ejemplo n.º 42
0
class CookieTransport(xmlrpclib.Transport):
    '''A subclass of xmlrpclib.Transport that supports cookies.'''
    cookiejar = None
    scheme = 'http'

    # Cribbed from xmlrpclib.Transport.send_user_agent
    def send_cookies(self, connection, cookie_request):
        if self.cookiejar is None:
            self.cookiejar = CookieJar()
        elif self.cookiejar:
            # Let the cookiejar figure out what cookies are appropriate
            self.cookiejar.add_cookie_header(cookie_request)
            # Pull the cookie headers out of the request object...
            cookielist=list()
            for h,v in cookie_request.header_items():
                if h.startswith('Cookie'):
                    cookielist.append([h,v])
            # ...and put them over the connection
            for h,v in cookielist:
                connection.putheader(h,v)

    # This is just python 2.7's xmlrpclib.Transport.single_request, with
    # send additions noted below to send cookies along with the request
    def single_request_with_cookies(self, host, handler, request_body, verbose=0):
        # ADDED: construct the URL and Request object for proper cookie handling
        request_url = "%s://%s%s" % (self.scheme,host,handler)
        #log.debug("request_url is %s" % request_url)
        cookie_request  = urllib2.Request(request_url)

        try:
            if six.PY2:
                h = self.make_connection(host)
                if verbose:
                    h.set_debuglevel(1)
                self.send_request(h, handler, request_body)
                self.send_host(h, host)
                self.send_cookies(h, cookie_request)  # ADDED. creates cookiejar if None.
                self.send_user_agent(h)
                self.send_content(h, request_body)
            else:
                # Python 3 xmlrpc.client.Transport makes its own connection
                h = self.send_request(host, handler, request_body, verbose)

            response = h.getresponse()

            # ADDED: parse headers and get cookies here
            cookie_response = CookieResponse(response.msg)
            # Okay, extract the cookies from the headers
            self.cookiejar.extract_cookies(cookie_response,cookie_request)
            #log.debug("cookiejar now contains: %s" % self.cookiejar._cookies)
            # And write back any changes
            if hasattr(self.cookiejar,'save'):
                try:
                    self.cookiejar.save(self.cookiejar.filename)
                except Exception as e:
                    raise
                    #log.error("Couldn't write cookiefile %s: %s" % \
                    #        (self.cookiejar.filename,str(e)))

            if response.status == 200:
                self.verbose = verbose
                return self.parse_response(response)

            if (response.getheader("content-length", 0)):
                response.read()
            raise xmlrpclib.ProtocolError(
                host + handler,
                response.status, response.reason,
                response.msg,
                )
        except xmlrpclib.Fault:
            raise
        finally:
            try:
                h.close()
            except NameError:  # h not initialized yet
                pass

    # Override the appropriate request method
    single_request = single_request_with_cookies # python 2.7+