示例#1
0
    def test_setdefault(self):
        dsn = 'scheme://*****:*****@host/foo'
        r = dsnparse3.parse(dsn)
        self.assertEqual(None, r.port)

        r.setdefault('port', 1234)
        self.assertEqual(1234, r.port)

        r = dsnparse3.parse(dsn, port=1235)
        self.assertEqual(1235, r.port)
示例#2
0
 def test___getitem__(self):
     dsn = 'scheme://*****:*****@host:1234/foo'
     r = dsnparse3.parse(dsn)
     self.assertEqual('scheme', r[0])
     self.assertEqual('username:password@host:1234', r[1])
     self.assertEqual('/foo', r[2])
     self.assertEqual('', r[3])
     self.assertEqual({}, r[4])
     self.assertEqual('', r[5])
示例#3
0
 def test_unpack(self):
     dsn = 'scheme://*****:*****@host:1234/foo'
     dsn_test = {
         'scheme': 'scheme',
         'netloc': 'username:password@host:1234',
         'path': '/foo',
         'params': "",
         'query': {},
         'fragment': ''
     }
     scheme, netloc, path, params, query, fragment = dsnparse3.parse(dsn)
     self.assertEqual('scheme', scheme)
     self.assertEqual('username:password@host:1234', netloc)
     self.assertEqual('/foo', path)
     self.assertEqual('', params)
     self.assertEqual({}, query)
     self.assertEqual('', fragment)
示例#4
0
    def _get_agent(self, request, timeout):
        bindAddress = request.meta.get('bindaddress') or self._bindAddress
        proxy = request.meta.get('proxy', '').lower()
        timeout = request.meta.pop('proxy_timeout', timeout)

        if proxy:
            parsed_proxy = dsnparse3.parse(proxy)
            if parsed_proxy.scheme in ('http', 'https'):
                return super(ScrapyAgent, self)._get_agent(request, timeout)
            if parsed_proxy.scheme not in ('socks4', 'socks4a', 'socks5',
                                           'socks5h'):
                raise SchemeNotSupported('unsupported scheme',
                                         parsed_proxy.scheme)

            return ProxyAgent(reactor=reactor,
                              proxy_scheme=parsed_proxy.scheme,
                              proxy_host=parsed_proxy.host,
                              proxy_port=parsed_proxy.port,
                              proxy_username=parsed_proxy.username,
                              proxy_password=parsed_proxy.password,
                              contextFactory=self._contextFactory,
                              bindAddress=bindAddress)

        return super(ScrapyAgent, self)._get_agent(request, timeout)
示例#5
0
    def test_parse(self):
        tests = [
            (
                'scheme://:password@host:1234/bar/che?option1=opt_val1&option2=opt_val2#anchor',
                {
                    'scheme': 'scheme',
                    'schemes': ['scheme'],
                    'username': '',
                    'password': '******',
                    'netloc': ':password@host:1234',
                    'host': 'host',
                    'hostloc': 'host:1234',
                    'path': '/bar/che',
                    'paths': ['bar', 'che'],
                    'hostname': 'host',
                    'query': {'option1': 'opt_val1', 'option2': 'opt_val2'},
                    'fragment': 'anchor'
                }
            ),
            (
                'scheme://username@host:1234/bar/che?option1=opt_val1&option2=opt_val2#anchor',
                {
                    'scheme': 'scheme',
                    'schemes': ['scheme'],
                    'username': '******',
                    'password': None,
                    'netloc': 'username@host:1234',
                    'host': 'host',
                    'hostloc': 'host:1234',
                    'path': '/bar/che',
                    'paths': ['bar', 'che'],
                    'hostname': 'host',
                    'query': {'option1': 'opt_val1', 'option2': 'opt_val2'},
                    'fragment': 'anchor'
                }
            ),
            (
                'scheme://*****:*****@host:1234/bar/che?option1=opt_val1&option2=opt_val2#anchor',
                {
                    'scheme': 'scheme',
                    'schemes': ['scheme'],
                    'username': '******',
                    'password': '******',
                    'netloc': 'username:password@host:1234',
                    'host': 'host',
                    'hostloc': 'host:1234',
                    'path': '/bar/che',
                    'paths': ['bar', 'che'],
                    'hostname': 'host',
                    'query': {'option1': 'opt_val1', 'option2': 'opt_val2'},
                    'fragment': 'anchor'
                }
            ),
            (
                'scheme://localhost',
                {
                    'scheme': 'scheme',
                    'schemes': ['scheme'],
                    'netloc': 'localhost',
                    'host': 'localhost',
                    'hostloc': 'localhost',
                    'path': '',
                    'paths': [],
                    'hostname': 'localhost',
                    'query': {}
                }
            ),
            (
                'scheme1+scheme2://username:[email protected]:9000/?opt=opt_val1&opt=opt_val2#anchor',
                {
                    'scheme': 'scheme1+scheme2',
                    'schemes': ['scheme1', 'scheme2'],
                    'username': '******',
                    'password': '******',
                    'netloc': 'username:[email protected]:9000',
                    'host': 'host.com',
                    'hostloc': 'host.com:9000',
                    'path': '/',
                    'paths': [],
                    'hostname': 'host.com',
                    'query': {'opt': ['opt_val1', 'opt_val2']},
                    'fragment': 'anchor'
                }
            ),
        ]

        for dsn, test_out in tests:
            r = dsnparse3.parse(dsn)
            for k, v in test_out.items():
                self.assertEqual(v, getattr(r, k))

        with self.assertRaises(AssertionError):
            r = dsnparse3.parse('//host.com:1234')
示例#6
0
 def test_geturl(self):
     dsn = 'scheme://*****:*****@host:1234/bar/che?option1=opt_val1&option2=opt_val2#anchor'
     r = dsnparse3.parse(dsn)
     self.assertEqual(dsn, r.geturl())