def test_get_random_ua(self): EXPECTED = ('Mozilla', 'Windows', 'MSIE', 'Opera') for _ in xrange(100): rnd_ua = get_random_user_agent() for estr in EXPECTED: if estr in rnd_ua: return self.assertTrue(False, 'Failed to find %s' % (EXPECTED, ))
def test_get_random_ua(self): EXPECTED = ('Mozilla', 'Windows', 'MSIE', 'Opera') for _ in xrange(100): rnd_ua = get_random_user_agent() for estr in EXPECTED: if estr in rnd_ua: return self.assertTrue(False, 'Failed to find %s' % (EXPECTED,))
def _add_headers(self, req, headers=Headers()): """ Add all custom Headers() if they exist """ for h, v in self.settings.header_list: req.add_header(h, v) for h, v in headers.iteritems(): req.add_header(h, v) if self.settings.rand_user_agent is True: req.add_header('User-Agent', get_random_user_agent()) return req
def _do_GET(self, url, with_rand_ua=True): if not isinstance(url, URL): msg = 'The url parameter of a _do_GET must be of url.URL type.' raise ValueError(msg) if with_rand_ua: random_ua = get_random_user_agent() headers = Headers([('User-Agent', random_ua)]) else: # Please note that some tests show that this is useful for the # mobile search. headers = Headers([('User-Agent', '')]) return self._uri_opener.GET(url, headers=headers)
def add_headers(self, req, headers=Headers()): """ Add all custom Headers() if they exist """ for h, v in self.settings.header_list: req.add_header(h, v) for h, v in headers.iteritems(): req.add_header(h, v) if self.settings.rand_user_agent is True: req.add_header('User-Agent', get_random_user_agent()) return req