def test_get_random_ua(self):
EXPECTED = ('Mozilla', 'Windows', 'MSIE', 'Opera')
for _ in xrange(100):
rnd_ua = get_random_user_agent()
for estr in EXPECTED:
if estr in rnd_ua:
return
self.assertTrue(False, 'Failed to find %s' % (EXPECTED, ))
def test_get_random_ua(self):
EXPECTED = ('Mozilla', 'Windows', 'MSIE', 'Opera')
for _ in xrange(100):
rnd_ua = get_random_user_agent()
for estr in EXPECTED:
if estr in rnd_ua:
return
self.assertTrue(False, 'Failed to find %s' % (EXPECTED,))
def _add_headers(self, req, headers=Headers()):
"""
Add all custom Headers() if they exist
"""
for h, v in self.settings.header_list:
req.add_header(h, v)
for h, v in headers.iteritems():
req.add_header(h, v)
if self.settings.rand_user_agent is True:
req.add_header('User-Agent', get_random_user_agent())
return req
def _do_GET(self, url, with_rand_ua=True):
if not isinstance(url, URL):
msg = 'The url parameter of a _do_GET must be of url.URL type.'
raise ValueError(msg)
if with_rand_ua:
random_ua = get_random_user_agent()
headers = Headers([('User-Agent', random_ua)])
else:
# Please note that some tests show that this is useful for the
# mobile search.
headers = Headers([('User-Agent', '')])
return self._uri_opener.GET(url, headers=headers)
def _do_GET(self, url, with_rand_ua=True):
if not isinstance(url, URL):
msg = 'The url parameter of a _do_GET must be of url.URL type.'
raise ValueError(msg)
if with_rand_ua:
random_ua = get_random_user_agent()
headers = Headers([('User-Agent', random_ua)])
else:
# Please note that some tests show that this is useful for the
# mobile search.
headers = Headers([('User-Agent', '')])
return self._uri_opener.GET(url, headers=headers)
def add_headers(self, req, headers=Headers()):
"""
Add all custom Headers() if they exist
"""
for h, v in self.settings.header_list:
req.add_header(h, v)
for h, v in headers.iteritems():
req.add_header(h, v)
if self.settings.rand_user_agent is True:
req.add_header('User-Agent', get_random_user_agent())
return req
