def test_session(self):
# Test that if Grab gets some cookies from the server
# then it sends it back
g = build_grab()
g.setup(reuse_cookies=True)
self.server.response["cookies"] = {"foo": "bar"}.items()
g.go(self.server.get_url())
self.assertEqual(g.response.cookies["foo"], "bar")
g.go(self.server.get_url())
self.assertEqual(self.server.request["headers"]["Cookie"], "foo=bar")
g.go(self.server.get_url())
self.assertEqual(self.server.request["headers"]["Cookie"], "foo=bar")
# Test reuse_cookies=False
g = build_grab()
g.setup(reuse_cookies=False)
self.server.response["cookies"] = {"foo": "baz"}.items()
g.go(self.server.get_url())
self.assertEqual(g.response.cookies["foo"], "baz")
g.go(self.server.get_url())
self.assertTrue(len(self.server.request["cookies"]) == 0)
# Test something
g = build_grab()
g.setup(reuse_cookies=True)
self.server.response["cookies"] = {"foo": "bar"}.items()
g.go(self.server.get_url())
self.assertEqual(g.response.cookies["foo"], "bar")
g.clear_cookies()
g.go(self.server.get_url())
self.assertTrue(len(self.server.request["cookies"]) == 0)
def test_load_proxylist(self):
with temp_file() as tmp_file:
content = "%s\n%s\n%s" % (PROXY1, PROXY2, PROXY3)
open(tmp_file, "w").write(content)
# By default auto_change is True
g = build_grab()
g.load_proxylist(tmp_file, "text_file")
self.assertEqual(g.config["proxy_auto_change"], True)
servers = set()
for x in six.moves.range(10):
g.go("http://yandex.ru")
servers.add(g.config["proxy"])
self.assertTrue(len(servers) > 1)
# Disable auto_change
# By default auto_init is True
g = build_grab()
g.load_proxylist(tmp_file, "text_file", auto_change=False)
self.assertEqual(g.config["proxy_auto_change"], False)
servers = set()
for x in six.moves.range(10):
g.go("http://yandex.ru")
servers.add(g.config["proxy"])
self.assertEqual(len(servers), 1)
# Disable auto_change
# Disable auto_init
# Proxylist will not be used by default
g = build_grab()
g.load_proxylist(tmp_file, "text_file", auto_change=False, auto_init=False)
self.assertEqual(g.config["proxy_auto_change"], False)
g.go(self.server.get_url())
self.assertEqual(g.config["proxy"], None)
def test_body_inmemory(self):
g = build_grab()
g.setup(body_inmemory=False)
self.assertRaises(GrabMisuseError, lambda: g.go(SERVER.BASE_URL))
SERVER.RESPONSE['get'] = 'foo'
g = build_grab()
g.setup(body_inmemory=False)
g.setup(body_storage_dir=TMP_DIR)
g.go(SERVER.BASE_URL)
self.assertTrue(os.path.exists(g.response.body_path))
self.assertTrue(TMP_DIR in g.response.body_path)
self.assertEqual('foo', open(g.response.body_path).read())
old_path = g.response.body_path
g.go(SERVER.BASE_URL)
self.assertTrue(old_path != g.response.body_path)
SERVER.RESPONSE['get'] = 'foo'
g = build_grab()
g.setup(body_inmemory=False)
g.setup(body_storage_dir=TMP_DIR)
g.setup(body_storage_filename='musik.mp3')
g.go(SERVER.BASE_URL)
self.assertTrue(os.path.exists(g.response.body_path))
self.assertTrue(TMP_DIR in g.response.body_path)
self.assertEqual('foo', open(g.response.body_path).read())
self.assertEqual(os.path.join(TMP_DIR, 'musik.mp3'), g.response.body_path)
def test_session(self):
# Test that if Grab gets some cookies from the server
# then it sends it back
g = build_grab()
g.setup(reuse_cookies=True)
self.server.response['cookies'] = {'foo': 'bar'}.items()
g.go(self.server.get_url())
self.assertEqual(g.response.cookies['foo'], 'bar')
g.go(self.server.get_url())
self.assertEqual(self.server.request['headers']['Cookie'], 'foo=bar')
g.go(self.server.get_url())
self.assertEqual(self.server.request['headers']['Cookie'], 'foo=bar')
# Test reuse_cookies=False
g = build_grab()
g.setup(reuse_cookies=False)
self.server.response['cookies'] = {'foo': 'baz'}.items()
g.go(self.server.get_url())
self.assertEqual(g.response.cookies['foo'], 'baz')
g.go(self.server.get_url())
self.assertTrue(len(self.server.request['cookies']) == 0)
# Test something
g = build_grab()
g.setup(reuse_cookies=True)
self.server.response['cookies'] = {'foo': 'bar'}.items()
g.go(self.server.get_url())
self.assertEqual(g.response.cookies['foo'], 'bar')
g.clear_cookies()
g.go(self.server.get_url())
self.assertTrue(len(self.server.request['cookies']) == 0)
def test_unicode_post(self):
# By default, unicode post should be converted into utf-8
g = build_grab()
data = u'фыва'
g.setup(post=data, url=self.server.get_url())
g.request()
self.assertEqual(self.server.request['data'], data.encode('utf-8'))
# Now try cp1251 with charset option
self.server.request['charset'] = 'cp1251'
g = build_grab()
data = u'фыва'
g.setup(post=data, url=self.server.get_url(),
charset='cp1251', debug=True)
g.request()
self.assertEqual(self.server.request['data'], data.encode('cp1251'))
# Now try dict with unicode value & charset option
self.server.request['charset'] = 'cp1251'
g = build_grab()
data = u'фыва'
g.setup(post={'foo': data}, url=self.server.get_url(),
charset='cp1251', debug=True)
g.request()
test = 'foo=%s' % quote(data.encode('cp1251'))
test = test.encode('utf-8') # py3 hack
self.assertEqual(self.server.request['data'], test)
def test_load_proxylist(self):
content = "%s\n%s\n%s" % (PROXY1, PROXY2, PROXY3)
open(TMP_FILE, "w").write(content)
# By default auto_change is True
g = build_grab()
g.load_proxylist(TMP_FILE, "text_file")
self.assertEqual(g.config["proxy_auto_change"], True)
servers = set()
for x in xrange(10):
g.go("http://yandex.ru")
servers.add(g.config["proxy"])
self.assertTrue(len(servers) > 1)
# Disable auto_change
# By default auto_init is True
g = build_grab()
g.load_proxylist(TMP_FILE, "text_file", auto_change=False)
self.assertEqual(g.config["proxy_auto_change"], False)
servers = set()
for x in xrange(10):
g.go("http://yandex.ru")
servers.add(g.config["proxy"])
self.assertEqual(len(servers), 1)
# Disable auto_change
# Disable auto_init
# Proxylist will not be used by default
g = build_grab()
g.load_proxylist(TMP_FILE, "text_file", auto_change=False, auto_init=False)
self.assertEqual(g.config["proxy_auto_change"], False)
g.go(SERVER.BASE_URL)
self.assertEqual(g.config["proxy"], None)
def test_load_dump(self):
with temp_file() as tmp_file:
g = build_grab()
cookies = {"foo": "bar", "spam": "ham"}
g.setup(cookies=cookies)
g.go(self.server.get_url())
g.dump_cookies(tmp_file)
self.assertEqual(set(cookies.items()), set((x["name"], x["value"]) for x in json.load(open(tmp_file))))
g = build_grab()
cookies = {"foo": "bar", "spam": u"begemot"}
g.setup(cookies=cookies)
g.go(self.server.get_url())
g.dump_cookies(tmp_file)
self.assertEqual(set(cookies.items()), set((x["name"], x["value"]) for x in json.load(open(tmp_file))))
# Test load cookies
g = build_grab()
cookies = [
{"name": "foo", "value": "bar", "domain": self.server.address},
{"name": "spam", "value": u"begemot", "domain": self.server.address},
]
json.dump(cookies, open(tmp_file, "w"))
g.load_cookies(tmp_file)
self.assertEqual(set(g.cookies.items()), set((x["name"], x["value"]) for x in cookies))
def test_body_inmemory_false(self):
g = build_grab()
g.setup(body_inmemory=False)
self.assertRaises(GrabMisuseError, lambda: g.go(self.server.get_url()))
self.server.response['get.data'] = b'foo'
g = build_grab()
g.setup(body_inmemory=False)
g.setup(body_storage_dir=TMP_DIR)
g.go(self.server.get_url())
self.assertTrue(os.path.exists(g.response.body_path))
self.assertTrue(TMP_DIR in g.response.body_path)
self.assertEqual(b'foo', open(g.response.body_path, 'rb').read())
self.assertEqual(g.response._bytes_body, None)
old_path = g.response.body_path
g.go(self.server.get_url())
self.assertTrue(old_path != g.response.body_path)
self.server.response['get.data'] = 'foo'
g = build_grab()
g.setup(body_inmemory=False)
g.setup(body_storage_dir=TMP_DIR)
g.setup(body_storage_filename='music.mp3')
g.go(self.server.get_url())
self.assertTrue(os.path.exists(g.response.body_path))
self.assertTrue(TMP_DIR in g.response.body_path)
self.assertEqual(b'foo', open(g.response.body_path, 'rb').read())
self.assertEqual(os.path.join(TMP_DIR, 'music.mp3'),
g.response.body_path)
self.assertEqual(g.response.body, b'foo')
self.assertEqual(g.response._bytes_body, None)
def test_session(self):
# Test that if Grab gets some cookies from the server
# then it sends it back
g = build_grab()
g.setup(reuse_cookies=True)
SERVER.RESPONSE['cookies'] = {'foo': 'bar'}
g.go(SERVER.BASE_URL)
self.assertEqual(g.response.cookies['foo'], 'bar')
g.go(SERVER.BASE_URL)
self.assertEqual(SERVER.REQUEST['headers']['Cookie'], 'foo=bar')
g.go(SERVER.BASE_URL)
self.assertEqual(SERVER.REQUEST['headers']['Cookie'], 'foo=bar')
# Test reuse_cookies=False
g = build_grab()
g.setup(reuse_cookies=False)
SERVER.RESPONSE['cookies'] = {'foo': 'baz'}
g.go(SERVER.BASE_URL)
self.assertEqual(g.response.cookies['foo'], 'baz')
g.go(SERVER.BASE_URL)
self.assertTrue(len(SERVER.REQUEST['cookies']) == 0)
# Test something
g = build_grab()
g.setup(reuse_cookies=True)
SERVER.RESPONSE['cookies'] = {'foo': 'bar'}
g.go(SERVER.BASE_URL)
self.assertEqual(g.response.cookies['foo'], 'bar')
g.clear_cookies()
g.go(SERVER.BASE_URL)
self.assertTrue(len(SERVER.REQUEST['cookies']) == 0)
def test_session(self):
# Test that if Grab gets some cookies from the server
# then it sends it back
g = build_grab()
g.setup(reuse_cookies=True)
SERVER.RESPONSE["cookies"] = {"foo": "bar"}
g.go(SERVER.BASE_URL)
self.assertEqual(g.response.cookies["foo"], "bar")
g.go(SERVER.BASE_URL)
self.assertEqual(SERVER.REQUEST["headers"]["Cookie"], "foo=bar")
g.go(SERVER.BASE_URL)
self.assertEqual(SERVER.REQUEST["headers"]["Cookie"], "foo=bar")
# Test reuse_cookies=False
g = build_grab()
g.setup(reuse_cookies=False)
SERVER.RESPONSE["cookies"] = {"foo": "baz"}
g.go(SERVER.BASE_URL)
self.assertEqual(g.response.cookies["foo"], "baz")
g.go(SERVER.BASE_URL)
self.assertTrue(len(SERVER.REQUEST["cookies"]) == 0)
# Test something
g = build_grab()
g.setup(reuse_cookies=True)
SERVER.RESPONSE["cookies"] = {"foo": "bar"}
g.go(SERVER.BASE_URL)
self.assertEqual(g.response.cookies["foo"], "bar")
g.clear_cookies()
g.go(SERVER.BASE_URL)
self.assertTrue(len(SERVER.REQUEST["cookies"]) == 0)
def test_load_proxylist(self):
content = '%s\n%s\n%s' % (PROXY1, PROXY2, PROXY3)
open(TMP_FILE, 'w').write(content)
# By default auto_change is True
g = build_grab()
g.load_proxylist(TMP_FILE, 'text_file')
self.assertEqual(g.config['proxy_auto_change'], True)
servers = set()
for x in six.moves.range(10):
g.go('http://yandex.ru')
servers.add(g.config['proxy'])
self.assertTrue(len(servers) > 1)
# Disable auto_change
# By default auto_init is True
g = build_grab()
g.load_proxylist(TMP_FILE, 'text_file', auto_change=False)
self.assertEqual(g.config['proxy_auto_change'], False)
servers = set()
for x in six.moves.range(10):
g.go('http://yandex.ru')
servers.add(g.config['proxy'])
self.assertEqual(len(servers), 1)
# Disable auto_change
# Disable auto_init
# Proxylist will not be used by default
g = build_grab()
g.load_proxylist(TMP_FILE, 'text_file', auto_change=False,
auto_init=False)
self.assertEqual(g.config['proxy_auto_change'], False)
g.go(self.server.get_url())
self.assertEqual(g.config['proxy'], None)
def test_load_dump(self):
g = build_grab()
cookies = {'foo': 'bar', 'spam': 'ham'}
g.setup(cookies=cookies)
g.go(self.server.get_url())
g.dump_cookies(TMP_FILE)
self.assertEqual(set(cookies.items()),
set((x['name'], x['value'])
for x in json.load(open(TMP_FILE))))
# Test non-ascii
g = build_grab()
cookies = {'foo': 'bar', 'spam': u'бегемот'}
g.setup(cookies=cookies)
g.go(self.server.get_url())
g.dump_cookies(TMP_FILE)
self.assertEqual(set(cookies.items()),
set((x['name'], x['value'])
for x in json.load(open(TMP_FILE))))
# Test load cookies
g = build_grab()
cookies = [{'name': 'foo', 'value': 'bar'},
{'name': 'spam', 'value': u'бегемот'}]
json.dump(cookies, open(TMP_FILE, 'w'))
g.load_cookies(TMP_FILE)
self.assertEqual(set(g.cookies.items()),
set((x['name'], x['value']) for x in cookies))
def test_adopt(self):
g = build_grab()
SERVER.RESPONSE['get'] = 'Moon'
g.go(SERVER.BASE_URL)
g2 = build_grab()
self.assertEqual(g2.config['url'], None)
g2.adopt(g)
self.assertTrue('Moon' in g2.response.body)
self.assertEqual(g2.config['url'], SERVER.BASE_URL)
def test_make_url_absolute(self):
g = build_grab()
self.server.response['get.data'] = '<base href="http://foo/bar/">'
g.go(self.server.get_url())
absolute_url = g.make_url_absolute('/foobar', resolve_base=True)
self.assertEqual(absolute_url, 'http://foo/foobar')
g = build_grab()
absolute_url = g.make_url_absolute('/foobar')
self.assertEqual(absolute_url, '/foobar')
def test_adopt(self):
g = build_grab()
self.server.response['get.data'] = 'Moon'
g.go(self.server.get_url())
g2 = build_grab()
self.assertEqual(g2.config['url'], None)
g2.adopt(g)
self.assertTrue(b'Moon' in g2.response.body)
self.assertEqual(g2.config['url'], self.server.get_url())
def test_clone(self):
g = build_grab()
SERVER.RESPONSE['get'] = 'Moon'
g.go(SERVER.BASE_URL)
self.assertTrue('Moon' in g.response.body)
g2 = build_grab()
self.assertEqual(g2.response, None)
g2 = g.clone()
self.assertTrue('Moon' in g.response.body)
def test_adopt(self):
g = build_grab()
self.server.response["get.data"] = "Moon"
g.go(self.server.get_url())
g2 = build_grab()
self.assertEqual(g2.config["url"], None)
g2.adopt(g)
self.assertTrue(b"Moon" in g2.response.body)
self.assertEqual(g2.config["url"], self.server.get_url())
def test_body_inmemory(self):
g = build_grab()
g.setup(body_inmemory=False)
self.assertRaises(GrabMisuseError, lambda: g.go(SERVER.BASE_URL))
SERVER.RESPONSE['get'] = 'foo'
g = build_grab()
g.setup(body_inmemory=False)
g.setup(body_storage_dir=TMP_DIR)
g.go(SERVER.BASE_URL)
def test_empty_document(self):
self.server.response['get.data'] = 'oops'
g = build_grab()
g.go(self.server.get_url())
g.xpath_exists('//anytag')
self.server.response['get.data'] = '<frameset></frameset>'
g = build_grab()
g.go(self.server.get_url())
g.xpath_exists('//anytag')
def test_body_inmemory(self):
g = build_grab()
g.setup(body_inmemory=False)
self.assertRaises(GrabMisuseError, lambda: g.go(self.server.get_url()))
self.server.response["get.data"] = "foo"
g = build_grab()
g.setup(body_inmemory=False)
g.setup(body_storage_dir=TMP_DIR)
g.go(self.server.get_url())
def test_empty_document(self):
SERVER.RESPONSE['get'] = 'oops'
g = build_grab()
g.go(SERVER.BASE_URL)
g.xpath_exists('//anytag')
SERVER.RESPONSE['get'] = '<frameset></frameset>'
g = build_grab()
g.go(SERVER.BASE_URL)
g.xpath_exists('//anytag')
def test_options_method(self):
g = build_grab()
g.setup(method="options", post=b"abc")
g.go(self.server.get_url())
self.assertEquals("OPTIONS", self.server.request["method"])
self.assertEquals("3", self.server.request["headers"]["Content-Length"])
g = build_grab()
g.setup(method="options")
g.go(self.server.get_url())
self.assertEquals("OPTIONS", self.server.request["method"])
self.assertTrue("Content-Length" not in self.server.request["headers"])
def test_options_method(self):
g = build_grab()
g.setup(method='options', post=b'abc')
g.go(self.server.get_url())
self.assertEquals('OPTIONS', self.server.request['method'])
self.assertEquals('3', self.server.request['headers']['Content-Length'])
g = build_grab()
g.setup(method='options')
g.go(self.server.get_url())
self.assertEquals('OPTIONS', self.server.request['method'])
self.assertTrue('Content-Length' not in self.server.request['headers'])
def test_redirect_session(self):
g = build_grab()
self.server.response["cookies"] = {"foo": "bar"}.items()
g.go(self.server.get_url())
self.assertEqual(g.response.cookies["foo"], "bar")
# Setup one-time redirect
g = build_grab()
self.server.response["cookies"] = {}
self.server.response_once["headers"] = [("Location", self.server.get_url()), ("Set-Cookie", "foo=bar")]
self.server.response_once["code"] = 302
g.go(self.server.get_url())
self.assertEqual(self.server.request["cookies"]["foo"].value, "bar")
def test_redirect_session(self):
g = build_grab()
SERVER.RESPONSE["cookies"] = {"foo": "bar"}
g.go(SERVER.BASE_URL)
self.assertEqual(g.response.cookies["foo"], "bar")
# Setup one-time redirect
g = build_grab()
SERVER.RESPONSE["cookies"] = {}
SERVER.RESPONSE_ONCE["headers"].append(("Location", SERVER.BASE_URL))
SERVER.RESPONSE_ONCE["headers"].append(("Set-Cookie", "foo=bar"))
SERVER.RESPONSE_ONCE["code"] = 302
g.go(SERVER.BASE_URL)
self.assertEqual(SERVER.REQUEST["cookies"]["foo"].value, "bar")
def test_redirect_session(self):
g = build_grab()
SERVER.RESPONSE['cookies'] = {'foo': 'bar'}
g.go(SERVER.BASE_URL)
self.assertEqual(g.response.cookies['foo'], 'bar')
# Setup one-time redirect
g = build_grab()
SERVER.RESPONSE['cookies'] = {}
SERVER.RESPONSE_ONCE['headers'].append(('Location', SERVER.BASE_URL))
SERVER.RESPONSE_ONCE['headers'].append(('Set-Cookie', 'foo=bar'))
SERVER.RESPONSE_ONCE['code'] = 302
g.go(SERVER.BASE_URL)
self.assertEqual(SERVER.REQUEST['cookies']['foo'].value, 'bar')
def test_useragent(self):
g = build_grab()
# Null value activates default random user-agent
# For some transports it just allow them to send default user-agent
# like in Kit transport case
g = build_grab()
g.setup(user_agent=None)
g.go(self.server.get_url())
self.assertTrue(len(self.server.request['headers']) > 0)
self.assertFalse('PycURL' in
self.server.request['headers']['user-agent'])
# By default user_agent is None => random user agent is generated
g = build_grab()
g.go(self.server.get_url())
self.assertTrue(len(self.server.request['headers']) > 0)
self.assertFalse('PycURL' in
self.server.request['headers']['user-agent'])
# Simple case: setup user agent manually
g.setup(user_agent='foo')
g.go(self.server.get_url())
self.assertEqual(self.server.request['headers']['user-agent'], 'foo')
# user agent from file should be loaded
path = '/tmp/__ua.txt'
open(path, 'w').write('GOD')
g.setup(user_agent=None, user_agent_file=path)
g.go(self.server.get_url())
self.assertEqual(self.server.request['headers']['user-agent'], 'GOD')
# random user agent from file should be loaded
path = '/tmp/__ua.txt'
open(path, 'w').write('GOD1\nGOD2')
g.setup(user_agent=None, user_agent_file=path)
g.go(self.server.get_url())
self.assertTrue(self.server.request['headers']['user-agent']
in ('GOD1', 'GOD2'))
ua = g.config['user_agent']
# User-agent should not change
g.go(self.server.get_url())
self.assertEqual(self.server.request['headers']['user-agent'], ua)
# User-agent should not change
g.go(self.server.get_url())
self.assertEqual(self.server.request['headers']['user-agent'], ua)
def test_useragent_simple(self):
g = build_grab()
# Simple case: setup user agent manually
g.setup(user_agent='foo')
g.go(SERVER.BASE_URL)
self.assertEqual(SERVER.REQUEST['headers']['user-agent'], 'foo')
def test_cookiefile(self):
g = build_grab()
# Empty file should not raise Exception
open(TMP_FILE, "w").write("")
g.setup(cookiefile=TMP_FILE)
g.go(SERVER.BASE_URL)
cookies = [{"name": "spam", "value": "ham"}]
json.dump(cookies, open(TMP_FILE, "w"))
# One cookie are sent in server reponse
# Another cookies is passed via the `cookiefile` option
SERVER.RESPONSE["cookies"] = {"godzilla": "monkey"}
g.setup(cookiefile=TMP_FILE)
g.go(SERVER.BASE_URL)
self.assertEqual(SERVER.REQUEST["cookies"]["spam"].value, "ham")
# This is correct reslt of combining two cookies
MERGED_COOKIES = [("godzilla", "monkey"), ("spam", "ham")]
# g.cookies should contains merged cookies
self.assertEqual(set(MERGED_COOKIES), set(g.cookies.items()))
# `cookiefile` file should contains merged cookies
self.assertEqual(set(MERGED_COOKIES), set((x["name"], x["value"]) for x in json.load(open(TMP_FILE))))
# Just ensure it works
g.go(SERVER.BASE_URL)
def test_empty_useragent_pycurl(self):
g = build_grab()
# Empty string disable default pycurl user-agent
g.setup(user_agent='')
g.go(SERVER.BASE_URL)
self.assertEqual(SERVER.REQUEST['headers'].get('user-agent', ''), '')
def test_debug_nonascii_post(self):
grab = build_grab(debug=True)
grab.setup(post=u'фыва'.encode('cp1251'))
grab.go(self.server.get_url())
def test_get_method(self):
grab = build_grab()
grab.go(self.server.get_url())
self.assertEqual('GET', self.server.request['method'])
def test_incorrect_option_name(self):
g = build_grab()
self.assertRaises(GrabMisuseError, g.setup, save_the_word=True)
def test_default_content_for_fake_response(self):
content = b'<strong>test</strong>'
g = build_grab(document_body=content)
self.assertEqual(g.response.body, content)
def test_empty_clone(self):
g = build_grab()
g.clone()
def setUp(self):
SERVER.reset()
# Create fake grab instance with fake response
self.g = build_grab()
self.g.fake_response(HTML, charset='cp1251')
def test_parsing_response_cookies(self):
grab = build_grab()
self.server.response['cookies'] = {'foo': 'bar', '1': '2'}.items()
grab.go(self.server.get_url())
self.assertEqual(grab.doc.cookies['foo'], 'bar')
def test_get(self):
self.server.response['get.data'] = 'Final Countdown'
g = build_grab()
g.go(self.server.get_url())
self.assertTrue(b'Final Countdown' in g.response.body)
def setUp(self):
self.g = build_grab(document_body=XML)
def test_parsing_response_headers(self):
self.server.response['headers'] = [('Hello', 'Grab')]
grab = build_grab()
grab.go(self.server.get_url())
self.assertTrue(grab.response.headers['Hello'] == 'Grab')
def test_status_code(self):
self.server.response['get.data'] = 'Simple String'
grab = build_grab()
grab.go(self.server.get_url())
self.assertEqual(200, grab.response.code)
def test_body_content(self):
self.server.response['get.data'] = 'Simple String'
grab = build_grab()
grab.go(self.server.get_url())
self.assertEqual(b'Simple String', grab.response.body)
def test_debug_post(self):
g = build_grab(debug_post=True)
g.setup(post={'foo': 'bar'})
self.server.response['post.data'] = 'x'
g.go(self.server.get_url())
self.assertEqual(b'x', g.doc.body)
def test_task_clone_kwargs(self):
g = build_grab()
g.setup(url='http://foo.com/')
task = Task('foo', grab=g, cache_timeout=1)
task2 = task.clone(cache_timeout=2)
self.assertEqual(2, task2.cache_timeout)
def test_update_invalid_cookie(self):
grab = build_grab()
self.assertRaises(GrabMisuseError, grab.cookies.update, None)
self.assertRaises(GrabMisuseError, grab.cookies.update, 'asdf')
self.assertRaises(GrabMisuseError, grab.cookies.update, ['asdf'])
def test_body_inmemory_true(self):
g = build_grab()
self.server.response['data'] = b'bar'
g.go(self.server.get_url())
self.assertEqual(g.response._bytes_body, b'bar')
def func():
g = build_grab()
g.go(self.server.get_url())
def test_request_headers(self):
grab = build_grab(debug=True)
grab.setup(headers={'Foo': 'Bar'})
grab.go(self.server.get_url())
self.assertEqual('Bar', grab.request_headers['foo'])
def test_empty_adopt(self):
g = build_grab()
g2 = build_grab()
g2.adopt(g)
def test_default_random_user_agent(self):
grab = build_grab()
grab.go(self.server.get_url())
self.assertTrue(self.server.request['headers'].get(
'user-agent').startswith('Mozilla/5.0 '))
def test_setup_document(self):
data = b'''
<h1>test</h1>
'''
g = build_grab(data)
self.assertTrue(b'test' in g.doc.body)
def test_put_method(self):
g = build_grab()
g.setup(method='put', post=b'')
g.go(SERVER.BASE_URL)
self.assertEquals('PUT', SERVER.REQUEST['method'])
def test_download(self):
with temp_file() as save_file:
g = build_grab()
self.server.response['get.data'] = 'FOO'
length = g.download(self.server.get_url(), save_file)
self.assertEqual(3, length)
def test_put_method(self):
grab = build_grab()
grab.setup(method='put', post=b'abc')
grab.go(self.server.get_url())
self.assertEqual('PUT', self.server.request['method'])
self.assertEqual('3', self.server.request['headers']['Content-Length'])
def test_debug_post_big_str(self):
grab = build_grab(debug_post=True)
big_value = 'x' * 1000
grab.setup(post=big_value)
grab.go(self.server.get_url())
self.assertEqual(self.server.request['data'], big_value.encode())
def test_setup_with_proxyline_custom_proxy_type(self):
g = build_grab()
g.setup_with_proxyline('1.1.1.1:8080', proxy_type='socks')
self.assertEqual(g.config['proxy'], '1.1.1.1:8080')
self.assertEqual(g.config['proxy_userpwd'], None)
self.assertEqual(g.config['proxy_type'], 'socks')
def test_debug_post_integer_bug(self):
grab = build_grab(debug_post=True)
grab.setup(post={'foo': 3})
self.server.response['post.data'] = 'x'
grab.go(self.server.get_url())
self.assertEqual(b'x', grab.doc.body)
def test_debug_nonascii_multipart_post(self):
grab = build_grab(debug=True)
grab.setup(charset='cp1251',
multipart_post=[('x', u'фыва'.encode('cp1251'))])
grab.go(self.server.get_url())
def test_setup_with_proxyline_userpwd(self):
g = build_grab()
g.setup_with_proxyline('1.1.1.1:8080:user:pass')
self.assertEqual(g.config['proxy'], '1.1.1.1:8080')
self.assertEqual(g.config['proxy_userpwd'], 'user:pass')
self.assertEqual(g.config['proxy_type'], 'http')
def test_delete_method(self):
grab = build_grab()
grab.setup(method='delete')
grab.go(self.server.get_url())
self.assertEqual('DELETE', self.server.request['method'])