def test_accumulate_updates_basic(self):
# test that all work with disabled accumulate_updates feature
pl = ProxyList()
path = self.generate_plist_file()
pl.setup(accumulate_updates=False)
pl.set_source('file', location=path)
self.assertEqual(2, len(pl.proxy_list))
# enable accumulate updates
pl = ProxyList()
pl.setup(accumulate_updates=True)
path = self.generate_plist_file()
pl.set_source('file', location=path)
self.assertEqual(2, len(pl.proxy_list))
def test_get_next_proxy(self):
pl = ProxyList()
path = self.generate_plist_file('foo:1\nbar:1')
pl.set_source('file', location=path)
self.assertEqual(pl.get_next_proxy().server, 'foo')
self.assertEqual(pl.get_next_proxy().server, 'bar')
self.assertEqual(pl.get_next_proxy().server, 'foo')
pl.set_source('file', location=path)
self.assertEqual(pl.get_next_proxy().server, 'foo')
def test_get_next_proxy_in_accumulate_mode(self):
pl = ProxyList()
pl.setup(accumulate_updates=True)
path = self.generate_plist_file('foo:1\nbar:1')
pl.set_source('file', location=path)
self.assertEqual(pl.get_next_proxy().server, 'foo')
path = self.generate_plist_file('baz:1')
pl.set_source('file', location=path)
self.assertEqual(pl.get_next_proxy().server, 'bar')
self.assertEqual(pl.get_next_proxy().server, 'baz')
self.assertEqual(pl.get_next_proxy().server, 'foo')
def test_accumulate_updates_basic(self):
pl = ProxyList()
pl.setup(accumulate_updates=True)
# load initial list
path = self.generate_plist_file('foo:1\nbar:1')
pl.set_source('file', location=path)
self.assertEqual(2, len(pl.proxy_list))
# load list with one new and one old proxies
with open(path, 'w') as out:
out.write('foo:1\nbaz:1')
pl.reload(force=True)
self.assertEqual(3, len(pl.proxy_list))
def __init__(self,
document_body=None,
transport='grab.transport.curl.CurlTransport',
**kwargs):
"""
Create Grab instance
"""
self._doc = None
self.config = default_config()
self.config['common_headers'] = self.common_headers()
self.cookies = CookieManager()
self.proxylist = ProxyList()
self.setup_transport(transport)
self.reset()
if kwargs:
self.setup(**kwargs)
if document_body is not None:
self.setup_document(document_body)
def test_remote_load(self):
pl = ProxyList()
SERVER.RESPONSE['get'] = DEFAULT_PLIST_DATA
pl.set_source('url', url=SERVER.BASE_URL)
self.assertEqual(2, len(pl.proxy_list))
def test_file_source(self):
pl = ProxyList()
path = self.generate_plist_file()
pl.set_source('file', location=path)
self.assertEqual(2, len(pl.proxy_list))
def test_basic(self):
pl = ProxyList()
self.assertEqual(0, len(pl.proxy_list))
def test_remote_load(self):
pl = ProxyList()
self.server.response['get.data'] = DEFAULT_PLIST_DATA
pl.set_source('url', url=self.server.get_url())
self.assertEqual(2, len(pl.proxy_list))