def test_task_clone_grab_config_and_url(self):
g = build_grab()
g.setup(url='http://foo.com/')
task = Task('foo', grab=g)
task2 = task.clone(url='http://bar.com/')
self.assertEqual(task2.url, 'http://bar.com/')
self.assertEqual(task2.grab_config['url'], 'http://bar.com/')
def test_task_clone_grab_config_and_url(self):
grab = build_grab()
grab.setup(url='http://foo.com/')
task = Task('foo', grab=grab)
task2 = task.clone(url='http://bar.com/')
self.assertEqual(task2.url, 'http://bar.com/')
self.assertEqual(task2.grab_config['url'], 'http://bar.com/')
def test_task_clone(self):
bot = build_spider(SimpleSpider, )
bot.setup_queue()
task = Task('baz', url='http://xxx.com')
bot.add_task(task.clone())
# Pass grab to clone
task = Task('baz', url='http://xxx.com')
grab = Grab()
grab.setup(url='zzz')
bot.add_task(task.clone(grab=grab))
# Pass grab_config to clone
task = Task('baz', url='http://xxx.com')
grab = Grab()
grab.setup(url='zzz')
bot.add_task(task.clone(grab_config=grab.config))
def test_task_clone(self):
bot = build_spider(SimpleSpider, )
bot.setup_queue()
task = Task('baz', url='xxx')
bot.add_task(task.clone())
# Pass grab to clone
task = Task('baz', url='xxx')
g = Grab()
g.setup(url='zzz')
bot.add_task(task.clone(grab=g))
# Pass grab_config to clone
task = Task('baz', url='xxx')
g = Grab()
g.setup(url='zzz')
bot.add_task(task.clone(grab_config=g.config))
def test_task_clone(self):
bot = SimpleSpider()
bot.setup_queue()
task = Task('baz', url='xxx')
bot.add_task(task.clone())
# Pass grab to clone
task = Task('baz', url='xxx')
g = Grab()
g.setup(url='zzz')
bot.add_task(task.clone(grab=g))
# Pass grab_config to clone
task = Task('baz', url='xxx')
g = Grab()
g.setup(url='zzz')
bot.add_task(task.clone(grab_config=g.config))
def test_task_useragent(self):
bot = build_spider(SimpleSpider, )
bot.setup_queue()
grab = Grab()
grab.setup(url=self.server.get_url())
grab.setup(user_agent='Foo')
task = Task('baz', grab=grab)
bot.add_task(task.clone())
bot.run()
self.assertEqual(self.server.request['headers']['User-Agent'], 'Foo')
def test_task_useragent(self):
bot = build_spider(SimpleSpider, )
bot.setup_queue()
g = Grab()
g.setup(url=self.server.get_url())
g.setup(user_agent='Foo')
task = Task('baz', grab=g)
bot.add_task(task.clone())
bot.run()
self.assertEqual(self.server.request['headers']['User-Agent'], 'Foo')
def test_task_useragent(self):
bot = SimpleSpider()
bot.setup_queue()
g = Grab()
g.setup(url=SERVER.BASE_URL)
g.setup(user_agent='Foo')
task = Task('baz', grab=g)
bot.add_task(task.clone())
bot.run()
self.assertEqual(SERVER.REQUEST['headers']['User-Agent'], 'Foo')
def test_task_useragent(self):
bot = SimpleSpider()
bot.setup_queue()
g = Grab()
g.setup(url=SERVER.BASE_URL)
g.setup(user_agent='Foo')
task = Task('baz', grab=g)
bot.add_task(task.clone())
bot.run()
self.assertEqual(SERVER.REQUEST['headers']['User-Agent'], 'Foo')
def test_task_clone_with_url_param(self):
task = Task('baz', url='http://example.com/path')
task2 = task.clone(url='http://example.com/new')
self.assertEqual(task2.name, 'baz')
self.assertEqual(task2.url, 'http://example.com/new')
def test_task_clone_kwargs(self):
grab = build_grab()
grab.setup(url='http://foo.com/')
task = Task('foo', grab=grab, cache_timeout=1)
task2 = task.clone(cache_timeout=2)
self.assertEqual(2, task2.cache_timeout)
def test_task_clone_with_url_param(self):
task = Task('baz', url='xxx')
task.clone(url='http://yandex.ru/')
def test_task_clone_kwargs(self):
g = build_grab()
g.setup(url='http://foo.com/')
task = Task('foo', grab=g, cache_timeout=1)
task2 = task.clone(cache_timeout=2)
self.assertEqual(2, task2.cache_timeout)
def test_task_clone_kwargs(self):
grab = build_grab()
grab.setup(url='http://foo.com/')
task = Task('foo', grab=grab, foo=1)
task2 = task.clone(foo=2)
self.assertEqual(2, task2.foo) # pylint: disable=no-member
def test_task_clone_with_url_param(self):
task = Task('baz', url='xxx')
task.clone(url='http://yandex.ru/')
