def test_constructor_meta_argument(self):
req = Request()
self.assertEqual(req.meta, {})
req = Request(meta=1)
self.assertEqual(req.meta, 1)
req = Request(meta={'place': 'hell'})
self.assertEqual(req.meta, {'place': 'hell'})
def task_generator(self):
yield Request('test',
url=server.get_url(),
timeout=0.1,
meta={'id': 1})
yield Request('test',
url=server.get_url(),
timeout=0.5,
meta={'id': 2})
def test_constructor_arguments(self):
req = Request()
self.assertEqual(req.tag, None)
self.assertEqual(req.url, None)
self.assertEqual(req.callback, None)
req = Request('TAG')
self.assertEqual(req.tag, 'TAG')
self.assertEqual(req.url, None)
req = Request('TAG', 'URL')
self.assertEqual(req.tag, 'TAG')
self.assertEqual(req.url, 'URL')
req = Request(url='URL', tag='TAG')
self.assertEqual(req.tag, 'TAG')
self.assertEqual(req.url, 'URL')
def test_single_request(self):
class SimpleCrawler(Crawler):
data = {}
def handler_test(self, req, res):
self.data['response'] = res.body
token = 'Python'
self.server.response['data'] = token
bot = SimpleCrawler()
bot.add_task(Request('test', url=self.server.get_url()))
bot.run()
self.assertEquals(token, bot.data['response'])
def task_generator(self):
yield Request('test', url=server.get_url(), meta={'id': 1})
def handler_test2(self, req, res):
self.points.append(req.meta['id'])
yield Request('test3', url=server.get_url(), meta={'id': 3})
def task_generator(self):
for url in self._urls_todo:
yield Request('test', url=url)
def task_generator(self):
for x in range(10):
yield Request('test', url=server.get_url())
def task_generator(self):
yield Request('test', url=server.get_url())
def task_generator(self):
for host in islice(open('var/domains.txt'), 20):
host = host.strip()
if host:
yield Request('http://%s/' % host, tag='page')
def task_generator(self):
yield Request('test', url=server.get_url(),
callback=self.handler_test)