def test_is_url(self):
self.assertTrue(is_url('http://github.com'))
self.assertTrue(is_url('https://github.com/'))
self.assertTrue(is_url('file://localhost/etc/fstab'))
self.assertFalse(is_url('github.com'))
self.assertFalse(is_url('/etc/conf'))
def test_is_url(self):
self.assertTrue(is_url('http://github.com'))
self.assertTrue(is_url('https://github.com/'))
self.assertTrue(is_url('file://localhost/etc/fstab'))
self.assertFalse(is_url('github.com'))
self.assertFalse(is_url('/etc/conf'))
def get_spider(self, engine, args, options):
spiders = engine.spiders
if getattr(options, 'spider', None):
return spiders.create_spider_by_name(options.spider)
if len(args) == 1 and is_url(args[0]):
spider = spiders.create_spider_by_url(args[0])
if spider:
return spider
return BaseSpider('default')
def get_spider(self, engine, args, options):
spiders = engine.spiders
if getattr(options, 'spider', None):
return spiders.create_spider_by_name(options.spider)
if len(args) == 1 and is_url(args[0]):
spider = spiders.create_spider_by_url(args[0])
if spider:
return spider
return BaseSpider('default')
