def test_is_url(self): self.assertTrue(is_url('http://github.com')) self.assertTrue(is_url('https://github.com/')) self.assertTrue(is_url('file://localhost/etc/fstab')) self.assertFalse(is_url('github.com')) self.assertFalse(is_url('/etc/conf'))
def get_spider(self, engine, args, options): spiders = engine.spiders if getattr(options, 'spider', None): return spiders.create_spider_by_name(options.spider) if len(args) == 1 and is_url(args[0]): spider = spiders.create_spider_by_url(args[0]) if spider: return spider return BaseSpider('default')