def test_allow_robots_txt(self): '''Allows robots.txt.''' robot = robots.AllowNone('http://example.com/robots.txt') self.assertTrue(robot.allowed('/robots.txt', 'agent'))
def test_allow(self): '''Allows nothing.''' robot = robots.AllowNone('http://example.com/robots.txt') self.assertFalse(robot.allowed('/', 'agent'))