def test_succeed(self):
responses.add(responses.GET, "http://google.com/robots.txt", status=200, body="Disallow: /failure")
self.assertEqual(fetch_robots_txt("http://google.com/test.html"), "Disallow: /failure")
def test_fail(self):
responses.add(responses.GET, "http://google.com/robots.txt", status=404)
with self.assertRaises(requests.exceptions.RequestException):
fetch_robots_txt("http://google.com/test.html")
