def test_robotstxt_test():
user_agents = ['Googlebot', 'Baiduspider', '*']
urls_to_check = ['/', '/help', 'something.html']
result = robotstxt_test(robots_file, user_agents, urls_to_check)
assert isinstance(result, pd.core.frame.DataFrame)
assert all(col in result for col in
['robotstxt_url', 'user_agent', 'url_path', 'can_fetch'])
def test_robots_converts_str_to_list():
result = robotstxt_test('https://www.apple.com/robots.txt', '*', 'hello')
assert isinstance(result, pd.core.frame.DataFrame)
assert all(
col in result
for col in ['robotstxt_url', 'user_agent', 'url_path', 'can_fetch'])
def test_robotstxt_raises():
with pytest.raises(ValueError):
robotstxt_test('http://www.wrong-url.com', '*', '/')
