def test_robotstxt_test(): user_agents = ['Googlebot', 'Baiduspider', '*'] urls_to_check = ['/', '/help', 'something.html'] result = robotstxt_test(robots_file, user_agents, urls_to_check) assert isinstance(result, pd.core.frame.DataFrame) assert all(col in result for col in ['robotstxt_url', 'user_agent', 'url_path', 'can_fetch'])
def test_robots_converts_str_to_list(): result = robotstxt_test('https://www.apple.com/robots.txt', '*', 'hello') assert isinstance(result, pd.core.frame.DataFrame) assert all( col in result for col in ['robotstxt_url', 'user_agent', 'url_path', 'can_fetch'])
def test_robotstxt_raises(): with pytest.raises(ValueError): robotstxt_test('http://www.wrong-url.com', '*', '/')