def test_robots_given_asterisk_path_allowed_url(self): # allowed by /*/*/tree/master url = "https://github.com/rivermont/spidy/tree/master" checker = init_robot_checker(True, 'duckduckbot', url) self.assertFalse(check_link(url, checker))
def test_robots_given_lower_path_allowed_url(self): # allowed by /search/about after /search is forbidden url = "https://google.com/search/about" checker = init_robot_checker(True, 'duckduckbot', url) self.assertFalse(check_link(url, checker))
def test_robots_given_allowed_url(self): # allowed expliticly url = "https://www.google.com/m/finance" checker = init_robot_checker(True, 'duckduckbot', url) self.assertFalse(check_link(url, checker))