Python init_robot_checker Examples

Example #1

    def test_robots_given_asterisk_path_allowed_url(self):
        # allowed by /*/*/tree/master
        url = "https://github.com/rivermont/spidy/tree/master"
        checker = init_robot_checker(True, 'duckduckbot', url)

        self.assertFalse(check_link(url, checker))

Example #2

    def test_robots_given_lower_path_allowed_url(self):
        # allowed by /search/about after /search is forbidden
        url = "https://google.com/search/about"
        checker = init_robot_checker(True, 'duckduckbot', url)

        self.assertFalse(check_link(url, checker))

Example #3

    def test_robots_given_allowed_url(self):
        # allowed expliticly
        url = "https://www.google.com/m/finance"
        checker = init_robot_checker(True, 'duckduckbot', url)

        self.assertFalse(check_link(url, checker))