Python get_absolute_url_split示例

示例#1

文件： tests.py 项目： jgriffith/pylinkchecker

 def test_get_absolute_url(self):
     base_url_split = get_clean_url_split(
             "https://www.example.com/hello/index.html")
     self.assertEqual("https://www.example2.com/test.js",
         get_absolute_url_split("//www.example2.com/test.js",
                 base_url_split).geturl())
     self.assertEqual("https://www.example.com/hello2/test.html",
         get_absolute_url_split("/hello2/test.html",
                 base_url_split).geturl())
     self.assertEqual("https://www.example.com/hello/test.html",
         get_absolute_url_split("test.html", base_url_split).geturl())
     self.assertEqual("https://www.example.com/test.html",
         get_absolute_url_split("../test.html", base_url_split).geturl())

示例#2

 def test_get_absolute_url(self):
     base_url_split = get_clean_url_split(
             "https://www.example.com/hello/index.html")
     self.assertEqual("https://www.example2.com/test.js",
         get_absolute_url_split("//www.example2.com/test.js",
                 base_url_split).geturl())
     self.assertEqual("https://www.example.com/hello2/test.html",
         get_absolute_url_split("/hello2/test.html",
                 base_url_split).geturl())
     self.assertEqual("https://www.example.com/hello/test.html",
         get_absolute_url_split("test.html", base_url_split).geturl())
     self.assertEqual("https://www.example.com/test.html",
         get_absolute_url_split("../test.html", base_url_split).geturl())

示例#3

文件： crawler.py 项目： shezadkhan137/pylinkchecker

    def _get_links(self, elements, attribute, base_url_split,
        original_url_split):
        links = []
        for element in elements:
            if attribute in element.attrs:
                url = element[attribute]
                if not is_link(url):
                    continue
                abs_url_split = get_absolute_url_split(url, base_url_split)

                if abs_url_split.scheme not in SUPPORTED_SCHEMES:
                    continue

                link = Link(type=unicode(element.name), url_split=abs_url_split,
                    original_url_split=original_url_split,
                    source_str=unicode(element))
                links.append(link)

        return links

示例#4

文件： crawler.py 项目： randomtask1155/pylinkchecker

    def _get_links(self, elements, attribute, base_url_split,
        original_url_split):
        links = []
        for element in elements:
            if attribute in element.attrs:
                url = element[attribute]

                if not self.worker_config.strict_mode:
                    url = url.strip()

                if not is_link(url):
                    continue
                abs_url_split = get_absolute_url_split(url, base_url_split)

                if abs_url_split.scheme not in SUPPORTED_SCHEMES:
                    continue

                link = Link(type=unicode(element.name), url_split=abs_url_split,
                    original_url_split=original_url_split,
                    source_str=unicode(element))
                links.append(link)

        return links