Пример #1
0
    def _fetch_build_info_from_url(self, url, index, lst):
        """
        Retrieve information from a build folder url.

        Stores in a list the url index and a dict instance with keys
        build_url and build_txt_url if respectively a build file and a
        build info file are found for the url.
        """
        LOG.debug("Fetching build info from {}".format(url))
        data = {}
        if not url.endswith("/"):
            url += "/"
        links = url_links(url)
        if not self.fetch_config.has_build_info:
            links += url_links(self.fetch_config.get_nightly_info_url(url))
        for link in links:
            name = os.path.basename(link)
            if "build_url" not in data and self.build_regex.match(name):
                data["build_url"] = link
            elif "build_txt_url" not in data and self.build_info_regex.match(
                    name):
                data["build_txt_url"] = link
        if data:
            # Check that we found all required data. The URL in build_url is
            # required. build_txt_url is optional.
            if "build_url" not in data:
                raise BuildInfoNotFound(
                    "Failed to find a build file in directory {} that "
                    "matches regex '{}'".format(url, self.build_regex.pattern))

            with self._fetch_lock:
                lst.append((index, data))
Пример #2
0
    def _fetch_build_info_from_url(self, url, index, lst):
        """
        Retrieve information from a build folder url.

        Stores in a list the url index and a dict instance with keys
        build_url and build_txt_url if respectively a build file and a
        build info file are found for the url.
        """
        data = {}
        if not url.endswith("/"):
            url += "/"
        for link in url_links(url):
            if "build_url" not in data and self.build_regex.match(link):
                data["build_url"] = url + link
            elif "build_txt_url" not in data and self.build_info_regex.match(
                    link):
                data["build_txt_url"] = url + link
        if data:
            # Check that we found all required data. The URL in build_url is
            # required. build_txt_url is optional.
            if "build_url" not in data:
                raise BuildInfoNotFound(
                    "Failed to find a build file in directory {} that "
                    "matches regex '{}'".format(url, self.build_regex.pattern))

            with self._fetch_lock:
                lst.append((index, data))
Пример #3
0
 def test_url_with_absolute_links(self, get):
     get.return_value = Mock(text="""
     <body>
     <a href="/useless/thing/">thing</a>
     <a href="/useless/thing2">thing2</a>
     </body>
     """)
     self.assertEquals(network.url_links(''), ['thing/', 'thing2'])
Пример #4
0
 def test_url_with_links_regex(self, get):
     get.return_value = Mock(text="""
     <body>
     <a href="thing/">thing</a>
     <a href="thing2/">thing2</a>
     </body>
     """)
     self.assertEquals(network.url_links('', regex="thing2.*"), ['thing2/'])
Пример #5
0
 def test_url_with_links(self, get):
     get.return_value = Mock(text="""
     <body>
     <a href="thing/">thing</a>
     <a href="thing2/">thing2</a>
     </body>
     """)
     self.assertEqual(network.url_links(""), ["thing/", "thing2/"])
Пример #6
0
 def test_url_with_absolute_links(self, get):
     get.return_value = Mock(text="""
     <body>
     <a href="/useless/thing/">thing</a>
     <a href="/useless/thing2">thing2</a>
     </body>
     """)
     self.assertEquals(network.url_links(''),
                       ['thing/', 'thing2'])
Пример #7
0
 def test_url_with_links_regex(self, get):
     get.return_value = Mock(text="""
     <body>
     <a href="thing/">thing</a>
     <a href="thing2/">thing2</a>
     </body>
     """)
     self.assertEquals(
         network.url_links('', regex="thing2.*"),
         ['thing2/'])
Пример #8
0
    def _fetch_build_info_from_url(self, url, index, lst):
        """
        Retrieve information from a build folder url.

        Stores in a list the url index and a dict instance with keys
        build_url and build_txt_url if respectively a build file and a
        build info file are found for the url.
        """
        data = {}
        if not url.endswith('/'):
            url += '/'
        for link in url_links(url):
            if 'build_url' not in data and self.build_regex.match(link):
                data['build_url'] = url + link
            elif 'build_txt_url' not in data  \
                    and self.build_info_regex.match(link):
                data['build_txt_url'] = url + link
        if data:
            with self._fetch_lock:
                lst.append((index, data))
Пример #9
0
    def _fetch_build_info_from_url(self, url, index, lst):
        """
        Retrieve information from a build folder url.

        Stores in a list the url index and a dict instance with keys
        build_url and build_txt_url if respectively a build file and a
        build info file are found for the url.
        """
        data = {}
        if not url.endswith('/'):
            url += '/'
        for link in url_links(url):
            if 'build_url' not in data and self.build_regex.match(link):
                data['build_url'] = url + link
            elif 'build_txt_url' not in data  \
                    and self.build_info_regex.match(link):
                data['build_txt_url'] = url + link
        if data:
            with self._fetch_lock:
                lst.append((index, data))
Пример #10
0
 def _get_month_links(self, url):
     with self._lock:
         if url not in self._cache_months:
             self._cache_months[url] = url_links(url)
         return self._cache_months[url]
Пример #11
0
 def _get_month_links(self, url):
     with self._lock:
         if url not in self._cache_months:
             self._cache_months[url] = url_links(url)
         return self._cache_months[url]
Пример #12
0
 def test_url_no_links(self, get):
     get.return_value = Mock(text='')
     self.assertEquals(network.url_links(''), [])
Пример #13
0
 def test_url_no_links(self, get):
     get.return_value = Mock(text='')
     self.assertEquals(network.url_links(''), [])