def _fetch_build_info_from_url(self, url, index, lst):
"""
Retrieve information from a build folder url.
Stores in a list the url index and a dict instance with keys
build_url and build_txt_url if respectively a build file and a
build info file are found for the url.
"""
LOG.debug("Fetching build info from {}".format(url))
data = {}
if not url.endswith("/"):
url += "/"
links = url_links(url)
if not self.fetch_config.has_build_info:
links += url_links(self.fetch_config.get_nightly_info_url(url))
for link in links:
name = os.path.basename(link)
if "build_url" not in data and self.build_regex.match(name):
data["build_url"] = link
elif "build_txt_url" not in data and self.build_info_regex.match(
name):
data["build_txt_url"] = link
if data:
# Check that we found all required data. The URL in build_url is
# required. build_txt_url is optional.
if "build_url" not in data:
raise BuildInfoNotFound(
"Failed to find a build file in directory {} that "
"matches regex '{}'".format(url, self.build_regex.pattern))
with self._fetch_lock:
lst.append((index, data))
def _fetch_build_info_from_url(self, url, index, lst):
"""
Retrieve information from a build folder url.
Stores in a list the url index and a dict instance with keys
build_url and build_txt_url if respectively a build file and a
build info file are found for the url.
"""
data = {}
if not url.endswith("/"):
url += "/"
for link in url_links(url):
if "build_url" not in data and self.build_regex.match(link):
data["build_url"] = url + link
elif "build_txt_url" not in data and self.build_info_regex.match(
link):
data["build_txt_url"] = url + link
if data:
# Check that we found all required data. The URL in build_url is
# required. build_txt_url is optional.
if "build_url" not in data:
raise BuildInfoNotFound(
"Failed to find a build file in directory {} that "
"matches regex '{}'".format(url, self.build_regex.pattern))
with self._fetch_lock:
lst.append((index, data))
def test_url_with_absolute_links(self, get):
get.return_value = Mock(text="""
<body>
<a href="/useless/thing/">thing</a>
<a href="/useless/thing2">thing2</a>
</body>
""")
self.assertEquals(network.url_links(''), ['thing/', 'thing2'])
def test_url_with_links_regex(self, get):
get.return_value = Mock(text="""
<body>
<a href="thing/">thing</a>
<a href="thing2/">thing2</a>
</body>
""")
self.assertEquals(network.url_links('', regex="thing2.*"), ['thing2/'])
def test_url_with_links(self, get):
get.return_value = Mock(text="""
<body>
<a href="thing/">thing</a>
<a href="thing2/">thing2</a>
</body>
""")
self.assertEqual(network.url_links(""), ["thing/", "thing2/"])
def test_url_with_absolute_links(self, get):
get.return_value = Mock(text="""
<body>
<a href="/useless/thing/">thing</a>
<a href="/useless/thing2">thing2</a>
</body>
""")
self.assertEquals(network.url_links(''),
['thing/', 'thing2'])
def test_url_with_links_regex(self, get):
get.return_value = Mock(text="""
<body>
<a href="thing/">thing</a>
<a href="thing2/">thing2</a>
</body>
""")
self.assertEquals(
network.url_links('', regex="thing2.*"),
['thing2/'])
def _fetch_build_info_from_url(self, url, index, lst):
"""
Retrieve information from a build folder url.
Stores in a list the url index and a dict instance with keys
build_url and build_txt_url if respectively a build file and a
build info file are found for the url.
"""
data = {}
if not url.endswith('/'):
url += '/'
for link in url_links(url):
if 'build_url' not in data and self.build_regex.match(link):
data['build_url'] = url + link
elif 'build_txt_url' not in data \
and self.build_info_regex.match(link):
data['build_txt_url'] = url + link
if data:
with self._fetch_lock:
lst.append((index, data))
def _fetch_build_info_from_url(self, url, index, lst):
"""
Retrieve information from a build folder url.
Stores in a list the url index and a dict instance with keys
build_url and build_txt_url if respectively a build file and a
build info file are found for the url.
"""
data = {}
if not url.endswith('/'):
url += '/'
for link in url_links(url):
if 'build_url' not in data and self.build_regex.match(link):
data['build_url'] = url + link
elif 'build_txt_url' not in data \
and self.build_info_regex.match(link):
data['build_txt_url'] = url + link
if data:
with self._fetch_lock:
lst.append((index, data))
def _get_month_links(self, url):
with self._lock:
if url not in self._cache_months:
self._cache_months[url] = url_links(url)
return self._cache_months[url]
def _get_month_links(self, url):
with self._lock:
if url not in self._cache_months:
self._cache_months[url] = url_links(url)
return self._cache_months[url]
def test_url_no_links(self, get):
get.return_value = Mock(text='')
self.assertEquals(network.url_links(''), [])
def test_url_no_links(self, get):
get.return_value = Mock(text='')
self.assertEquals(network.url_links(''), [])