def test_content_encoding(self):
"""Ensure we perform (or don't) content decoding properly."""
# Use an existing .gz file, at data/server-content/www.eclipse.org/.../eclipse-standard-luna-R-linux-gtk.tar.gz
filename = "www.eclipse.org/technology/epp/downloads/release/luna/R/eclipse-standard-luna-R-linux-gtk.tar.gz"
length = 10240
compressed_length = 266
url = self.build_server_address(filename +
'-setheaders?content-encoding=gzip')
request = DownloadItem(url)
DownloadCenter([request], self.callback, download=False)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
self.assertEqual(length, len(result.buffer.getvalue()))
# Reset the callback mock.
self.callback = Mock()
request = DownloadItem(url, ignore_encoding=True)
DownloadCenter([request], self.callback, download=False)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
self.assertEqual(compressed_length, len(result.buffer.getvalue()))
def test_download_with_multiple_progress(self):
"""we deliver multiple progress hooks on bigger files"""
filename = "biggerfile"
filesize = getsize(join(self.server_dir, filename))
report = CopyingMock()
request = DownloadItem(self.build_server_address(filename), None)
dl_center = DownloadCenter([request], self.callback, report=report)
self.wait_for_callback(self.callback)
self.assertEqual(report.call_count, 3)
self.assertEqual(report.call_args_list, [
call({
self.build_server_address(filename): {
'size': filesize,
'current': 0
}
}),
call({
self.build_server_address(filename): {
'size': filesize,
'current': dl_center.BLOCK_SIZE
}
}),
call({
self.build_server_address(filename): {
'size': filesize,
'current': filesize
}
})
])
def get_metadata(self, result):
"""Download files to download + and download license license and check it"""
logger.debug("Parse download metadata")
error_msg = result[self.download_page].error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(self.download_page, error_msg))
UI.return_main_screen(status_code=1)
arch = platform.machine()
download_re = r'\'linux64\': \'([^\']+)\''
if arch == 'i686':
download_re = r'\'linux32\': \'([^\']+)\''
url = None
for line in result[self.download_page].buffer:
line = line.decode()
p = re.search(download_re, line)
with suppress(AttributeError):
url = p.group(1)
logger.debug("Found download link for {}".format(url))
if url is None:
logger.error("Download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
self.download_requests.append(DownloadItem(url, Checksum(self.checksum_type, None), headers=self.headers))
if not self.auto_accept_license:
logger.debug("Downloading License page")
DownloadCenter([DownloadItem(self.license_url, headers=self.headers)], self.check_external_license,
download=False)
else:
self.start_download_and_install()
def get_metadata_and_check_license(self, result):
"""We diverge from the BaseInstaller implementation a little here."""
logger.debug("Parse download metadata")
error_msg = result[self.download_page].error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(self.download_page, error_msg))
UI.return_main_screen(status_code=1)
soup = BeautifulSoup(result[self.download_page].buffer, 'html.parser')
# We need to avoid matching arduino-nightly-...
download_link_pat = r'arduino-[\d\.\-r]+-linux' + self.bits + '.tar.xz$'
# Trap no match found, then, download/checksum url will be empty and will raise the error
# instead of crashing.
with suppress(TypeError):
self.scraped_download_url = soup.find('a', href=re.compile(download_link_pat))['href']
self.scraped_checksum_url = soup.find('a', text=re.compile('Checksums'))['href']
self.scraped_download_url = 'http:' + self.scraped_download_url
self.scraped_checksum_url = 'http:' + self.scraped_checksum_url
if not self.scraped_download_url:
logger.error("Can't parse the download link from %s.", self.download_page)
UI.return_main_screen(status_code=1)
if not self.scraped_checksum_url:
logger.error("Can't parse the checksum link from %s.", self.download_page)
UI.return_main_screen(status_code=1)
DownloadCenter([DownloadItem(self.scraped_download_url), DownloadItem(self.scraped_checksum_url)],
on_done=self.prepare_to_download_archive, download=False)
def get_metadata_and_check_license(self, result):
logger.debug("Fetched download page, parsing.")
page = result[self.download_page]
error_msg = page.error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(self.download_page, error_msg))
UI.return_main_screen(status_code=1)
try:
key, content = json.loads(page.buffer.read().decode()).popitem()
download_url = content[0]['downloads']['linux']['link']
checksum_url = content[0]['downloads']['linux']['checksumLink']
except (json.JSONDecodeError, IndexError):
logger.error("Can't parse the download URL from the download page.")
UI.return_main_screen(status_code=1)
logger.debug("Found download URL: " + download_url)
logger.debug("Downloading checksum first, from " + checksum_url)
def checksum_downloaded(results):
checksum_result = next(iter(results.values())) # Just get the first.
if checksum_result.error:
logger.error(checksum_result.error)
UI.return_main_screen(status_code=1)
checksum = checksum_result.buffer.getvalue().decode('utf-8').split()[0]
logger.info('Obtained SHA256 checksum: ' + checksum)
self.download_requests.append(DownloadItem(download_url,
checksum=Checksum(ChecksumType.sha256, checksum),
ignore_encoding=True))
self.start_download_and_install()
DownloadCenter([DownloadItem(checksum_url)], on_done=checksum_downloaded, download=False)
def test_multiple_downloads_with_reports(self):
"""we deliver more than on download in parallel"""
requests = [
DownloadItem(self.build_server_address("biggerfile"), None),
DownloadItem(self.build_server_address("simplefile"), None)
]
report = CopyingMock()
DownloadCenter(requests, self.callback, report=report)
self.wait_for_callback(self.callback)
self.assertEqual(report.call_count, 5)
# ensure that first call only contains one file
callback_args, callback_kwargs = report.call_args_list[0]
map_result = callback_args[0]
self.assertEqual(len(map_result), 1, str(map_result))
# ensure that last call is what we expect
result_dict = {}
for filename in ("biggerfile", "simplefile"):
file_size = getsize(join(self.server_dir, filename))
result_dict[self.build_server_address(filename)] = {
'size': file_size,
'current': file_size
}
self.assertEqual(report.call_args, call(result_dict))
self.assertEqual(self.callback.call_count, 1,
"Global done callback is only called once")
def test_download_with_no_size(self):
"""we deliver one successful download, even if size isn't provided. Progress returns -1 though"""
filename = "simplefile-with-no-content-length"
url = self.build_server_address(filename)
request = DownloadItem(url, None)
report = CopyingMock()
DownloadCenter([request], self.callback, report=report)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(), result.fd.read())
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
self.assertEqual(report.call_count, 2)
self.assertEqual(report.call_args_list, [
call({
self.build_server_address(filename): {
'size': -1,
'current': 0
}
}),
call({
self.build_server_address(filename): {
'size': -1,
'current': 8192
}
})
])
def test_redirect_download(self):
"""we deliver one successful download after being redirected"""
filename = "simplefile"
# We add a suffix to make the server redirect us.
url = TestDownloadCenter.build_server_address(self,
filename + "-redirect",
localhost=True)
request = DownloadItem(url, None)
os.environ['REQUESTS_CA_BUNDLE'] = join(get_data_dir(),
'localhost.pem')
try:
DownloadCenter([request], self.callback)
TestDownloadCenter.wait_for_callback(self, self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
self.assertEqual(
result.final_url,
TestDownloadCenter.build_server_address(self,
filename,
localhost=True))
with open(os.path.join(self.server_dir, filename),
'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(), result.fd.read())
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
finally:
del os.environ['REQUESTS_CA_BUNDLE']
def complete_download_url(self, result):
"""Parse the download page and get the SHASUMS256.txt page"""
version = None
logger.debug("Set the version in the download url")
error_msg = result[self.download_page].error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(
self.download_page, error_msg))
UI.return_main_screen(status_code=1)
for line in result[self.download_page].buffer:
line_content = line.decode()
with suppress(AttributeError):
if self.lts and "most_recent_lts" in line_content:
version = re.search(r': (.*),', line_content).group(1)
elif not self.lts and "most_recent_feature_release" in line_content:
version = re.search(r': (.*),', line_content).group(1)
if not result:
logger.error("Download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
self.download_page = "https://api.adoptopenjdk.net/v3/assets/latest/{}/".format(version) + \
"{}?release=latest&".format(self.jvm_impl) + \
"jvm_impl={}".format(self.jvm_impl) + \
"&vendor=adoptopenjdk&="
DownloadCenter([DownloadItem(self.download_page)],
self.get_metadata_and_check_license,
download=False)
def parse_shasum_page(self, result):
"""Parse the download page and get the SHASUMS256.txt page"""
logger.debug("Parse download metadata")
error_msg = result[self.download_page].error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(
self.download_page, error_msg))
UI.return_main_screen(status_code=1)
url = False
for line in result[self.download_page].buffer:
line_content = line.decode()
with suppress(AttributeError):
shasum_url = re.search('a href="(.*SHASUMS\d\d\d\.txt\.asc)"',
line_content).group(1)
if not result:
logger.error("Download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
self.download_page = shasum_url
DownloadCenter([DownloadItem(self.download_page)],
self.get_metadata_and_check_license,
download=False)
def get_metadata_and_check_license(self, result):
"""We diverge from the BaseInstaller implementation a little here."""
logger.debug("Parse download metadata")
error_msg = result[self.download_page].error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(self.download_page, error_msg))
UI.return_main_screen()
soup = BeautifulSoup(result[self.download_page].buffer)
# We need to avoid matching arduino-nightly-...
download_link_pat = r'arduino-(\d\.?){1,3}-linux' + self.bits + '.tar.xz$'
self.scraped_download_url = soup.find('a', href=re.compile(download_link_pat))['href']
self.scraped_checksum_url = soup.find('a', text=re.compile('Checksums'))['href']
self.scraped_download_url = 'http:' + self.scraped_download_url
self.scraped_checksum_url = 'http:' + self.scraped_checksum_url
if not self.scraped_download_url:
logger.error("Can't parse the download link from %s.", self.download_page)
UI.return_main_screen()
if not self.scraped_checksum_url:
logger.error("Can't parse the checksum link from %s.", self.download_page)
UI.return_main_screen()
DownloadCenter([DownloadItem(self.scraped_download_url), DownloadItem(self.scraped_checksum_url)],
on_done=self.prepare_to_download_archive, download=False)
def get_metadata_and_check_license(self, result):
logger.debug("Fetched download page, parsing.")
page = result[self.download_page]
error_msg = page.error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(self.download_page_url, error_msg))
UI.return_main_screen()
soup = BeautifulSoup(page.buffer)
link = soup.find('a', text="HTTPS")
if link is None:
logger.error("Can't parse the download URL from the download page.")
UI.return_main_screen()
download_url = link.attrs['href']
checksum_url = download_url + '.sha256'
logger.debug("Found download URL: " + download_url)
logger.debug("Downloading checksum first, from " + checksum_url)
def checksum_downloaded(results):
checksum_result = next(iter(results.values())) # Just get the first.
if checksum_result.error:
logger.error(checksum_result.error)
UI.return_main_screen()
checksum = checksum_result.buffer.getvalue().decode('utf-8').split()[0]
logger.info('Obtained SHA256 checksum: ' + checksum)
self.download_requests.append(DownloadItem(download_url,
checksum=Checksum(ChecksumType.sha256, checksum),
ignore_encoding=True))
self.start_download_and_install()
DownloadCenter([DownloadItem(checksum_url)], on_done=checksum_downloaded, download=False)
def get_metadata_and_check_license(self, result):
"""Get the latest version and trigger the download of the download_page file.
:param result: the file downloaded by DownloadCenter, contains a web page
"""
# Processing the string to obtain metadata (version)
try:
url_version_str = result[self.download_page].buffer.read().decode(
'utf-8')
except AttributeError:
# The file could not be parsed or there is no network connection
logger.error(
"The download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
preg = re.compile(".*/images_www/v6/download/.*")
for line in url_version_str.split("\n"):
if preg.match(line):
line = line.replace(
"var PAGE_ARTIFACTS_LOCATION = \"/images"
"_www/v6/download/", "").replace("/\";",
"").replace('/final', '')
self.version = line.strip()
if not self.version:
# Fallback
logger.error("Could not determine latest version")
UI.return_main_screen(status_code=1)
self.version_download_page = "https://netbeans.org/images_www/v6/download/" \
"{}/final/js/files.js".format(self.version)
DownloadCenter([DownloadItem(self.version_download_page)],
self.parse_download_page_callback,
download=False)
def get_metadata_and_check_license(self, result):
"""Download files to download + license and check it"""
logger.debug("Parse download metadata")
error_msg = result[self.download_page].error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(
self.download_page, error_msg))
UI.return_main_screen(status_code=1)
in_download = False
url_found = False
for line in result[self.download_page].buffer:
if url_found is None or self.match_last_link:
line_content = line.decode()
(_url_found, in_download) = self.parse_download_link(
line_content, in_download)
if not url_found:
url_found = _url_found
if not url_found:
logger.error("Download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
DownloadCenter(urls=[DownloadItem(self.download_url, None)],
on_done=self.get_url_and_start_download,
download=False)
def get_metadata_and_check_license(self, result):
"""Download files to download + license and check it"""
logger.debug("Parse download metadata")
error_msg = result[self.download_page].error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(
self.download_page, error_msg))
UI.return_main_screen(status_code=1)
in_download = False
sig_url = None
for line in result[self.download_page].buffer:
line_content = line.decode()
(new_sig_url,
in_download) = self.parse_download_link(line_content, in_download)
if str(new_sig_url) > str(sig_url):
# Avoid fetching development snapshots
if 'DEVELOPMENT-SNAPSHOT' not in new_sig_url:
tmp_release = re.search("ubuntu(.....).tar",
new_sig_url).group(1)
if tmp_release <= get_current_distro_version():
sig_url = new_sig_url
if not sig_url:
logger.error("Download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
DownloadCenter(urls=[
DownloadItem(sig_url, None),
DownloadItem(self.asc_url, None)
],
on_done=self.check_gpg_and_start_download,
download=False)
def post_install(self):
"""Create the GDevelop launcher"""
DownloadCenter(urls=[DownloadItem(self.icon_url, None)],
on_done=self.save_icon, download=True)
create_launcher(self.desktop_filename, get_application_desktop_file(name=_("GDevelop"),
icon_path=os.path.join(self.install_path, self.icon_filename),
try_exec=self.exec_path,
exec=self.exec_link_name,
comment=self.description,
categories="Development;IDE;"))
def test_download_with_wrong_sha256(self):
"""we raise an error if we don't have the correct sha256"""
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request, Checksum(ChecksumType.sha256, 'AAAAA'))], self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertIn("Corrupted download", result.error)
self.assertIsNone(result.buffer)
self.assertIsNone(result.fd)
self.expect_warn_error = True
def test_download_with_progress(self):
"""we deliver progress hook while downloading"""
filename = "simplefile"
filesize = getsize(join(self.server_dir, filename))
report = CopyingMock()
request = DownloadItem(self.build_server_address(filename), None)
DownloadCenter([request], self.callback, report=report)
self.wait_for_callback(self.callback)
self.assertEqual(report.call_count, 2)
self.assertEqual(report.call_args_list,
[call({self.build_server_address(filename): {'size': filesize, 'current': 0}}),
call({self.build_server_address(filename): {'size': filesize, 'current': filesize}})])
def test_with_invalid_certificate(self):
"""we error on invalid ssl certificate"""
filename = "simplefile"
url = TestDownloadCenter.build_server_address(self, filename)
request = DownloadItem(url, None)
DownloadCenter([request], self.callback)
TestDownloadCenter.wait_for_callback(self, self.callback)
result = self.callback.call_args[0][0][url]
self.assertIn("CERTIFICATE_VERIFY_FAILED", result.error)
self.assertIsNone(result.buffer)
self.assertIsNone(result.fd)
self.expect_warn_error = True
def post_install(self):
"""Create the Eclipse launcher"""
DownloadCenter(urls=[DownloadItem(self.icon_url, None)],
on_done=self.save_icon, download=True)
icon_path = join(self.install_path, self.icon_filename)
comment = self.description
categories = "Development;IDE;"
create_launcher(self.desktop_filename,
get_application_desktop_file(name=self.name,
icon_path=icon_path,
exec='"{}" %f'.format(self.exec_path),
comment=comment,
categories=categories))
def test_unsupported_protocol(self):
"""Raises an exception when trying to download for an unsupported protocol"""
filename = "simplefile"
url = self.build_server_address(filename).replace('http', 'sftp')
request = DownloadItem(url, None)
DownloadCenter([request], self.callback, download=False)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertIn("Protocol not supported", result.error)
self.assertIsNone(result.buffer)
self.assertIsNone(result.fd)
self.expect_warn_error = True
def test_download_keep_extensions(self):
"""we deliver successful downloads keeping the extension"""
filename = "android-studio-fake.tgz"
url = self.build_server_address(filename)
request = DownloadItem(url, None)
DownloadCenter([request], self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb'):
self.assertTrue(result.fd.name.endswith('.tgz'), result.fd.name)
def test_404_url(self):
"""we return an error for a request including a 404 url"""
request = DownloadItem(self.build_server_address("does_not_exist"), None)
DownloadCenter([request], self.callback)
self.wait_for_callback(self.callback)
# no download means the file isn't in the result
callback_args, callback_kwargs = self.callback.call_args
result = callback_args[0][self.build_server_address("does_not_exist")]
self.assertIn("404", result.error)
self.assertIn("File not found", result.error)
self.assertIsNone(result.buffer)
self.assertIsNone(result.fd)
self.expect_warn_error = True
def start_download_and_install(self):
self.last_progress_download = None
self.last_progress_requirement = None
self.balance_requirement_download = None
self.pkg_size_download = 0
self.result_requirement = None
self.result_download = None
self._download_done_callback_called = False
UI.display(DisplayMessage("Downloading and installing requirements"))
self.pbar = ProgressBar().start()
self.pkg_to_install = RequirementsHandler().install_bucket(self.packages_requirements,
self.get_progress_requirement,
self.requirement_done)
DownloadCenter(urls=self.download_requests, on_done=self.download_done, report=self.get_progress_download)
def test_multiple_with_one_404_url(self):
"""we raise an error when we try to download 404 urls"""
requests = [DownloadItem(self.build_server_address("does_not_exist"), None),
DownloadItem(self.build_server_address("simplefile"), None)]
DownloadCenter(requests, self.callback)
self.wait_for_callback(self.callback)
# we should have the two content, one in error
callback_args, callback_kwargs = self.callback.call_args
map_result = callback_args[0]
self.assertEqual(len(map_result), 2, str(map_result))
self.assertIsNotNone(map_result[self.build_server_address("does_not_exist")].error)
self.assertIsNotNone(map_result[self.build_server_address("simplefile")].fd)
self.expect_warn_error = True
def parse_download_link(self, line, in_download):
"""Parse Maven download link, expect to find a url"""
url_found = False
if '-bin.tar.gz.md5' in line:
in_download = True
else:
in_download = False
if in_download:
p = re.search(r'href="(.*)"', line)
with suppress(AttributeError):
self.checksum_url = p.group(1)
url_found = True
DownloadCenter(urls=[DownloadItem(self.checksum_url, None)],
on_done=self.get_sha_and_start_download, download=False)
return (url_found, in_download)
def parse_download_link(self, line, in_download):
"""Parse STS download links"""
url_found = False
if 'linux-gtk{}.tar.gz'.format(self.arch) in line:
in_download = True
else:
in_download = False
if in_download:
p = re.search(r'href="(.*.tar.gz)"', line)
with suppress(AttributeError):
self.checksum_url = p.group(1) + '.sha1'
url_found = True
DownloadCenter(urls=[DownloadItem(self.checksum_url, None)],
on_done=self.get_sha_and_start_download, download=False)
return (url_found, in_download)
def parse_download_link(self, line, in_download):
"""Parse Eclipse download links"""
url_found = False
if self.download_keyword in line and self.bits in line:
in_download = True
else:
in_download = False
if in_download:
p = re.search(r'href="(.*)" title', line)
with suppress(AttributeError):
self.sha512_url = self.download_page + p.group(1) + '.sha512&r=1'
url_found = True
DownloadCenter(urls=[DownloadItem(self.sha512_url, None)],
on_done=self.get_sha_and_start_download, download=False)
return (url_found, in_download)
def test_in_memory_download(self):
"""we deliver download on memory objects"""
filename = "simplefile"
url = self.build_server_address(filename)
request = DownloadItem(url, None)
DownloadCenter([request], self.callback, download=False)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(), result.buffer.read())
self.assertIsNone(result.fd)
self.assertIsNone(result.error)
def test_cookies(self):
"""Test handing of outgoing and incoming cookies."""
filename = "simplefile"
url = self.build_server_address(filename)
# The server is rigged to inc the 'int' cookie by one.
cookies = {'int': '5'}
request = DownloadItem(url, None, cookies=cookies)
DownloadCenter([request], self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
self.assertEqual('6', result.cookies['int'])