def test_404_url(self):
"""we return an error for a request including a 404 url"""
request = DownloadItem(self.build_server_address("does_not_exist"), None)
DownloadCenter([request], self.callback)
self.wait_for_callback(self.callback)
# no download means the file isn't in the result
callback_args, callback_kwargs = self.callback.call_args
result = callback_args[0][self.build_server_address("does_not_exist")]
self.assertIn("404", result.error)
self.assertIn("File not found", result.error)
self.assertIsNone(result.buffer)
self.assertIsNone(result.fd)
self.expect_warn_error = True
def prepare_to_download_archive(self, results):
"""Store the md5 for later and fire off the actual download."""
download_page = results[self.scraped_download_url]
checksum_page = results[self.scraped_checksum_url]
if download_page.error:
logger.error("Error fetching download page: %s",
download_page.error)
UI.return_main_screen(status_code=1)
if checksum_page.error:
logger.error("Error fetching checksums: %s", checksum_page.error)
UI.return_main_screen(status_code=1)
match = re.search(
r'^(\S+)\s+arduino-[\d\.\-r]+-linux' + self.bits + '.tar.xz$',
checksum_page.buffer.getvalue().decode('ascii'), re.M)
if not match:
logger.error("Can't find a checksum.")
UI.return_main_screen(status_code=1)
checksum = match.group(1)
soup = BeautifulSoup(download_page.buffer.getvalue(), 'html.parser')
btn = soup.find('button', text=re.compile('JUST DOWNLOAD'))
if not btn:
logger.error("Can't parse download button.")
UI.return_main_screen(status_code=1)
base_url = download_page.final_url
cookies = download_page.cookies
final_download_url = parse.urljoin(base_url, btn.parent['href'])
logger.info('Final download url: %s, cookies: %s.', final_download_url,
cookies)
self.download_requests = [
DownloadItem(final_download_url,
checksum=Checksum(ChecksumType.md5, checksum),
cookies=cookies)
]
# add the user to arduino group
if not self.was_in_arduino_group:
with futures.ProcessPoolExecutor(max_workers=1) as executor:
f = executor.submit(_add_to_group, self._current_user,
self.ARDUINO_GROUP)
if not f.result():
UI.return_main_screen(status_code=1)
self.start_download_and_install()
def test_download_with_wrong_sha1(self):
"""we raise an error if we don't have the correct md5sum"""
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter(
[DownloadItem(request, Checksum(ChecksumType.sha1, 'AAAAA'))],
self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertIn("Corrupted download", result.error)
self.assertIsNone(result.buffer)
self.assertIsNone(result.fd)
self.expect_warn_error = True
def get_sha_and_start_download(self, download_result):
res = download_result[self.sha512_url]
sha512 = res.buffer.getvalue().decode('utf-8').split()[0]
# you get and store self.download_url
url = re.sub('.sha512', '', self.sha512_url)
if url is None:
logger.error("Download page changed its syntax or is not parsable (missing url)")
UI.return_main_screen(status_code=1)
if sha512 is None:
logger.error("Download page changed its syntax or is not parsable (missing sha512)")
UI.return_main_screen(status_code=1)
logger.debug("Found download link for {}, checksum: {}".format(url, sha512))
self.download_requests.append(DownloadItem(url, Checksum(self.checksum_type, sha512)))
self.start_download_and_install()
def post_install(self):
"""Create the Eclipse launcher"""
DownloadCenter(urls=[DownloadItem(self.icon_url, None)],
on_done=self.save_icon, download=True)
icon_path = os.path.join(self.install_path, self.icon_filename)
comment = self.description
categories = "Development;IDE;"
create_launcher(self.desktop_filename,
get_application_desktop_file(name=self.name,
icon_path=icon_path,
try_exec=self.exec_path,
exec=self.exec_link_name,
comment=comment,
categories=categories))
def __init__(self, category):
super().__init__(
name="Twine",
description=_(
"Twine tool for creating interactive and nonlinear stories"),
category=category,
only_on_archs=['i386', 'amd64'],
download_page="http://twinery.org/",
dir_to_decompress_in_tarball='twine*',
desktop_filename="twine.desktop",
required_files_path=["Twine"])
# add logo download as the tar doesn't provide one
self.download_requests.append(
DownloadItem("http://twinery.org/img/logo.svg", None))
def get_metadata_and_check_license(self, result):
logger.debug("Fetched download page, parsing.")
page = result[self.download_page]
error_msg = page.error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(
self.download_page, error_msg))
UI.return_main_screen(status_code=1)
try:
key, content = json.loads(page.buffer.read().decode()).popitem()
download_url = content[0]['downloads']['linux']['link']
checksum_url = content[0]['downloads']['linux']['checksumLink']
except (IndexError):
if '&type=eap' in self.download_page:
logger.error("No EAP version available.")
else:
logger.error(
"Can't parse the download URL from the download page.")
UI.return_main_screen(status_code=1)
except (json.JSONDecodeError):
logger.error(
"Can't parse the download URL from the download page.")
UI.return_main_screen(status_code=1)
logger.debug("Found download URL: " + download_url)
logger.debug("Downloading checksum first, from " + checksum_url)
def checksum_downloaded(results):
checksum_result = next(iter(
results.values())) # Just get the first.
if checksum_result.error:
logger.error(checksum_result.error)
UI.return_main_screen(status_code=1)
checksum = checksum_result.buffer.getvalue().decode(
'utf-8').split()[0]
logger.info('Obtained SHA256 checksum: ' + checksum)
self.download_requests.append(
DownloadItem(download_url,
checksum=Checksum(ChecksumType.sha256, checksum),
ignore_encoding=True))
self.start_download_and_install()
DownloadCenter([DownloadItem(checksum_url)],
on_done=checksum_downloaded,
download=False)
def test_multiple_downloads_with_reports(self):
"""we deliver more than on download in parallel"""
requests = [
DownloadItem(self.build_server_address("biggerfile"), None),
DownloadItem(self.build_server_address("simplefile"), None)
]
report = CopyingMock()
DownloadCenter(requests, self.callback, report=report)
self.wait_for_callback(self.callback)
self.assertEqual(report.call_count, 5)
# ensure that first call only contains one file
callback_args, callback_kwargs = report.call_args_list[0]
map_result = callback_args[0]
self.assertEqual(len(map_result), 1, str(map_result))
# ensure that last call is what we expect
result_dict = {}
for filename in ("biggerfile", "simplefile"):
file_size = getsize(join(self.server_dir, filename))
result_dict[self.build_server_address(filename)] = {
'size': file_size,
'current': file_size
}
self.assertEqual(report.call_args, call(result_dict))
def test_in_memory_download(self):
"""we deliver download on memory objects"""
filename = "simplefile"
url = self.build_server_address(filename)
request = DownloadItem(url, None)
DownloadCenter([request], self.callback, download=False)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(), result.buffer.read())
self.assertIsNone(result.fd)
self.assertIsNone(result.error)
def parse_download_link(self, line, in_download):
"""Parse Eclipse download links"""
url_found = False
if self.download_keyword in line and self.bits in line:
in_download = True
else:
in_download = False
if in_download:
p = re.search(r'href="(.*)" title', line)
with suppress(AttributeError):
self.sha512_url = self.download_page + p.group(1) + '.sha512&r=1'
url_found = True
DownloadCenter(urls=[DownloadItem(self.sha512_url, None)],
on_done=self.get_sha_and_start_download, download=False)
return (url_found, in_download)
def parse_download_link(self, line, in_download):
"""Parse STS download links"""
url_found = False
if 'linux-gtk{}.tar.gz'.format(self.arch) in line:
in_download = True
else:
in_download = False
if in_download:
p = re.search(r'href="(.*.tar.gz)"', line)
with suppress(AttributeError):
self.checksum_url = p.group(1) + '.sha1'
url_found = True
DownloadCenter(urls=[DownloadItem(self.checksum_url, None)],
on_done=self.get_sha_and_start_download, download=False)
return (url_found, in_download)
def test_download_with_wrong_checksumtype(self):
"""we raise an error if we don't have a support checksum type"""
class WrongChecksumType(Enum):
didrocksha = "didrocksha"
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request, Checksum(WrongChecksumType.didrocksha, 'AAAAA'))], self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertIn("Unsupported checksum type", result.error)
self.assertIsNone(result.buffer)
self.assertIsNone(result.fd)
self.expect_warn_error = True
def test_cookies(self):
"""Test handing of outgoing and incoming cookies."""
filename = "simplefile"
url = self.build_server_address(filename)
# The server is rigged to inc the 'int' cookie by one.
cookies = {'int': '5'}
request = DownloadItem(url, None, cookies=cookies)
DownloadCenter([request], self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
self.assertEqual('6', result.cookies['int'])
def test_download_with_multiple_progress(self):
"""we deliver multiple progress hooks on bigger files"""
filename = "biggerfile"
filesize = getsize(join(self.server_dir, filename))
report = CopyingMock()
request = DownloadItem(self.build_server_address(filename), None)
dl_center = DownloadCenter([request], self.callback, report=report)
self.wait_for_callback(self.callback)
self.assertEqual(report.call_count, 3)
self.assertEqual(report.call_args_list,
[call({self.build_server_address(filename): {'size': filesize, 'current': 0}}),
call({self.build_server_address(filename): {'size': filesize,
'current': dl_center.BLOCK_SIZE}}),
call({self.build_server_address(filename): {'size': filesize, 'current': filesize}})])
def parse_download_link(self, line, in_download):
"""Parse Maven download link, expect to find a url"""
url_found = False
if '-bin.tar.gz.md5' in line:
in_download = True
else:
in_download = False
if in_download:
p = re.search(r'href="(.*)"', line)
with suppress(AttributeError):
self.checksum_url = p.group(1)
url_found = True
DownloadCenter(urls=[DownloadItem(self.checksum_url, None)],
on_done=self.get_sha_and_start_download, download=False)
return (url_found, in_download)
def get_metadata_and_check_license(self, result):
"""Override this so we can use BS and fetch the checksum separately."""
logger.debug("Fetched download page, parsing.")
page = result[self.download_page]
error_msg = page.error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(
self.download_page_url, error_msg))
UI.return_main_screen(status_code=1)
soup = BeautifulSoup(page.buffer, 'html.parser')
link = (soup.find('div', class_="install").find(
'td', class_="inst-type", text="Linux (.tar.gz)").parent.find(
text=self.arch_trans[get_current_arch()]).parent.parent)
if link is None:
logger.error(
"Can't parse the download URL from the download page.")
UI.return_main_screen(status_code=1)
download_url = link.attrs['href']
checksum_url = download_url + '.sha256'
logger.debug("Found download URL: " + download_url)
logger.debug("Downloading checksum first, from " + checksum_url)
def checksum_downloaded(results):
checksum_result = next(iter(
results.values())) # Just get the first.
if checksum_result.error:
logger.error(checksum_result.error)
UI.return_main_screen(status_code=1)
checksum = checksum_result.buffer.getvalue().decode(
'utf-8').split()[0]
logger.info('Obtained SHA256 checksum: ' + checksum)
self.download_requests.append(
DownloadItem(download_url,
checksum=Checksum(ChecksumType.sha256, checksum),
ignore_encoding=True))
self.start_download_and_install()
DownloadCenter([DownloadItem(checksum_url)],
on_done=checksum_downloaded,
download=False)
def test_redirect_download(self):
"""we deliver one successful download after being redirected"""
filename = "simplefile"
# We add a suffix to make the server redirect us.
url = self.build_server_address(filename + "-redirect")
request = DownloadItem(url, None)
DownloadCenter([request], self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(), result.fd.read())
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
def test_download_with_sha1sum(self):
"""we deliver once successful download, matching sha1sum"""
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request, Checksum(ChecksumType.sha1, '0562f08aef399135936d6fb4eb0cc7bc1890d5b4'))],
self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(),
result.fd.read())
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
def test_download_with_md5(self):
"""we deliver once successful download, matching md5sum"""
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request, Checksum(ChecksumType.md5, '268a5059001855fef30b4f95f82044ed'))],
self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(),
result.fd.read())
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
def post_install(self):
"""Create the Godot launcher"""
# Override the exec_path.
# Rename the binary to remove the version.
self.set_exec_path()
shutil.move(self.exec_path, os.path.join(self.install_path, 'godot'))
self.exec_path = os.path.join(self.install_path, 'godot')
DownloadCenter(urls=[DownloadItem(self.icon_url, None)],
on_done=self.save_icon, download=True)
create_launcher(self.desktop_filename, get_application_desktop_file(name=_("Godot"),
icon_path=os.path.join(self.install_path, self.icon_filename),
try_exec=self.exec_path,
exec=self.exec_link_name,
comment=self.description,
categories="Development;IDE;"))
def checksum_downloaded(results):
checksum_result = next(iter(
results.values())) # Just get the first.
if checksum_result.error:
logger.error(checksum_result.error)
UI.return_main_screen(status_code=1)
checksum = checksum_result.buffer.getvalue().decode(
'utf-8').split()[0]
logger.info('Obtained SHA256 checksum: ' + checksum)
self.download_requests.append(
DownloadItem(download_url,
checksum=Checksum(ChecksumType.sha256, checksum),
ignore_encoding=True))
self.start_download_and_install()
def test_download(self):
"""we deliver one successful download"""
filename = "simplefile"
url = self.build_server_address(filename)
request = DownloadItem(url, None)
DownloadCenter([request], self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(), result.fd.read())
self.assertTrue('.' not in result.fd.name, result.fd.name)
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
def parse_download_page_callback(self, result):
"""Get the download_url and trigger the download and installation of the app.
:param result: the file downloaded by DownloadCenter, contains js functions with download urls
"""
logger.info("Netbeans {}".format(self.version))
# Processing the string to obtain metadata (download url)
try:
url_file = result[self.version_download_page].buffer.read().decode(
'utf-8')
except AttributeError:
# The file could not be parsed
logger.error(
"The download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
preg = re.compile(
'add_file\("zip/netbeans-{}-[0-9]{{12}}{}.zip"'.format(
self.version, self.flavour))
for line in url_file.split("\n"):
if preg.match(line):
# Clean up the string from js (it's a function call)
line = line.replace("add_file(",
"").replace(");", "").replace('"', "")
url_string = line
if not url_string:
# The file could not be parsed
logger.error(
"The download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
string_array = url_string.split(", ")
try:
url_suffix = string_array[0]
sha256 = string_array[2]
except IndexError:
# The file could not be parsed
logger.error(
"The download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
download_url = "{}/{}/final/{}".format(self.BASE_URL, self.version,
url_suffix)
self.download_requests.append(
DownloadItem(download_url, Checksum(ChecksumType.sha256, sha256)))
self.start_download_and_install()
def get_metadata_and_check_license(self, result):
"""Download files to download + license and check it"""
logger.debug("Parse download metadata")
error_msg = result[self.download_page].error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(self.download_page, error_msg))
UI.return_main_screen()
url, checksum = (None, None)
with StringIO() as license_txt:
in_license = False
in_download = False
for line in result[self.download_page].buffer:
line_content = line.decode()
if self.expect_license and not self.auto_accept_license:
in_license = self.parse_license(line_content, license_txt, in_license)
(download, in_download) = self.parse_download_link(line_content, in_download)
if download is not None:
(newurl, new_checksum) = download
url = newurl if newurl is not None else url
checksum = new_checksum if new_checksum is not None else checksum
if url is not None:
if self.checksum_type and checksum:
logger.debug("Found download link for {}, checksum: {}".format(url, checksum))
break
elif not self.checksum_type:
logger.debug("Found download link for {}".format(url))
break
if url is None or (self.checksum_type and checksum is None):
logger.error("Download page changed its syntax or is not parsable")
UI.return_main_screen()
self.download_requests.append(DownloadItem(url, Checksum(self.checksum_type, checksum)))
if license_txt.getvalue() != "":
logger.debug("Check license agreement.")
UI.display(LicenseAgreement(strip_tags(license_txt.getvalue()).strip(),
self.start_download_and_install,
UI.return_main_screen))
elif self.expect_license and not self.auto_accept_license:
logger.error("We were expecting to find a license on the download page, we didn't.")
UI.return_main_screen()
else:
self.start_download_and_install()
def parse_download_link(self, line, in_download):
"""Parse STS download link"""
url_found = False
in_download = False
keyword = 'linux-gtk{}.tar.gz'.format(self.arch)
if keyword in line:
in_download = True
if in_download:
regexp = r'href="(.*.tar.gz)"'
p = re.search(regexp, line)
with suppress(AttributeError):
url_found = True
self.sha1_url = '{}.sha1'.format(p.group(1))
DownloadCenter(urls=[DownloadItem(self.sha1_url, None)],
on_done=self.get_sha_and_start_download, download=False)
return (url_found, in_download)
def parse_download_link(self, line, in_download):
"""Parse Eclipse download links"""
if self.download_keyword in line and self.bits in line and 'linux' in line:
in_download = True
else:
in_download = False
if in_download:
p = re.search(r"href='(https:)?\/\/www.eclipse.org(.*)'", line)
logger.debug(p.group(2))
with suppress(AttributeError):
self.sha512_url = "https://www.eclipse.org" + p.group(
2) + '.sha512&r=1'
url_found = True
DownloadCenter(urls=[DownloadItem(self.sha512_url, None)],
on_done=self.get_sha_and_start_download,
download=False)
return (url_found, in_download)
def parse_download_link(self, line, in_download):
"""Parse Blender download links"""
url = None
if '.tar.bz2' in line:
p = re.search(
r'href="(http://download.blender.org/[^<]*{}.tar.bz2)"'.format(
self.arch), line)
with suppress(AttributeError):
url = p.group(1)
filename = 'release' + re.search('blender-(.*)-linux',
url).group(1).replace(
'.', '') + '.md5'
self.checksum_url = os.path.join(os.path.dirname(url),
filename)
DownloadCenter(urls=[DownloadItem(self.checksum_url, None)],
on_done=self.get_checksum_and_start_download,
download=False)
return (url, in_download)
def test_download_with_sha256sum(self):
"""we deliver once successful download, matching sha256sum"""
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request,
Checksum(ChecksumType.sha256,
'b1b113c6ed8ab3a14779f7c54179eac2b87d39fcebbf65a50556b8d68caaa2fb'))],
self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(),
result.fd.read())
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
def test_with_invalid_certificate(self):
"""we error on invalid ssl certificate"""
filename = "simplefile"
url = TestDownloadCenter.build_server_address(self, filename)
request = DownloadItem(url, None)
DownloadCenter([request], self.callback)
TestDownloadCenter.wait_for_callback(self, self.callback)
result = self.callback.call_args[0][0][url]
# disco python changed the error type
found = False
for invalidError in ["CERTIFICATE_VERIFY_FAILED", "bad handshake"]:
if invalidError in result.error:
found = True
self.assertTrue(found)
self.assertIsNone(result.buffer)
self.assertIsNone(result.fd)
self.expect_warn_error = True
def get_url_and_start_download(self, download_result):
res = download_result[self.download_url]
text = res.buffer.getvalue().decode('utf-8')
url = re.search(r'http.*?.sh', text).group(0)
if url is None:
logger.error(
"Download page changed its syntax or is not parsable (missing url)"
)
UI.return_main_screen(status_code=1)
if self.checksum is None:
logger.error(
"Download page changed its syntax or is not parsable (missing checksum)"
)
UI.return_main_screen(status_code=1)
logger.debug("Found download link for {}, checksum: {}".format(
url, self.checksum))
self.download_requests.append(
DownloadItem(url, Checksum(self.checksum_type, self.checksum)))
self.start_download_and_install()
