def get_metadata(self, result):
"""Download files to download + and download license license and check it"""
logger.debug("Parse download metadata")
error_msg = result[self.download_page].error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(self.download_page, error_msg))
UI.return_main_screen(status_code=1)
arch = platform.machine()
download_re = r'\'linux64\': \'([^\']+)\''
if arch == 'i686':
download_re = r'\'linux32\': \'([^\']+)\''
url = None
for line in result[self.download_page].buffer:
line = line.decode()
p = re.search(download_re, line)
with suppress(AttributeError):
url = p.group(1)
logger.debug("Found download link for {}".format(url))
if url is None:
logger.error("Download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
self.download_requests.append(DownloadItem(url, Checksum(self.checksum_type, None), headers=self.headers))
if not self.auto_accept_license:
logger.debug("Downloading License page")
DownloadCenter([DownloadItem(self.license_url, headers=self.headers)], self.check_external_license,
download=False)
else:
self.start_download_and_install()
def get_metadata_and_check_license(self, result):
"""Download files to download + license and check it"""
logger.debug("Parse download metadata")
error_msg = result[self.download_page].error
if error_msg:
logger.error("An error occurred while downloading {}: {}".format(
self.download_page, error_msg))
UI.return_main_screen()
url, checksum = (None, None)
with StringIO() as license_txt:
in_license = False
in_download = False
for line in result[self.download_page].buffer:
line_content = line.decode()
if self.expect_license:
in_license = self.parse_license(line_content, license_txt,
in_license)
(download, in_download) = self.parse_download_link(
line_content, in_download)
if download is not None:
(newurl, new_checksum) = download
url = newurl if newurl is not None else url
checksum = new_checksum if new_checksum is not None else checksum
if url is not None:
if self.checksum_type and checksum:
logger.debug(
"Found download link for {}, checksum: {}".
format(url, checksum))
break
elif not self.checksum_type:
logger.debug(
"Found download link for {}".format(url))
break
if url is None or (self.checksum_type and checksum is None):
logger.error(
"Download page changed its syntax or is not parsable")
UI.return_main_screen()
self.download_requests.append(
DownloadItem(url, Checksum(self.checksum_type, checksum)))
if license_txt.getvalue() != "":
logger.debug("Check license agreement.")
UI.display(
LicenseAgreement(
strip_tags(license_txt.getvalue()).strip(),
self.start_download_and_install,
UI.return_main_screen))
elif self.expect_license:
logger.error(
"We were expecting to find a license on the download page, we didn't."
)
UI.return_main_screen()
else:
self.start_download_and_install()
return
def get_sha_and_start_download(self, download_result):
res = download_result[self.checksum_url].buffer.getvalue().decode()
line = re.search(
r'.*linux{}.tar.xz'.format(self.arch_trans[get_current_arch()]),
res).group(0)
# you get and store url and checksum
checksum = line.split()[0]
url = os.path.join(
self.checksum_url.rpartition('/')[0],
line.split()[1])
if url is None:
logger.error(
"Download page changed its syntax or is not parsable (missing url)"
)
UI.return_main_screen(status_code=1)
if checksum is None:
logger.error(
"Download page changed its syntax or is not parsable (missing sha512)"
)
UI.return_main_screen(status_code=1)
logger.debug("Found download link for {}, checksum: {}".format(
url, checksum))
self.download_requests.append(
DownloadItem(url, Checksum(self.checksum_type, checksum)))
# add the user to arduino group
if not self.was_in_arduino_group:
with futures.ProcessPoolExecutor(max_workers=1) as executor:
f = executor.submit(_add_to_group, self._current_user,
self.ARDUINO_GROUP)
if not f.result():
UI.return_main_screen(status_code=1)
self.start_download_and_install()
def check_data_and_start_download(self,
url=None,
checksum=None,
license_txt=StringIO()):
if url is None:
logger.error(
"Download page changed its syntax or is not parsable (url missing)"
)
UI.return_main_screen(status_code=1)
if (self.checksum_type and checksum is None):
logger.error(
"Download page changed its syntax or is not parsable (checksum missing)"
)
logger.error("URL is: {}".format(url))
UI.return_main_screen(status_code=1)
self.download_requests.append(
DownloadItem(url, Checksum(self.checksum_type, checksum)))
if license_txt.getvalue() != "":
logger.debug("Check license agreement.")
UI.display(
LicenseAgreement(
strip_tags(license_txt.getvalue()).strip(),
self.start_download_and_install, UI.return_main_screen))
elif self.expect_license and not self.auto_accept_license:
logger.error(
"We were expecting to find a license on the download page, we didn't."
)
UI.return_main_screen(status_code=1)
else:
self.start_download_and_install()
def get_checksum_and_start_download(self, download_result):
for line in download_result[self.checksum_url].buffer:
line = line.decode()
if '{}.tar.bz2'.format(
self.arch_trans[get_current_arch()]) in line:
break
checksum = line.split()[0]
self.download_requests.append(
DownloadItem(self.url, Checksum(self.checksum_type, checksum)))
self.start_download_and_install()
def test_download_with_wrong_sha256(self):
"""we raise an error if we don't have the correct sha256"""
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request, Checksum(ChecksumType.sha256, 'AAAAA'))], self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertIn("Corrupted download", result.error)
self.assertIsNone(result.buffer)
self.assertIsNone(result.fd)
self.expect_warn_error = True
def download_provider_page(self):
"""Skip download provider page and directly use the download links"""
arch = platform.machine()
tag_machine = ''
if arch == 'x86_64':
tag_machine = '64'
self.download_requests.append(
DownloadItem(
"https://download.mozilla.org/?product=firefox-aurora-latest&os=linux{}"
.format(tag_machine), Checksum(None, None)))
self.start_download_and_install()
def checksum_downloaded(results):
checksum_result = next(iter(results.values())) # Just get the first.
if checksum_result.error:
logger.error(checksum_result.error)
UI.return_main_screen(status_code=1)
checksum = checksum_result.buffer.getvalue().decode('utf-8').split()[0]
logger.info('Obtained SHA256 checksum: ' + checksum)
self.download_requests.append(DownloadItem(download_url,
checksum=Checksum(ChecksumType.sha256, checksum),
ignore_encoding=True))
self.start_download_and_install()
def get_url_and_start_download(self, download_result):
res = download_result[self.download_url]
text = res.buffer.getvalue().decode('utf-8')
url = re.search(r'http.*?.sh', text).group(0)
if url is None:
logger.error("Download page changed its syntax or is not parsable (missing url)")
UI.return_main_screen(status_code=1)
if self.checksum is None:
logger.error("Download page changed its syntax or is not parsable (missing checksum)")
UI.return_main_screen(status_code=1)
logger.debug("Found download link for {}, checksum: {}".format(url, self.checksum))
self.download_requests.append(DownloadItem(url, Checksum(self.checksum_type, self.checksum)))
self.start_download_and_install()
def get_sha_and_start_download(self, download_result):
res = download_result[self.sha1_url]
sha1 = res.buffer.getvalue().decode('utf-8').split()[0]
url = re.sub('.sha1', '', self.sha1_url)
if url is None:
logger.error("Download page changed its syntax or is not parsable (missing url)")
UI.return_main_screen(status_code=1)
if sha1 is None:
logger.error("Download page changed its syntax or is not parsable (missing sha512)")
UI.return_main_screen(status_code=1)
logger.debug("Found download link for {}, checksum: {}".format(url, sha1))
self.download_requests.append(DownloadItem(url, Checksum(ChecksumType.sha1, sha1)))
self.start_download_and_install()
def prepare_to_download_archive(self, results):
"""Store the md5 for later and fire off the actual download."""
download_page = results[self.scraped_download_url]
checksum_page = results[self.scraped_checksum_url]
if download_page.error:
logger.error("Error fetching download page: %s",
download_page.error)
UI.return_main_screen(status_code=1)
if checksum_page.error:
logger.error("Error fetching checksums: %s", checksum_page.error)
UI.return_main_screen(status_code=1)
match = re.search(
r'^(\S+)\s+arduino-[\d\.\-r]+-linux' + self.bits + '.tar.xz$',
checksum_page.buffer.getvalue().decode('ascii'), re.M)
if not match:
logger.error("Can't find a checksum.")
UI.return_main_screen(status_code=1)
checksum = match.group(1)
soup = BeautifulSoup(download_page.buffer.getvalue(), 'html.parser')
btn = soup.find('button', text=re.compile('JUST DOWNLOAD'))
if not btn:
logger.error("Can't parse download button.")
UI.return_main_screen(status_code=1)
base_url = download_page.final_url
cookies = download_page.cookies
final_download_url = parse.urljoin(base_url, btn.parent['href'])
logger.info('Final download url: %s, cookies: %s.', final_download_url,
cookies)
self.download_requests = [
DownloadItem(final_download_url,
checksum=Checksum(ChecksumType.md5, checksum),
cookies=cookies)
]
# add the user to arduino group
if not self.was_in_arduino_group:
with futures.ProcessPoolExecutor(max_workers=1) as executor:
f = executor.submit(_add_to_group, self._current_user,
self.ARDUINO_GROUP)
if not f.result():
UI.return_main_screen(status_code=1)
self.start_download_and_install()
def test_download_with_wrong_checksumtype(self):
"""we raise an error if we don't have a support checksum type"""
class WrongChecksumType(Enum):
didrocksha = "didrocksha"
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request, Checksum(WrongChecksumType.didrocksha, 'AAAAA'))], self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertIn("Unsupported checksum type", result.error)
self.assertIsNone(result.buffer)
self.assertIsNone(result.fd)
self.expect_warn_error = True
def test_download_with_no_checksum_value(self):
"""we deliver one successful download with a checksum type having no value"""
filename = "simplefile"
url = self.build_server_address(filename)
request = DownloadItem(url, Checksum(ChecksumType.md5, None))
DownloadCenter([request], self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][url]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(), result.fd.read())
self.assertTrue('.' not in result.fd.name, result.fd.name)
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
def test_download_with_sha1sum(self):
"""we deliver once successful download, matching sha1sum"""
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request, Checksum(ChecksumType.sha1, '0562f08aef399135936d6fb4eb0cc7bc1890d5b4'))],
self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(),
result.fd.read())
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
def test_download_with_md5(self):
"""we deliver once successful download, matching md5sum"""
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request, Checksum(ChecksumType.md5, '268a5059001855fef30b4f95f82044ed'))],
self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(),
result.fd.read())
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
def parse_download_page_callback(self, result):
"""Get the download_url and trigger the download and installation of the app.
:param result: the file downloaded by DownloadCenter, contains js functions with download urls
"""
logger.info("Netbeans {}".format(self.version))
# Processing the string to obtain metadata (download url)
try:
url_file = result[self.version_download_page].buffer.read().decode(
'utf-8')
except AttributeError:
# The file could not be parsed
logger.error(
"The download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
preg = re.compile(
'add_file\("zip/netbeans-{}-[0-9]{{12}}{}.zip"'.format(
self.version, self.flavour))
for line in url_file.split("\n"):
if preg.match(line):
# Clean up the string from js (it's a function call)
line = line.replace("add_file(",
"").replace(");", "").replace('"', "")
url_string = line
if not url_string:
# The file could not be parsed
logger.error(
"The download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
string_array = url_string.split(", ")
try:
url_suffix = string_array[0]
sha256 = string_array[2]
except IndexError:
# The file could not be parsed
logger.error(
"The download page changed its syntax or is not parsable")
UI.return_main_screen(status_code=1)
download_url = "{}/{}/final/{}".format(self.BASE_URL, self.version,
url_suffix)
self.download_requests.append(
DownloadItem(download_url, Checksum(ChecksumType.sha256, sha256)))
self.start_download_and_install()
def test_download_with_sha256sum(self):
"""we deliver once successful download, matching sha256sum"""
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request,
Checksum(ChecksumType.sha256,
'b1b113c6ed8ab3a14779f7c54179eac2b87d39fcebbf65a50556b8d68caaa2fb'))],
self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(),
result.fd.read())
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
def done(download_result):
res = download_result[md5_url]
if res.error:
logger.error(res.error)
UI.return_main_screen()
# Should be ASCII anyway.
md5 = res.buffer.getvalue().decode('utf-8').split()[0]
logger.debug("Downloaded MD5 is {}".format(md5))
logger.debug("Preparing to download the main archive.")
if arch == 'i686':
download_url = self.DOWNLOAD_URL_PAT.format(arch='', suf='')
elif arch == 'x86_64':
download_url = self.DOWNLOAD_URL_PAT.format(arch='-x86_64',
suf='')
self.download_requests.append(DownloadItem(download_url, Checksum(ChecksumType.md5, md5)))
self.start_download_and_install()
def test_download_with_sha512sum(self):
"""we deliver once successful download, matching sha512sum"""
filename = "simplefile"
request = self.build_server_address(filename)
DownloadCenter([DownloadItem(request,
Checksum(ChecksumType.sha512,
'74e20d520ba4ecfdb59d98ac213deccecf591c9c6bfc5996ac158ab6facd6611cce7dd2'
'2120b63ebe9217f159506f352ce0ee6c0c2a1d200841ae21635dc5f9a'))],
self.callback)
self.wait_for_callback(self.callback)
result = self.callback.call_args[0][0][request]
self.assertTrue(self.callback.called)
self.assertEqual(self.callback.call_count, 1)
with open(join(self.server_dir, filename), 'rb') as file_on_disk:
self.assertEqual(file_on_disk.read(),
result.fd.read())
self.assertIsNone(result.buffer)
self.assertIsNone(result.error)
def get_sha_and_start_download(self, download_result):
res = download_result[self.checksum_url]
checksum = res.buffer.getvalue().decode('utf-8').split()[0]
# you get and store self.download_url
url = re.sub('.' + self.checksum_type.name, '', self.checksum_url)
if url is None:
logger.error(
"Download page changed its syntax or is not parsable (missing url)"
)
UI.return_main_screen(status_code=1)
if checksum is None:
logger.error(
"Download page changed its syntax or is not parsable (missing checksum)"
)
UI.return_main_screen(status_code=1)
logger.debug("Found download link for {}, checksum: {}".format(
url, checksum))
self.download_requests.append(
DownloadItem(url, Checksum(self.checksum_type, checksum)))
self.start_download_and_install()