def test_404_url(self): """we return an error for a request including a 404 url""" request = DownloadItem(self.build_server_address("does_not_exist"), None) DownloadCenter([request], self.callback) self.wait_for_callback(self.callback) # no download means the file isn't in the result callback_args, callback_kwargs = self.callback.call_args result = callback_args[0][self.build_server_address("does_not_exist")] self.assertIn("404", result.error) self.assertIn("File not found", result.error) self.assertIsNone(result.buffer) self.assertIsNone(result.fd) self.expect_warn_error = True
def prepare_to_download_archive(self, results): """Store the md5 for later and fire off the actual download.""" download_page = results[self.scraped_download_url] checksum_page = results[self.scraped_checksum_url] if download_page.error: logger.error("Error fetching download page: %s", download_page.error) UI.return_main_screen(status_code=1) if checksum_page.error: logger.error("Error fetching checksums: %s", checksum_page.error) UI.return_main_screen(status_code=1) match = re.search( r'^(\S+)\s+arduino-[\d\.\-r]+-linux' + self.bits + '.tar.xz$', checksum_page.buffer.getvalue().decode('ascii'), re.M) if not match: logger.error("Can't find a checksum.") UI.return_main_screen(status_code=1) checksum = match.group(1) soup = BeautifulSoup(download_page.buffer.getvalue(), 'html.parser') btn = soup.find('button', text=re.compile('JUST DOWNLOAD')) if not btn: logger.error("Can't parse download button.") UI.return_main_screen(status_code=1) base_url = download_page.final_url cookies = download_page.cookies final_download_url = parse.urljoin(base_url, btn.parent['href']) logger.info('Final download url: %s, cookies: %s.', final_download_url, cookies) self.download_requests = [ DownloadItem(final_download_url, checksum=Checksum(ChecksumType.md5, checksum), cookies=cookies) ] # add the user to arduino group if not self.was_in_arduino_group: with futures.ProcessPoolExecutor(max_workers=1) as executor: f = executor.submit(_add_to_group, self._current_user, self.ARDUINO_GROUP) if not f.result(): UI.return_main_screen(status_code=1) self.start_download_and_install()
def test_download_with_wrong_sha1(self): """we raise an error if we don't have the correct md5sum""" filename = "simplefile" request = self.build_server_address(filename) DownloadCenter( [DownloadItem(request, Checksum(ChecksumType.sha1, 'AAAAA'))], self.callback) self.wait_for_callback(self.callback) result = self.callback.call_args[0][0][request] self.assertIn("Corrupted download", result.error) self.assertIsNone(result.buffer) self.assertIsNone(result.fd) self.expect_warn_error = True
def get_sha_and_start_download(self, download_result): res = download_result[self.sha512_url] sha512 = res.buffer.getvalue().decode('utf-8').split()[0] # you get and store self.download_url url = re.sub('.sha512', '', self.sha512_url) if url is None: logger.error("Download page changed its syntax or is not parsable (missing url)") UI.return_main_screen(status_code=1) if sha512 is None: logger.error("Download page changed its syntax or is not parsable (missing sha512)") UI.return_main_screen(status_code=1) logger.debug("Found download link for {}, checksum: {}".format(url, sha512)) self.download_requests.append(DownloadItem(url, Checksum(self.checksum_type, sha512))) self.start_download_and_install()
def post_install(self): """Create the Eclipse launcher""" DownloadCenter(urls=[DownloadItem(self.icon_url, None)], on_done=self.save_icon, download=True) icon_path = os.path.join(self.install_path, self.icon_filename) comment = self.description categories = "Development;IDE;" create_launcher(self.desktop_filename, get_application_desktop_file(name=self.name, icon_path=icon_path, try_exec=self.exec_path, exec=self.exec_link_name, comment=comment, categories=categories))
def __init__(self, category): super().__init__( name="Twine", description=_( "Twine tool for creating interactive and nonlinear stories"), category=category, only_on_archs=['i386', 'amd64'], download_page="http://twinery.org/", dir_to_decompress_in_tarball='twine*', desktop_filename="twine.desktop", required_files_path=["Twine"]) # add logo download as the tar doesn't provide one self.download_requests.append( DownloadItem("http://twinery.org/img/logo.svg", None))
def get_metadata_and_check_license(self, result): logger.debug("Fetched download page, parsing.") page = result[self.download_page] error_msg = page.error if error_msg: logger.error("An error occurred while downloading {}: {}".format( self.download_page, error_msg)) UI.return_main_screen(status_code=1) try: key, content = json.loads(page.buffer.read().decode()).popitem() download_url = content[0]['downloads']['linux']['link'] checksum_url = content[0]['downloads']['linux']['checksumLink'] except (IndexError): if '&type=eap' in self.download_page: logger.error("No EAP version available.") else: logger.error( "Can't parse the download URL from the download page.") UI.return_main_screen(status_code=1) except (json.JSONDecodeError): logger.error( "Can't parse the download URL from the download page.") UI.return_main_screen(status_code=1) logger.debug("Found download URL: " + download_url) logger.debug("Downloading checksum first, from " + checksum_url) def checksum_downloaded(results): checksum_result = next(iter( results.values())) # Just get the first. if checksum_result.error: logger.error(checksum_result.error) UI.return_main_screen(status_code=1) checksum = checksum_result.buffer.getvalue().decode( 'utf-8').split()[0] logger.info('Obtained SHA256 checksum: ' + checksum) self.download_requests.append( DownloadItem(download_url, checksum=Checksum(ChecksumType.sha256, checksum), ignore_encoding=True)) self.start_download_and_install() DownloadCenter([DownloadItem(checksum_url)], on_done=checksum_downloaded, download=False)
def test_multiple_downloads_with_reports(self): """we deliver more than on download in parallel""" requests = [ DownloadItem(self.build_server_address("biggerfile"), None), DownloadItem(self.build_server_address("simplefile"), None) ] report = CopyingMock() DownloadCenter(requests, self.callback, report=report) self.wait_for_callback(self.callback) self.assertEqual(report.call_count, 5) # ensure that first call only contains one file callback_args, callback_kwargs = report.call_args_list[0] map_result = callback_args[0] self.assertEqual(len(map_result), 1, str(map_result)) # ensure that last call is what we expect result_dict = {} for filename in ("biggerfile", "simplefile"): file_size = getsize(join(self.server_dir, filename)) result_dict[self.build_server_address(filename)] = { 'size': file_size, 'current': file_size } self.assertEqual(report.call_args, call(result_dict))
def test_in_memory_download(self): """we deliver download on memory objects""" filename = "simplefile" url = self.build_server_address(filename) request = DownloadItem(url, None) DownloadCenter([request], self.callback, download=False) self.wait_for_callback(self.callback) result = self.callback.call_args[0][0][url] self.assertTrue(self.callback.called) self.assertEqual(self.callback.call_count, 1) with open(join(self.server_dir, filename), 'rb') as file_on_disk: self.assertEqual(file_on_disk.read(), result.buffer.read()) self.assertIsNone(result.fd) self.assertIsNone(result.error)
def parse_download_link(self, line, in_download): """Parse Eclipse download links""" url_found = False if self.download_keyword in line and self.bits in line: in_download = True else: in_download = False if in_download: p = re.search(r'href="(.*)" title', line) with suppress(AttributeError): self.sha512_url = self.download_page + p.group(1) + '.sha512&r=1' url_found = True DownloadCenter(urls=[DownloadItem(self.sha512_url, None)], on_done=self.get_sha_and_start_download, download=False) return (url_found, in_download)
def parse_download_link(self, line, in_download): """Parse STS download links""" url_found = False if 'linux-gtk{}.tar.gz'.format(self.arch) in line: in_download = True else: in_download = False if in_download: p = re.search(r'href="(.*.tar.gz)"', line) with suppress(AttributeError): self.checksum_url = p.group(1) + '.sha1' url_found = True DownloadCenter(urls=[DownloadItem(self.checksum_url, None)], on_done=self.get_sha_and_start_download, download=False) return (url_found, in_download)
def test_download_with_wrong_checksumtype(self): """we raise an error if we don't have a support checksum type""" class WrongChecksumType(Enum): didrocksha = "didrocksha" filename = "simplefile" request = self.build_server_address(filename) DownloadCenter([DownloadItem(request, Checksum(WrongChecksumType.didrocksha, 'AAAAA'))], self.callback) self.wait_for_callback(self.callback) result = self.callback.call_args[0][0][request] self.assertIn("Unsupported checksum type", result.error) self.assertIsNone(result.buffer) self.assertIsNone(result.fd) self.expect_warn_error = True
def test_cookies(self): """Test handing of outgoing and incoming cookies.""" filename = "simplefile" url = self.build_server_address(filename) # The server is rigged to inc the 'int' cookie by one. cookies = {'int': '5'} request = DownloadItem(url, None, cookies=cookies) DownloadCenter([request], self.callback) self.wait_for_callback(self.callback) result = self.callback.call_args[0][0][url] self.assertTrue(self.callback.called) self.assertEqual(self.callback.call_count, 1) self.assertEqual('6', result.cookies['int'])
def test_download_with_multiple_progress(self): """we deliver multiple progress hooks on bigger files""" filename = "biggerfile" filesize = getsize(join(self.server_dir, filename)) report = CopyingMock() request = DownloadItem(self.build_server_address(filename), None) dl_center = DownloadCenter([request], self.callback, report=report) self.wait_for_callback(self.callback) self.assertEqual(report.call_count, 3) self.assertEqual(report.call_args_list, [call({self.build_server_address(filename): {'size': filesize, 'current': 0}}), call({self.build_server_address(filename): {'size': filesize, 'current': dl_center.BLOCK_SIZE}}), call({self.build_server_address(filename): {'size': filesize, 'current': filesize}})])
def parse_download_link(self, line, in_download): """Parse Maven download link, expect to find a url""" url_found = False if '-bin.tar.gz.md5' in line: in_download = True else: in_download = False if in_download: p = re.search(r'href="(.*)"', line) with suppress(AttributeError): self.checksum_url = p.group(1) url_found = True DownloadCenter(urls=[DownloadItem(self.checksum_url, None)], on_done=self.get_sha_and_start_download, download=False) return (url_found, in_download)
def get_metadata_and_check_license(self, result): """Override this so we can use BS and fetch the checksum separately.""" logger.debug("Fetched download page, parsing.") page = result[self.download_page] error_msg = page.error if error_msg: logger.error("An error occurred while downloading {}: {}".format( self.download_page_url, error_msg)) UI.return_main_screen(status_code=1) soup = BeautifulSoup(page.buffer, 'html.parser') link = (soup.find('div', class_="install").find( 'td', class_="inst-type", text="Linux (.tar.gz)").parent.find( text=self.arch_trans[get_current_arch()]).parent.parent) if link is None: logger.error( "Can't parse the download URL from the download page.") UI.return_main_screen(status_code=1) download_url = link.attrs['href'] checksum_url = download_url + '.sha256' logger.debug("Found download URL: " + download_url) logger.debug("Downloading checksum first, from " + checksum_url) def checksum_downloaded(results): checksum_result = next(iter( results.values())) # Just get the first. if checksum_result.error: logger.error(checksum_result.error) UI.return_main_screen(status_code=1) checksum = checksum_result.buffer.getvalue().decode( 'utf-8').split()[0] logger.info('Obtained SHA256 checksum: ' + checksum) self.download_requests.append( DownloadItem(download_url, checksum=Checksum(ChecksumType.sha256, checksum), ignore_encoding=True)) self.start_download_and_install() DownloadCenter([DownloadItem(checksum_url)], on_done=checksum_downloaded, download=False)
def test_redirect_download(self): """we deliver one successful download after being redirected""" filename = "simplefile" # We add a suffix to make the server redirect us. url = self.build_server_address(filename + "-redirect") request = DownloadItem(url, None) DownloadCenter([request], self.callback) self.wait_for_callback(self.callback) result = self.callback.call_args[0][0][url] self.assertTrue(self.callback.called) self.assertEqual(self.callback.call_count, 1) with open(join(self.server_dir, filename), 'rb') as file_on_disk: self.assertEqual(file_on_disk.read(), result.fd.read()) self.assertIsNone(result.buffer) self.assertIsNone(result.error)
def test_download_with_sha1sum(self): """we deliver once successful download, matching sha1sum""" filename = "simplefile" request = self.build_server_address(filename) DownloadCenter([DownloadItem(request, Checksum(ChecksumType.sha1, '0562f08aef399135936d6fb4eb0cc7bc1890d5b4'))], self.callback) self.wait_for_callback(self.callback) result = self.callback.call_args[0][0][request] self.assertTrue(self.callback.called) self.assertEqual(self.callback.call_count, 1) with open(join(self.server_dir, filename), 'rb') as file_on_disk: self.assertEqual(file_on_disk.read(), result.fd.read()) self.assertIsNone(result.buffer) self.assertIsNone(result.error)
def test_download_with_md5(self): """we deliver once successful download, matching md5sum""" filename = "simplefile" request = self.build_server_address(filename) DownloadCenter([DownloadItem(request, Checksum(ChecksumType.md5, '268a5059001855fef30b4f95f82044ed'))], self.callback) self.wait_for_callback(self.callback) result = self.callback.call_args[0][0][request] self.assertTrue(self.callback.called) self.assertEqual(self.callback.call_count, 1) with open(join(self.server_dir, filename), 'rb') as file_on_disk: self.assertEqual(file_on_disk.read(), result.fd.read()) self.assertIsNone(result.buffer) self.assertIsNone(result.error)
def post_install(self): """Create the Godot launcher""" # Override the exec_path. # Rename the binary to remove the version. self.set_exec_path() shutil.move(self.exec_path, os.path.join(self.install_path, 'godot')) self.exec_path = os.path.join(self.install_path, 'godot') DownloadCenter(urls=[DownloadItem(self.icon_url, None)], on_done=self.save_icon, download=True) create_launcher(self.desktop_filename, get_application_desktop_file(name=_("Godot"), icon_path=os.path.join(self.install_path, self.icon_filename), try_exec=self.exec_path, exec=self.exec_link_name, comment=self.description, categories="Development;IDE;"))
def checksum_downloaded(results): checksum_result = next(iter( results.values())) # Just get the first. if checksum_result.error: logger.error(checksum_result.error) UI.return_main_screen(status_code=1) checksum = checksum_result.buffer.getvalue().decode( 'utf-8').split()[0] logger.info('Obtained SHA256 checksum: ' + checksum) self.download_requests.append( DownloadItem(download_url, checksum=Checksum(ChecksumType.sha256, checksum), ignore_encoding=True)) self.start_download_and_install()
def test_download(self): """we deliver one successful download""" filename = "simplefile" url = self.build_server_address(filename) request = DownloadItem(url, None) DownloadCenter([request], self.callback) self.wait_for_callback(self.callback) result = self.callback.call_args[0][0][url] self.assertTrue(self.callback.called) self.assertEqual(self.callback.call_count, 1) with open(join(self.server_dir, filename), 'rb') as file_on_disk: self.assertEqual(file_on_disk.read(), result.fd.read()) self.assertTrue('.' not in result.fd.name, result.fd.name) self.assertIsNone(result.buffer) self.assertIsNone(result.error)
def parse_download_page_callback(self, result): """Get the download_url and trigger the download and installation of the app. :param result: the file downloaded by DownloadCenter, contains js functions with download urls """ logger.info("Netbeans {}".format(self.version)) # Processing the string to obtain metadata (download url) try: url_file = result[self.version_download_page].buffer.read().decode( 'utf-8') except AttributeError: # The file could not be parsed logger.error( "The download page changed its syntax or is not parsable") UI.return_main_screen(status_code=1) preg = re.compile( 'add_file\("zip/netbeans-{}-[0-9]{{12}}{}.zip"'.format( self.version, self.flavour)) for line in url_file.split("\n"): if preg.match(line): # Clean up the string from js (it's a function call) line = line.replace("add_file(", "").replace(");", "").replace('"', "") url_string = line if not url_string: # The file could not be parsed logger.error( "The download page changed its syntax or is not parsable") UI.return_main_screen(status_code=1) string_array = url_string.split(", ") try: url_suffix = string_array[0] sha256 = string_array[2] except IndexError: # The file could not be parsed logger.error( "The download page changed its syntax or is not parsable") UI.return_main_screen(status_code=1) download_url = "{}/{}/final/{}".format(self.BASE_URL, self.version, url_suffix) self.download_requests.append( DownloadItem(download_url, Checksum(ChecksumType.sha256, sha256))) self.start_download_and_install()
def get_metadata_and_check_license(self, result): """Download files to download + license and check it""" logger.debug("Parse download metadata") error_msg = result[self.download_page].error if error_msg: logger.error("An error occurred while downloading {}: {}".format(self.download_page, error_msg)) UI.return_main_screen() url, checksum = (None, None) with StringIO() as license_txt: in_license = False in_download = False for line in result[self.download_page].buffer: line_content = line.decode() if self.expect_license and not self.auto_accept_license: in_license = self.parse_license(line_content, license_txt, in_license) (download, in_download) = self.parse_download_link(line_content, in_download) if download is not None: (newurl, new_checksum) = download url = newurl if newurl is not None else url checksum = new_checksum if new_checksum is not None else checksum if url is not None: if self.checksum_type and checksum: logger.debug("Found download link for {}, checksum: {}".format(url, checksum)) break elif not self.checksum_type: logger.debug("Found download link for {}".format(url)) break if url is None or (self.checksum_type and checksum is None): logger.error("Download page changed its syntax or is not parsable") UI.return_main_screen() self.download_requests.append(DownloadItem(url, Checksum(self.checksum_type, checksum))) if license_txt.getvalue() != "": logger.debug("Check license agreement.") UI.display(LicenseAgreement(strip_tags(license_txt.getvalue()).strip(), self.start_download_and_install, UI.return_main_screen)) elif self.expect_license and not self.auto_accept_license: logger.error("We were expecting to find a license on the download page, we didn't.") UI.return_main_screen() else: self.start_download_and_install()
def parse_download_link(self, line, in_download): """Parse STS download link""" url_found = False in_download = False keyword = 'linux-gtk{}.tar.gz'.format(self.arch) if keyword in line: in_download = True if in_download: regexp = r'href="(.*.tar.gz)"' p = re.search(regexp, line) with suppress(AttributeError): url_found = True self.sha1_url = '{}.sha1'.format(p.group(1)) DownloadCenter(urls=[DownloadItem(self.sha1_url, None)], on_done=self.get_sha_and_start_download, download=False) return (url_found, in_download)
def parse_download_link(self, line, in_download): """Parse Eclipse download links""" if self.download_keyword in line and self.bits in line and 'linux' in line: in_download = True else: in_download = False if in_download: p = re.search(r"href='(https:)?\/\/www.eclipse.org(.*)'", line) logger.debug(p.group(2)) with suppress(AttributeError): self.sha512_url = "https://www.eclipse.org" + p.group( 2) + '.sha512&r=1' url_found = True DownloadCenter(urls=[DownloadItem(self.sha512_url, None)], on_done=self.get_sha_and_start_download, download=False) return (url_found, in_download)
def parse_download_link(self, line, in_download): """Parse Blender download links""" url = None if '.tar.bz2' in line: p = re.search( r'href="(http://download.blender.org/[^<]*{}.tar.bz2)"'.format( self.arch), line) with suppress(AttributeError): url = p.group(1) filename = 'release' + re.search('blender-(.*)-linux', url).group(1).replace( '.', '') + '.md5' self.checksum_url = os.path.join(os.path.dirname(url), filename) DownloadCenter(urls=[DownloadItem(self.checksum_url, None)], on_done=self.get_checksum_and_start_download, download=False) return (url, in_download)
def test_download_with_sha256sum(self): """we deliver once successful download, matching sha256sum""" filename = "simplefile" request = self.build_server_address(filename) DownloadCenter([DownloadItem(request, Checksum(ChecksumType.sha256, 'b1b113c6ed8ab3a14779f7c54179eac2b87d39fcebbf65a50556b8d68caaa2fb'))], self.callback) self.wait_for_callback(self.callback) result = self.callback.call_args[0][0][request] self.assertTrue(self.callback.called) self.assertEqual(self.callback.call_count, 1) with open(join(self.server_dir, filename), 'rb') as file_on_disk: self.assertEqual(file_on_disk.read(), result.fd.read()) self.assertIsNone(result.buffer) self.assertIsNone(result.error)
def test_with_invalid_certificate(self): """we error on invalid ssl certificate""" filename = "simplefile" url = TestDownloadCenter.build_server_address(self, filename) request = DownloadItem(url, None) DownloadCenter([request], self.callback) TestDownloadCenter.wait_for_callback(self, self.callback) result = self.callback.call_args[0][0][url] # disco python changed the error type found = False for invalidError in ["CERTIFICATE_VERIFY_FAILED", "bad handshake"]: if invalidError in result.error: found = True self.assertTrue(found) self.assertIsNone(result.buffer) self.assertIsNone(result.fd) self.expect_warn_error = True
def get_url_and_start_download(self, download_result): res = download_result[self.download_url] text = res.buffer.getvalue().decode('utf-8') url = re.search(r'http.*?.sh', text).group(0) if url is None: logger.error( "Download page changed its syntax or is not parsable (missing url)" ) UI.return_main_screen(status_code=1) if self.checksum is None: logger.error( "Download page changed its syntax or is not parsable (missing checksum)" ) UI.return_main_screen(status_code=1) logger.debug("Found download link for {}, checksum: {}".format( url, self.checksum)) self.download_requests.append( DownloadItem(url, Checksum(self.checksum_type, self.checksum))) self.start_download_and_install()