def _get_remote_file_contents(self, url, use_cache): # We don't want traces in output of these downloads, they are ugly in output downloader = FileDownloader(self.requester, None, self.verify_ssl, self._config) if use_cache and self._config.download_cache: downloader = CachedFileDownloader(self._config.download_cache, downloader) contents = downloader.download(url, auth=self.auth) return contents
def _download_files(self, file_urls, snapshot_md5): """ :param: file_urls is a dict with {filename: url} :param snapshot_md5: dict with {filaname: md5 checksum} of files to be downloaded Its a generator, so it yields elements for memory performance """ downloader = FileDownloader(self.requester, None, self.verify_ssl, self._config) download_cache = self._config.download_cache if download_cache: assert snapshot_md5 is not None, "if download_cache is set, we need the file checksums" downloader = CachedFileDownloader(download_cache, downloader) # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz # can be < conanfile, conaninfo, and sent always the last, so smaller files go first for filename, resource_url in sorted(file_urls.items(), reverse=True): auth, _ = self._file_server_capabilities(resource_url) if download_cache: md5 = snapshot_md5[filename] contents = downloader.download(resource_url, auth=auth, md5=md5) else: contents = downloader.download(resource_url, auth=auth) yield os.path.normpath(filename), contents
def _download_files_to_folder(self, file_urls, to_folder, snapshot_md5): """ :param: file_urls is a dict with {filename: abs_path} It writes downloaded files to disk (appending to file, only keeps chunks in memory) """ downloader = FileDownloader(self.requester, self._output, self.verify_ssl, self._config) download_cache = self._config.download_cache if download_cache: assert snapshot_md5 is not None, "if download_cache is set, we need the file checksums" downloader = CachedFileDownloader(download_cache, downloader) ret = {} # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz # can be < conanfile, conaninfo, and sent always the last, so smaller files go first for filename, resource_url in sorted(file_urls.items(), reverse=True): if self._output and not self._output.is_terminal: self._output.writeln("Downloading %s" % filename) auth, _ = self._file_server_capabilities(resource_url) abs_path = os.path.join(to_folder, filename) if download_cache: md5 = snapshot_md5[filename] downloader.download(resource_url, abs_path, auth=auth, md5=md5) else: downloader.download(resource_url, abs_path, auth=auth) ret[filename] = abs_path return ret
def setUp(self): cache_folder = temp_folder() class FakeFileDownloader(object): def __init__(self): self.calls = Counter() def download(self, url, file_path_=None, *args, **kwargs): if "slow" in url: time.sleep(0.5) self.calls[url] += 1 if file_path_: save(file_path_, url) else: return url self.file_downloader = FakeFileDownloader() self.cached_downloader = CachedFileDownloader(cache_folder, self.file_downloader)
def _download_and_save_files(self, urls, dest_folder, files, use_cache): downloader = FileDownloader(self.requester, self._output, self.verify_ssl, self._config) if use_cache and self._config.download_cache: downloader = CachedFileDownloader(self._config.download_cache, downloader) # Take advantage of filenames ordering, so that conan_package.tgz and conan_export.tgz # can be < conanfile, conaninfo, and sent always the last, so smaller files go first for filename in sorted(files, reverse=True): if self._output and not self._output.is_terminal: self._output.writeln("Downloading %s" % filename) resource_url = urls[filename] abs_path = os.path.join(dest_folder, filename) downloader.download(resource_url, abs_path, auth=self.auth)
def download(url, filename, verify=True, out=None, retry=None, retry_wait=None, overwrite=False, auth=None, headers=None, requester=None, md5='', sha1='', sha256=''): """Retrieves a file from a given URL into a file with a given filename. It uses certificates from a list of known verifiers for https downloads, but this can be optionally disabled. :param url: URL to download. It can be a list, which only the first one will be downloaded, and the follow URLs will be used as mirror in case of download error. :param filename: Name of the file to be created in the local storage :param verify: When False, disables https certificate validation :param out: An object with a write() method can be passed to get the output. stdout will use if not specified :param retry: Number of retries in case of failure. Default is overriden by general.retry in the conan.conf file or an env variable CONAN_RETRY :param retry_wait: Seconds to wait between download attempts. Default is overriden by general.retry_wait in the conan.conf file or an env variable CONAN_RETRY_WAIT :param overwrite: When True, Conan will overwrite the destination file if exists. Otherwise it will raise an exception :param auth: A tuple of user and password to use HTTPBasic authentication :param headers: A dictionary with additional headers :param requester: HTTP requests instance :param md5: MD5 hash code to check the downloaded file :param sha1: SHA-1 hash code to check the downloaded file :param sha256: SHA-256 hash code to check the downloaded file :return: None """ out = default_output(out, 'conans.client.tools.net.download') requester = default_requester(requester, 'conans.client.tools.net.download') from conans.tools import _global_config as config # It might be possible that users provide their own requester retry = retry if retry is not None else config.retry retry = retry if retry is not None else 1 retry_wait = retry_wait if retry_wait is not None else config.retry_wait retry_wait = retry_wait if retry_wait is not None else 5 checksum = sha256 or sha1 or md5 downloader = FileDownloader(requester=requester, output=out, verify=verify, config=config) if config and config.download_cache and checksum: downloader = CachedFileDownloader(config.download_cache, downloader, user_download=True) def _download_file(file_url): # The download cache is only used if a checksum is provided, otherwise, a normal download if isinstance(downloader, CachedFileDownloader): downloader.download(file_url, filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) else: downloader.download(file_url, filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, auth=auth, headers=headers) if md5: check_md5(filename, md5) if sha1: check_sha1(filename, sha1) if sha256: check_sha256(filename, sha256) out.writeln("") if not isinstance(url, (list, tuple)): _download_file(url) else: # We were provided several URLs to try for url_it in url: try: _download_file(url_it) break except Exception as error: message = "Could not download from the URL {}: {}.".format( url_it, str(error)) out.warn(message + " Trying another mirror.") else: raise ConanException( "All downloads from ({}) URLs have failed.".format(len(url)))
class CachedDownloaderUnitTest(unittest.TestCase): def setUp(self): cache_folder = temp_folder() class FakeFileDownloader(object): def __init__(self): self.calls = Counter() def download(self, url, file_path_=None, *args, **kwargs): if "slow" in url: time.sleep(0.5) self.calls[url] += 1 if file_path_: save(file_path_, url) else: return url self.file_downloader = FakeFileDownloader() self.cached_downloader = CachedFileDownloader(cache_folder, self.file_downloader) def test_concurrent_locks(self): folder = temp_folder() def download(index): if index % 2: content = self.cached_downloader.download("slow_testurl") content = content.decode("utf-8") else: file_path = os.path.join(folder, "myfile%s.txt" % index) self.cached_downloader.download("slow_testurl", file_path) content = load(file_path) self.assertEqual(content, "slow_testurl") self.assertEqual(self.file_downloader.calls["slow_testurl"], 1) ps = [] for i in range(8): thread = Thread(target=download, args=(i,)) thread.start() ps.append(thread) for p in ps: p.join() self.assertEqual(self.file_downloader.calls["slow_testurl"], 1) def test_basic(self): folder = temp_folder() file_path = os.path.join(folder, "myfile.txt") self.cached_downloader.download("testurl", file_path) self.assertEqual(self.file_downloader.calls["testurl"], 1) self.assertEqual("testurl", load(file_path)) # Try again, the count will be the same self.cached_downloader.download("testurl", file_path) self.assertEqual(self.file_downloader.calls["testurl"], 1) # Try direct content content = self.cached_downloader.download("testurl") self.assertEqual(content.decode("utf-8"), "testurl") self.assertEqual(self.file_downloader.calls["testurl"], 1) # Try another file file_path = os.path.join(folder, "myfile2.txt") self.cached_downloader.download("testurl2", file_path) self.assertEqual(self.file_downloader.calls["testurl2"], 1) self.assertEqual("testurl2", load(file_path)) self.cached_downloader.download("testurl2", file_path) self.assertEqual(self.file_downloader.calls["testurl"], 1) self.assertEqual(self.file_downloader.calls["testurl2"], 1) def test_content_first(self): # first calling content without path also caches content = self.cached_downloader.download("testurl") self.assertEqual(content.decode("utf-8"), "testurl") # content is binary here self.assertEqual(self.file_downloader.calls["testurl"], 1) # Now the file folder = temp_folder() file_path = os.path.join(folder, "myfile.txt") self.cached_downloader.download("testurl", file_path) self.assertEqual(self.file_downloader.calls["testurl"], 1) self.assertEqual("testurl", load(file_path))
def download(url, filename, verify=True, out=None, retry=None, retry_wait=None, overwrite=False, auth=None, headers=None, requester=None, md5='', sha1='', sha256=''): out = default_output(out, 'conans.client.tools.net.download') requester = default_requester(requester, 'conans.client.tools.net.download') from conans.tools import _global_config as config # It might be possible that users provide their own requester retry = retry if retry is not None else config.retry retry = retry if retry is not None else 1 retry_wait = retry_wait if retry_wait is not None else config.retry_wait retry_wait = retry_wait if retry_wait is not None else 5 downloader = FileDownloader(requester=requester, output=out, verify=verify, config=config) checksum = sha256 or sha1 or md5 # The download cache is only used if a checksum is provided, otherwise, a normal download if config and config.download_cache and checksum: downloader = CachedFileDownloader(config.download_cache, downloader, user_download=True) downloader.download(url, filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, auth=auth, headers=headers, md5=md5, sha1=sha1, sha256=sha256) else: downloader.download(url, filename, retry=retry, retry_wait=retry_wait, overwrite=overwrite, auth=auth, headers=headers) if md5: check_md5(filename, md5) if sha1: check_sha1(filename, sha1) if sha256: check_sha256(filename, sha256) out.writeln("")