def DownloadFiles(files, check_hashes=True, parallel=False): """Download one of more files to the local disk. Args: files: List of FileInfo objects to download. check_hashes: When False assume local files have the correct hash otherwise always check the hashes match the onces in the FileInfo ojects. Returns: List of (filename, url) tuples. """ files_to_download = [] filenames = [] download_dir = naclports.package_index.PREBUILT_ROOT if not os.path.exists(download_dir): os.makedirs(download_dir) for file_info in files: basename = os.path.basename(file_info.url) file_info.name = os.path.join(download_dir, basename) filenames.append((file_info.name, file_info.url)) if os.path.exists(file_info.name): if not check_hashes or CheckHash(file_info.name, file_info.md5): Log('Up-to-date: %s' % file_info.name) continue files_to_download.append(file_info) def Check(file_info): if check_hashes and not CheckHash(file_info.name, file_info.md5): raise naclports.Error( 'Checksum failed: %s\nExpected=%s\nActual=%s' % (file_info.name, file_info.md5, GetHash(file_info.name))) if not files_to_download: Log('All files up-to-date') else: total_size = sum(f.size for f in files_to_download) Log('Need to download %d/%d files [%s]' % (len(files_to_download), len(files), FormatSize(total_size))) gsutil = FindGsutil() if parallel: remaining_files = files_to_download num_files = 20 while remaining_files: files = remaining_files[:num_files] remaining_files = remaining_files[num_files:] cmd = gsutil + ['-m', 'cp'] + [f.gsurl for f in files] + [download_dir] LogVerbose(cmd) subprocess.check_call(cmd) for file_info in files: Check(file_info) else: for file_info in files_to_download: naclports.DownloadFile(file_info.name, file_info.url) Check(file_info) return filenames
def Download(self, mirror=True): filename = self.DownloadLocation() if not filename or os.path.exists(filename): return if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) if mirror: # First try downloading form the mirror URL and silently fall # back to the original if this fails. mirror_url = self.GetMirrorURL() try: naclports.DownloadFile(filename, mirror_url) return except Error: pass naclports.DownloadFile(filename, self.URL)
def DownloadAndInstallSDK(url): bz2_dir = OUT_DIR if not os.path.exists(bz2_dir): os.makedirs(bz2_dir) bz2_filename = os.path.join(bz2_dir, url.split('/')[-1]) if sys.platform in ['win32', 'cygwin']: cygbin = os.path.join(FindCygwin(), 'bin') print('Downloading "%s" to "%s"...' % (url, bz2_filename)) sys.stdout.flush() # Download it. naclports.DownloadFile(bz2_filename, url) # Extract toolchain. old_cwd = os.getcwd() os.chdir(bz2_dir) Untar(bz2_filename) os.chdir(old_cwd) # Calculate pepper_dir by taking common prefix of tar # file contents tar = tarfile.open(bz2_filename) names = tar.getnames() tar.close() pepper_dir = os.path.commonprefix(names) actual_dir = os.path.join(bz2_dir, pepper_dir) # Drop old versions. if os.path.exists(TARGET_DIR): print('Cleaning up old SDK...') if sys.platform in ['win32', 'cygwin']: cmd = [os.path.join(cygbin, 'bin', 'rm.exe'), '-rf'] else: cmd = ['rm', '-rf'] cmd.append(TARGET_DIR) returncode = subprocess.call(cmd) assert returncode == 0 print('Renaming toolchain "%s" -> "%s"' % (actual_dir, TARGET_DIR)) os.rename(actual_dir, TARGET_DIR) if sys.platform in ['win32', 'cygwin']: time.sleep(2) # Wait for windows. # Clean up: remove the sdk bz2. os.remove(bz2_filename) print('Install complete.')
def DownloadToCache(url, sha1): filename = os.path.basename(url) full_name = os.path.join(naclports.paths.CACHE_ROOT, filename) if os.path.exists(full_name): try: naclports.util.VerifyHash(full_name, sha1) naclports.Log("Verified cached file: %s" % filename) return full_name except naclports.util.HashVerificationError: naclports.Log("Hash mistmatch on cached download: %s" % filename) naclports.DownloadFile(full_name, url) naclports.util.VerifyHash(full_name, sha1) return full_name
def Download(self, package_name, config): PREBUILT_ROOT = os.path.join(package.PACKAGES_ROOT, 'prebuilt') if not os.path.exists(PREBUILT_ROOT): os.makedirs(PREBUILT_ROOT) info = self.packages[(package_name, config)] filename = os.path.join(PREBUILT_ROOT, os.path.basename(info['BIN_URL'])) if os.path.exists(filename): if VerifyHash(filename, info['BIN_SHA1']): return filename naclports.Log('Downloading prebuilt binary ...') naclports.DownloadFile(filename, info['BIN_URL']) if not VerifyHash(filename, info['BIN_SHA1']): raise naclports.Error('Unexepected SHA1: %s' % filename) return filename
def DownloadFiles(files, check_hashes=True): """Download one of more files to the local disk. Args: files: List of FileInfo objects to download. check_hashes: When False assume local files have the correct hash otherwise always check the hashes match the onces in the FileInfo ojects. Returns: List of (filename, url) tuples. """ files_to_download = [] filenames = [] download_dir = naclports.package_index.PREBUILT_ROOT if not os.path.exists(download_dir): os.makedirs(download_dir) for file_info in files: basename = os.path.basename(file_info.url) fullname = os.path.join(download_dir, basename) filenames.append((fullname, file_info.url)) if os.path.exists(fullname): if not check_hashes or CheckHash(fullname, file_info.etag): Log('Up-to-date: %s' % file_info.name) continue files_to_download.append(FileInfo(fullname, file_info.size, file_info.url, file_info.etag)) if not files_to_download: Log('All files up-to-date') else: total_size = sum(f[1] for f in files_to_download) Log('Need to download %d/%d files [%s]' % (len(files_to_download), len(files), FormatSize(total_size))) for file_info in files_to_download: naclports.DownloadFile(file_info.name, file_info.url) if check_hashes and not CheckHash(file_info.name, file_info.etag): raise naclports.Error('Checksum failed: %s' % file_info.name) return filenames