def GsUpload(options, filename, url): """Upload a file to Google cloud storage using gsutil""" naclports.Log("Uploading to mirror: %s" % url) cmd = [options.gsutil, 'cp', '-a', 'public-read', filename, url] if options.dry_run: naclports.Log(cmd) else: subprocess.check_call(cmd)
def DownloadToCache(url, sha1): filename = os.path.basename(url) full_name = os.path.join(naclports.paths.CACHE_ROOT, filename) if os.path.exists(full_name): try: naclports.util.VerifyHash(full_name, sha1) naclports.Log("Verified cached file: %s" % filename) return full_name except naclports.util.HashVerificationError: naclports.Log("Hash mistmatch on cached download: %s" % filename) naclports.DownloadFile(full_name, url) naclports.util.VerifyHash(full_name, sha1) return full_name
def DownloadAndInstallSDK(url, target_dir): bz2_dir = OUT_DIR if not os.path.exists(bz2_dir): os.makedirs(bz2_dir) bz2_filename = os.path.join(bz2_dir, url.split('/')[-1]) if sys.platform in ['win32', 'cygwin']: cygbin = os.path.join(FindCygwin(), 'bin') # Download it. naclports.DownloadFile(bz2_filename, url) # Extract toolchain. old_cwd = os.getcwd() os.chdir(bz2_dir) Untar(bz2_filename) os.chdir(old_cwd) # Calculate pepper_dir by taking common prefix of tar # file contents tar = tarfile.open(bz2_filename) names = tar.getnames() tar.close() pepper_dir = os.path.commonprefix(names) actual_dir = os.path.join(bz2_dir, pepper_dir) # Drop old versions. if os.path.exists(target_dir): naclports.Log('Cleaning up old SDK...') if sys.platform in ['win32', 'cygwin']: cmd = [os.path.join(cygbin, 'bin', 'rm.exe'), '-rf'] else: cmd = ['rm', '-rf'] cmd.append(target_dir) returncode = subprocess.call(cmd) assert returncode == 0 naclports.Log('Renaming toolchain "%s" -> "%s"' % (actual_dir, target_dir)) os.rename(actual_dir, target_dir) if sys.platform in ['win32', 'cygwin']: time.sleep(2) # Wait for windows. # Clean up: remove the sdk bz2. os.remove(bz2_filename) naclports.Log('Install complete.')
def DoCmd(package): try: pkg_commands[command](package, options) except DisabledError as e: if options.ignore_disabled: naclports.Log('naclports: %s' % e) else: raise e
def Untar(bz2_filename): if sys.platform == 'win32': tar_file = None try: naclports.Log('Unpacking tarball...') tar_file = cygtar.CygTar(bz2_filename, 'r:bz2') tar_file.Extract() except Exception, err: raise naclports.Error('Error unpacking %s' % str(err)) finally:
def main(argv): if sys.platform in ['win32', 'cygwin']: naclports.Error( 'Emscripten support is currently not available on Windows.') return 1 DownloadAndExtract(EMSDK_URL, EMSDK_SHA1, 'emsdk_portable') DownloadAndExtract(NODEJS_URL, NODEJS_SHA1, 'node-v0.12.1-linux-x64') naclports.Log('Emscripten SDK Install complete') return 0
def DownloadAndExtract(url, sha1, target_dir): tar_file = DownloadToCache(url, sha1) if not os.path.exists(OUT_DIR): os.makedirs(OUT_DIR) os.chdir(OUT_DIR) # Remove previously extracted archive if os.path.exists(target_dir): naclports.Log('Cleaning up existing %s...' % target_dir) cmd = ['rm', '-rf'] cmd.append(target_dir) subprocess.check_call(cmd) # Extract archive naclports.Log('Exctacting %s...' % os.path.basename(tar_file)) if subprocess.call(['tar', 'xf', tar_file]): raise naclports.Error('Error unpacking Emscripten SDK')
def CheckPackages(options, source_packages, mirror_listing): count = 0 for package in source_packages: CheckMirror(options, package, mirror_listing) count += 1 if options.check: naclports.Log("Verfied mirroring for %d packages" % count) return 0
def CmdPkgCheck(package, options): # The fact that we got this far means the pkg_info is basically valid. # This final check verifies the dependencies are valid. # Cache the list of all packages names since this function could be called # a lot in the case of "naclports check --all". packages = PackageIterator() if not CmdPkgCheck.all_package_names: CmdPkgCheck.all_package_names = [os.path.basename(p.root) for p in packages] naclports.Log("Checking deps for %s .." % package.NAME) package.CheckDeps(CmdPkgCheck.all_package_names)
def Download(self, package_name, config): PREBUILT_ROOT = os.path.join(package.PACKAGES_ROOT, 'prebuilt') if not os.path.exists(PREBUILT_ROOT): os.makedirs(PREBUILT_ROOT) info = self.packages[(package_name, config)] filename = os.path.join(PREBUILT_ROOT, os.path.basename(info['BIN_URL'])) if os.path.exists(filename): if VerifyHash(filename, info['BIN_SHA1']): return filename naclports.Log('Downloading prebuilt binary ...') naclports.DownloadFile(filename, info['BIN_URL']) if not VerifyHash(filename, info['BIN_SHA1']): raise naclports.Error('Unexepected SHA1: %s' % filename) return filename
def DetermineSDKURL(flavor, base_url, version): """Download one Native Client toolchain and extract it. Arguments: flavor: flavor of the sdk to download base_url: base url to download toolchain tarballs from version: version directory to select tarballs from Returns: The URL of the SDK archive """ # gsutil.py ships with depot_tools, which should be in PATH gsutil = [sys.executable, naclports.util.FindInPath('gsutil.py')] path = flavor + '.tar.bz2' def GSList(path): """Run gsutil 'ls' on a path and return just the basenames of the elements within. """ cmd = gsutil + ['ls', base_url + path] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) p_stdout = p.communicate()[0] if p.returncode: raise naclports.Error('gsutil command failed: %s' % str(cmd)) elements = p_stdout.splitlines() return [os.path.basename(os.path.normpath(elem)) for elem in elements] if version == 'latest': naclports.Log('Looking for latest SDK build...') # List the top level of the nacl_sdk folder versions = GSList('') # Find all trunk revision versions = [v for v in versions if v.startswith('trunk')] # Look backwards through all trunk revisions # Only look back HISTORY_SIZE revisions so this script doesn't take # forever. versions = list(reversed(sorted(versions))) for version_dir in versions[:HISTORY_SIZE]: contents = GSList(version_dir) if path in contents: version = version_dir.rsplit('.', 1)[1] break else: raise naclports.Error( 'No SDK build (%s) found in last %d trunk builds' % (path, HISTORY_SIZE)) return '%strunk.%s/%s' % (GSTORE, version, path)
def main(argv): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-v', '--version', default='latest', help='which version of the SDK to download') parser.add_argument( '--bionic', action='store_true', help='download bionic version of the SDK (linux only).') options = parser.parse_args(argv) if options.bionic: flavor = 'naclsdk_bionic' else: flavor = 'naclsdk_' + PLATFORM_COLLAPSE[sys.platform] url = DetermineSDKURL(flavor, base_url=GS_URL_BASE, version=options.version) stamp_file = os.path.join(TARGET_DIR, 'stamp') if os.path.exists(stamp_file): with open(stamp_file) as f: installed_url = f.read().strip() if installed_url == url: naclports.Log('SDK already installed: %s' % url) return 0 else: naclports.Log('Ignoring currently installed SDK: %s' % installed_url) DownloadAndInstallSDK(url, TARGET_DIR) with open(stamp_file, 'w') as f: f.write(url + '\n') return 0
def main(args): parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') options = parser.parse_args(args) if options.verbose: naclports.SetVerbose(True) count = 0 package_names = [os.path.basename(p.root) for p in naclports.source_package.SourcePackageIterator()] for package in naclports.source_package.SourcePackageIterator(): if not package.CheckDeps(package_names): return 1 count += 1 naclports.Log("Verfied dependencies for %d packages" % count) return 0
def main(args): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') options = parser.parse_args(args) if options.verbose: naclports.SetVerbose(True) count = 0 for package in naclports.source_package.SourcePackageIterator(): package.Download() if not package.Verify(): return 1 count += 1 naclports.Log("Verfied checksums for %d packages" % count) return 0
def CheckMirror(options, package, mirror_listing): """Check that is package has is archive mirrors on Google cloud storage""" naclports.Trace('Checking %s' % package.NAME) basename = package.GetArchiveFilename() if not basename: return if basename in mirror_listing: # already mirrored return if options.check: naclports.Log('update_mirror: Archive missing from mirror: %s' % basename) sys.exit(1) # Download upstream URL package.Download(force_mirror=False) url = '%s/%s' % (MIRROR_GS, basename) GsUpload(options, package.DownloadLocation(), url)
def Install(self): """Install binary package into toolchain directory.""" dest = naclports.GetInstallRoot(self.config) dest_tmp = os.path.join(dest, 'install_tmp') if os.path.exists(dest_tmp): shutil.rmtree(dest_tmp) naclports.Log("Installing '%s' [%s]" % (self.NAME, self.config)) os.makedirs(dest_tmp) try: with tarfile.open(self.filename) as tar: names = [] for info in tar: if info.isdir(): continue name = posixpath.normpath(info.name) if name == 'pkg_info': continue if not name.startswith(PAYLOAD_DIR + '/'): raise naclports.PkgFormatError('invalid file in package: %s' % name) name = name[len(PAYLOAD_DIR) + 1:] names.append(name) tar.extractall(dest_tmp) payload_tree = os.path.join(dest_tmp, PAYLOAD_DIR) for name in names: self.InstallFile(name, payload_tree, dest) for name in names: self.RelocateFile(name, dest) self.WriteFileList(names) self.WriteStamp() finally: shutil.rmtree(dest_tmp)
def rmtree(path): naclports.Log('removing %s' % path) if os.path.exists(path): shutil.rmtree(path)