def untar(bz2_filename): if sys.platform == 'win32': tar_file = None try: webports.log('Unpacking tarball...') tar_file = cygtar.CygTar(bz2_filename, 'r:bz2') tar_file.extract() except Exception as err: raise webports.Error('Error unpacking %s' % str(err)) finally: if tar_file: tar_file.Close() else: if subprocess.call(['tar', 'jxf', bz2_filename]): raise webports.Error('Error unpacking')
def FindCygwin(): if os.path.exists(r'\cygwin'): return r'\cygwin' elif os.path.exists(r'C:\cygwin'): return r'C:\cygwin' else: raise webports.Error(r'failed to find cygwin in \cygwin or c:\cygwin')
def main(args): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('revision', metavar='REVISION', help='webports revision to to scan for.') parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') parser.add_argument('-p', '--parallel', action='store_true', help='Download packages in parallel.') parser.add_argument('-l', '--cache-listing', action='store_true', help='Cached output of gsutil -L (for testing).') parser.add_argument( '--skip-md5', action='store_true', help='Assume on-disk files are up-to-date (for testing).') args = parser.parse_args(args) if args.verbose: webports.set_verbose(True) sdk_version = webports.util.get_sdk_version() log('Scanning packages built for pepper_%s at revsion %s' % (sdk_version, args.revision)) base_path = '%s/builds/pepper_%s/%s/packages' % ( webports.GS_BUCKET, sdk_version, args.revision) gs_url = 'gs://' + base_path + '/*' listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', 'listing.txt') if args.cache_listing and os.path.exists(listing_file): log('Using pre-cached gs listing: %s' % listing_file) with open(listing_file) as f: listing = f.read() else: log('Searching for packages at: %s' % gs_url) cmd = find_gsutil() + ['stat', gs_url] log_verbose('Running: %s' % str(cmd)) try: listing = subprocess.check_output(cmd) except subprocess.CalledProcessError as e: raise webports.Error("Command '%s' failed: %s" % (cmd, e)) if args.cache_listing: with open(listing_file, 'w') as f: f.write(listing) all_files = parse_gs_util_output(listing) log('Found %d packages [%s]' % (len(all_files), format_size(sum(f.size for f in all_files)))) binaries = download_files(all_files, not args.skip_md5, args.parallel) index_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt') log('Generating %s' % index_file) webports.package_index.write_index(index_file, binaries) log('Done') return 0
def Untar(bz2_filename): if sys.platform == 'win32': tar_file = None try: webports.Log('Unpacking tarball...') tar_file = cygtar.CygTar(bz2_filename, 'r:bz2') tar_file.Extract() except Exception, err: raise webports.Error('Error unpacking %s' % str(err)) finally:
def main(argv): if sys.platform in ['win32', 'cygwin']: webports.Error('Emscripten support is currently not available on Windows.') return 1 download_and_extract(EMSDK_URL, EMSDK_SHA1, 'emsdk') download_and_extract(NODE_URL, NODE_SHA1, 'node-v0.12.1-linux-x64', 'node') build_optimizer() webports.log('Emscripten SDK Install complete') return 0
def GSList(path): """Run gsutil 'ls' on a path and return just the basenames of the elements within. """ cmd = gsutil + ['ls', base_url + path] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) p_stdout = p.communicate()[0] if p.returncode: raise webports.Error('gsutil command failed: %s' % str(cmd)) elements = p_stdout.splitlines() return [os.path.basename(os.path.normpath(elem)) for elem in elements]
def DetermineSDKURL(flavor, base_url, version): """Download one Native Client toolchain and extract it. Arguments: flavor: flavor of the sdk to download base_url: base url to download toolchain tarballs from version: version directory to select tarballs from Returns: The URL of the SDK archive """ # gsutil.py ships with depot_tools, which should be in PATH gsutil = [sys.executable, webports.util.FindInPath('gsutil.py')] path = flavor + '.tar.bz2' def GSList(path): """Run gsutil 'ls' on a path and return just the basenames of the elements within. """ cmd = gsutil + ['ls', base_url + path] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) p_stdout = p.communicate()[0] if p.returncode: raise webports.Error('gsutil command failed: %s' % str(cmd)) elements = p_stdout.splitlines() return [os.path.basename(os.path.normpath(elem)) for elem in elements] if version == 'latest': webports.Log('Looking for latest SDK build...') # List the top level of the nacl_sdk folder versions = GSList('') # Find all trunk revision versions = [v for v in versions if v.startswith('trunk')] # Look backwards through all trunk revisions # Only look back HISTORY_SIZE revisions so this script doesn't take # forever. versions = list(reversed(sorted(versions))) for version_dir in versions[:HISTORY_SIZE]: contents = GSList(version_dir) if path in contents: version = version_dir.rsplit('.', 1)[1] break else: raise webports.Error( 'No SDK build (%s) found in last %d trunk builds' % (path, HISTORY_SIZE)) return '%strunk.%s/%s' % (GSTORE, version, path)
def DownloadAndExtract(url, sha1, target_dir, link_name=None): tar_file = DownloadToCache(url, sha1) if not os.path.exists(OUT_DIR): os.makedirs(OUT_DIR) os.chdir(OUT_DIR) # Remove previously extracted archive if os.path.exists(target_dir): webports.Log('Cleaning up existing %s...' % target_dir) cmd = ['rm', '-rf'] cmd.append(target_dir) subprocess.check_call(cmd) # Extract archive webports.Log('Exctacting %s...' % os.path.basename(tar_file)) if subprocess.call(['tar', 'xf', tar_file]): raise webports.Error('Error unpacking Emscripten SDK') if link_name: if os.path.exists(link_name): os.remove(link_name) os.symlink(target_dir, link_name)
def Untar(bz2_filename): if sys.platform == 'win32': tar_file = None try: webports.Log('Unpacking tarball...') tar_file = cygtar.CygTar(bz2_filename, 'r:bz2') tar_file.Extract() except Exception, err: raise webports.Error('Error unpacking %s' % str(err)) finally: if tar_file: tar_file.Close() else: if subprocess.call(['tar', 'jxf', bz2_filename]): raise webports.Error('Error unpacking') def FindCygwin(): if os.path.exists(r'\cygwin'): return r'\cygwin' elif os.path.exists(r'C:\cygwin'): return r'C:\cygwin' else: raise webports.Error(r'failed to find cygwin in \cygwin or c:\cygwin') def DownloadAndInstallSDK(url, target_dir): bz2_dir = OUT_DIR if not os.path.exists(bz2_dir): os.makedirs(bz2_dir)
def check(file_info): if check_hashes and not check_hash(file_info.name, file_info.md5): raise webports.Error( 'Checksum failed: %s\nExpected=%s\nActual=%s' % (file_info.name, file_info.md5, get_hash(file_info.name)))
def main(args): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('revision', metavar='REVISION', help='webports revision to to scan for.') parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') parser.add_argument('-p', '--parallel', action='store_true', help='Download packages in parallel.') parser.add_argument('-l', '--cache-listing', action='store_true', help='Cached output of gsutil -L (for testing).') parser.add_argument( '--skip-md5', action='store_true', help='Assume on-disk files are up-to-date (for testing).') args = parser.parse_args(args) if args.verbose: webports.SetVerbose(True) sdk_version = webports.util.GetSDKVersion() Log('Scanning packages built for pepper_%s at revsion %s' % (sdk_version, args.revision)) base_path = '%s/builds/pepper_%s/%s/publish' % (webports.GS_BUCKET, sdk_version, args.revision) gs_base_url = 'gs://' + base_path cmd = FindGsutil() + ['ls', gs_base_url] LogVerbose('Running: %s' % str(cmd)) try: all_published = subprocess.check_output(cmd) except subprocess.CalledProcessError as e: raise webports.Error("Command '%s' failed: %s" % (cmd, e)) pkg_dir = re.findall(r'pkg_[\w-]+', all_published) for pkg in pkg_dir: listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', pkg + '_' + 'listing.txt') if args.cache_listing and os.path.exists(listing_file): Log('Using pre-cached gs listing: %s' % listing_file) with open(listing_file) as f: listing = f.read() else: gs_url = gs_base_url + '/' + pkg + '/*' Log('Searching for packages at: %s' % gs_url) cmd = FindGsutil() + ['stat', gs_url] LogVerbose('Running: %s' % str(cmd)) try: listing = subprocess.check_output(cmd) except subprocess.CalledProcessError as e: raise webports.Error("Command '%s' failed: %s" % (cmd, e)) if args.cache_listing: with open(listing_file, 'w') as f: f.write(listing) all_files = ParseGsUtilOutput(listing) Log('Found %d packages [%s] for %s' % (len(all_files), FormatSize(sum(f.size for f in all_files)), pkg)) DownloadFiles(pkg, all_files, not args.skip_md5, args.parallel) Log('Done') return 0