def untar(bz2_filename):
if sys.platform == 'win32':
tar_file = None
try:
webports.log('Unpacking tarball...')
tar_file = cygtar.CygTar(bz2_filename, 'r:bz2')
tar_file.extract()
except Exception as err:
raise webports.Error('Error unpacking %s' % str(err))
finally:
if tar_file:
tar_file.Close()
else:
if subprocess.call(['tar', 'jxf', bz2_filename]):
raise webports.Error('Error unpacking')
def FindCygwin():
if os.path.exists(r'\cygwin'):
return r'\cygwin'
elif os.path.exists(r'C:\cygwin'):
return r'C:\cygwin'
else:
raise webports.Error(r'failed to find cygwin in \cygwin or c:\cygwin')
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('revision',
metavar='REVISION',
help='webports revision to to scan for.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Output extra information.')
parser.add_argument('-p',
'--parallel',
action='store_true',
help='Download packages in parallel.')
parser.add_argument('-l',
'--cache-listing',
action='store_true',
help='Cached output of gsutil -L (for testing).')
parser.add_argument(
'--skip-md5',
action='store_true',
help='Assume on-disk files are up-to-date (for testing).')
args = parser.parse_args(args)
if args.verbose:
webports.set_verbose(True)
sdk_version = webports.util.get_sdk_version()
log('Scanning packages built for pepper_%s at revsion %s' %
(sdk_version, args.revision))
base_path = '%s/builds/pepper_%s/%s/packages' % (
webports.GS_BUCKET, sdk_version, args.revision)
gs_url = 'gs://' + base_path + '/*'
listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', 'listing.txt')
if args.cache_listing and os.path.exists(listing_file):
log('Using pre-cached gs listing: %s' % listing_file)
with open(listing_file) as f:
listing = f.read()
else:
log('Searching for packages at: %s' % gs_url)
cmd = find_gsutil() + ['stat', gs_url]
log_verbose('Running: %s' % str(cmd))
try:
listing = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise webports.Error("Command '%s' failed: %s" % (cmd, e))
if args.cache_listing:
with open(listing_file, 'w') as f:
f.write(listing)
all_files = parse_gs_util_output(listing)
log('Found %d packages [%s]' %
(len(all_files), format_size(sum(f.size for f in all_files))))
binaries = download_files(all_files, not args.skip_md5, args.parallel)
index_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt')
log('Generating %s' % index_file)
webports.package_index.write_index(index_file, binaries)
log('Done')
return 0
def Untar(bz2_filename):
if sys.platform == 'win32':
tar_file = None
try:
webports.Log('Unpacking tarball...')
tar_file = cygtar.CygTar(bz2_filename, 'r:bz2')
tar_file.Extract()
except Exception, err:
raise webports.Error('Error unpacking %s' % str(err))
finally:
def main(argv):
if sys.platform in ['win32', 'cygwin']:
webports.Error('Emscripten support is currently not available on Windows.')
return 1
download_and_extract(EMSDK_URL, EMSDK_SHA1, 'emsdk')
download_and_extract(NODE_URL, NODE_SHA1, 'node-v0.12.1-linux-x64', 'node')
build_optimizer()
webports.log('Emscripten SDK Install complete')
return 0
def GSList(path):
"""Run gsutil 'ls' on a path and return just the basenames of the
elements within.
"""
cmd = gsutil + ['ls', base_url + path]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
p_stdout = p.communicate()[0]
if p.returncode:
raise webports.Error('gsutil command failed: %s' % str(cmd))
elements = p_stdout.splitlines()
return [os.path.basename(os.path.normpath(elem)) for elem in elements]
def DetermineSDKURL(flavor, base_url, version):
"""Download one Native Client toolchain and extract it.
Arguments:
flavor: flavor of the sdk to download
base_url: base url to download toolchain tarballs from
version: version directory to select tarballs from
Returns:
The URL of the SDK archive
"""
# gsutil.py ships with depot_tools, which should be in PATH
gsutil = [sys.executable, webports.util.FindInPath('gsutil.py')]
path = flavor + '.tar.bz2'
def GSList(path):
"""Run gsutil 'ls' on a path and return just the basenames of the
elements within.
"""
cmd = gsutil + ['ls', base_url + path]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
p_stdout = p.communicate()[0]
if p.returncode:
raise webports.Error('gsutil command failed: %s' % str(cmd))
elements = p_stdout.splitlines()
return [os.path.basename(os.path.normpath(elem)) for elem in elements]
if version == 'latest':
webports.Log('Looking for latest SDK build...')
# List the top level of the nacl_sdk folder
versions = GSList('')
# Find all trunk revision
versions = [v for v in versions if v.startswith('trunk')]
# Look backwards through all trunk revisions
# Only look back HISTORY_SIZE revisions so this script doesn't take
# forever.
versions = list(reversed(sorted(versions)))
for version_dir in versions[:HISTORY_SIZE]:
contents = GSList(version_dir)
if path in contents:
version = version_dir.rsplit('.', 1)[1]
break
else:
raise webports.Error(
'No SDK build (%s) found in last %d trunk builds' %
(path, HISTORY_SIZE))
return '%strunk.%s/%s' % (GSTORE, version, path)
def DownloadAndExtract(url, sha1, target_dir, link_name=None):
tar_file = DownloadToCache(url, sha1)
if not os.path.exists(OUT_DIR):
os.makedirs(OUT_DIR)
os.chdir(OUT_DIR)
# Remove previously extracted archive
if os.path.exists(target_dir):
webports.Log('Cleaning up existing %s...' % target_dir)
cmd = ['rm', '-rf']
cmd.append(target_dir)
subprocess.check_call(cmd)
# Extract archive
webports.Log('Exctacting %s...' % os.path.basename(tar_file))
if subprocess.call(['tar', 'xf', tar_file]):
raise webports.Error('Error unpacking Emscripten SDK')
if link_name:
if os.path.exists(link_name):
os.remove(link_name)
os.symlink(target_dir, link_name)
def Untar(bz2_filename):
if sys.platform == 'win32':
tar_file = None
try:
webports.Log('Unpacking tarball...')
tar_file = cygtar.CygTar(bz2_filename, 'r:bz2')
tar_file.Extract()
except Exception, err:
raise webports.Error('Error unpacking %s' % str(err))
finally:
if tar_file:
tar_file.Close()
else:
if subprocess.call(['tar', 'jxf', bz2_filename]):
raise webports.Error('Error unpacking')
def FindCygwin():
if os.path.exists(r'\cygwin'):
return r'\cygwin'
elif os.path.exists(r'C:\cygwin'):
return r'C:\cygwin'
else:
raise webports.Error(r'failed to find cygwin in \cygwin or c:\cygwin')
def DownloadAndInstallSDK(url, target_dir):
bz2_dir = OUT_DIR
if not os.path.exists(bz2_dir):
os.makedirs(bz2_dir)
def check(file_info):
if check_hashes and not check_hash(file_info.name, file_info.md5):
raise webports.Error(
'Checksum failed: %s\nExpected=%s\nActual=%s' %
(file_info.name, file_info.md5, get_hash(file_info.name)))
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('revision',
metavar='REVISION',
help='webports revision to to scan for.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Output extra information.')
parser.add_argument('-p',
'--parallel',
action='store_true',
help='Download packages in parallel.')
parser.add_argument('-l',
'--cache-listing',
action='store_true',
help='Cached output of gsutil -L (for testing).')
parser.add_argument(
'--skip-md5',
action='store_true',
help='Assume on-disk files are up-to-date (for testing).')
args = parser.parse_args(args)
if args.verbose:
webports.SetVerbose(True)
sdk_version = webports.util.GetSDKVersion()
Log('Scanning packages built for pepper_%s at revsion %s' %
(sdk_version, args.revision))
base_path = '%s/builds/pepper_%s/%s/publish' % (webports.GS_BUCKET,
sdk_version, args.revision)
gs_base_url = 'gs://' + base_path
cmd = FindGsutil() + ['ls', gs_base_url]
LogVerbose('Running: %s' % str(cmd))
try:
all_published = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise webports.Error("Command '%s' failed: %s" % (cmd, e))
pkg_dir = re.findall(r'pkg_[\w-]+', all_published)
for pkg in pkg_dir:
listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib',
pkg + '_' + 'listing.txt')
if args.cache_listing and os.path.exists(listing_file):
Log('Using pre-cached gs listing: %s' % listing_file)
with open(listing_file) as f:
listing = f.read()
else:
gs_url = gs_base_url + '/' + pkg + '/*'
Log('Searching for packages at: %s' % gs_url)
cmd = FindGsutil() + ['stat', gs_url]
LogVerbose('Running: %s' % str(cmd))
try:
listing = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise webports.Error("Command '%s' failed: %s" % (cmd, e))
if args.cache_listing:
with open(listing_file, 'w') as f:
f.write(listing)
all_files = ParseGsUtilOutput(listing)
Log('Found %d packages [%s] for %s' %
(len(all_files), FormatSize(sum(f.size for f in all_files)), pkg))
DownloadFiles(pkg, all_files, not args.skip_md5, args.parallel)
Log('Done')
return 0
