def GsUpload(options, filename, url):
"""Upload a file to Google cloud storage using gsutil"""
naclports.Log("Uploading to mirror: %s" % url)
cmd = [options.gsutil, 'cp', '-a', 'public-read', filename, url]
if options.dry_run:
naclports.Log(cmd)
else:
subprocess.check_call(cmd)
def DownloadToCache(url, sha1):
filename = os.path.basename(url)
full_name = os.path.join(naclports.paths.CACHE_ROOT, filename)
if os.path.exists(full_name):
try:
naclports.util.VerifyHash(full_name, sha1)
naclports.Log("Verified cached file: %s" % filename)
return full_name
except naclports.util.HashVerificationError:
naclports.Log("Hash mistmatch on cached download: %s" % filename)
naclports.DownloadFile(full_name, url)
naclports.util.VerifyHash(full_name, sha1)
return full_name
def DownloadAndInstallSDK(url, target_dir):
bz2_dir = OUT_DIR
if not os.path.exists(bz2_dir):
os.makedirs(bz2_dir)
bz2_filename = os.path.join(bz2_dir, url.split('/')[-1])
if sys.platform in ['win32', 'cygwin']:
cygbin = os.path.join(FindCygwin(), 'bin')
# Download it.
naclports.DownloadFile(bz2_filename, url)
# Extract toolchain.
old_cwd = os.getcwd()
os.chdir(bz2_dir)
Untar(bz2_filename)
os.chdir(old_cwd)
# Calculate pepper_dir by taking common prefix of tar
# file contents
tar = tarfile.open(bz2_filename)
names = tar.getnames()
tar.close()
pepper_dir = os.path.commonprefix(names)
actual_dir = os.path.join(bz2_dir, pepper_dir)
# Drop old versions.
if os.path.exists(target_dir):
naclports.Log('Cleaning up old SDK...')
if sys.platform in ['win32', 'cygwin']:
cmd = [os.path.join(cygbin, 'bin', 'rm.exe'), '-rf']
else:
cmd = ['rm', '-rf']
cmd.append(target_dir)
returncode = subprocess.call(cmd)
assert returncode == 0
naclports.Log('Renaming toolchain "%s" -> "%s"' % (actual_dir, target_dir))
os.rename(actual_dir, target_dir)
if sys.platform in ['win32', 'cygwin']:
time.sleep(2) # Wait for windows.
# Clean up: remove the sdk bz2.
os.remove(bz2_filename)
naclports.Log('Install complete.')
def DoCmd(package):
try:
pkg_commands[command](package, options)
except DisabledError as e:
if options.ignore_disabled:
naclports.Log('naclports: %s' % e)
else:
raise e
def Untar(bz2_filename):
if sys.platform == 'win32':
tar_file = None
try:
naclports.Log('Unpacking tarball...')
tar_file = cygtar.CygTar(bz2_filename, 'r:bz2')
tar_file.Extract()
except Exception, err:
raise naclports.Error('Error unpacking %s' % str(err))
finally:
def main(argv):
if sys.platform in ['win32', 'cygwin']:
naclports.Error(
'Emscripten support is currently not available on Windows.')
return 1
DownloadAndExtract(EMSDK_URL, EMSDK_SHA1, 'emsdk_portable')
DownloadAndExtract(NODEJS_URL, NODEJS_SHA1, 'node-v0.12.1-linux-x64')
naclports.Log('Emscripten SDK Install complete')
return 0
def DownloadAndExtract(url, sha1, target_dir):
tar_file = DownloadToCache(url, sha1)
if not os.path.exists(OUT_DIR):
os.makedirs(OUT_DIR)
os.chdir(OUT_DIR)
# Remove previously extracted archive
if os.path.exists(target_dir):
naclports.Log('Cleaning up existing %s...' % target_dir)
cmd = ['rm', '-rf']
cmd.append(target_dir)
subprocess.check_call(cmd)
# Extract archive
naclports.Log('Exctacting %s...' % os.path.basename(tar_file))
if subprocess.call(['tar', 'xf', tar_file]):
raise naclports.Error('Error unpacking Emscripten SDK')
def CheckPackages(options, source_packages, mirror_listing):
count = 0
for package in source_packages:
CheckMirror(options, package, mirror_listing)
count += 1
if options.check:
naclports.Log("Verfied mirroring for %d packages" % count)
return 0
def CmdPkgCheck(package, options):
# The fact that we got this far means the pkg_info is basically valid.
# This final check verifies the dependencies are valid.
# Cache the list of all packages names since this function could be called
# a lot in the case of "naclports check --all".
packages = PackageIterator()
if not CmdPkgCheck.all_package_names:
CmdPkgCheck.all_package_names = [os.path.basename(p.root) for p in packages]
naclports.Log("Checking deps for %s .." % package.NAME)
package.CheckDeps(CmdPkgCheck.all_package_names)
def Download(self, package_name, config):
PREBUILT_ROOT = os.path.join(package.PACKAGES_ROOT, 'prebuilt')
if not os.path.exists(PREBUILT_ROOT):
os.makedirs(PREBUILT_ROOT)
info = self.packages[(package_name, config)]
filename = os.path.join(PREBUILT_ROOT, os.path.basename(info['BIN_URL']))
if os.path.exists(filename):
if VerifyHash(filename, info['BIN_SHA1']):
return filename
naclports.Log('Downloading prebuilt binary ...')
naclports.DownloadFile(filename, info['BIN_URL'])
if not VerifyHash(filename, info['BIN_SHA1']):
raise naclports.Error('Unexepected SHA1: %s' % filename)
return filename
def DetermineSDKURL(flavor, base_url, version):
"""Download one Native Client toolchain and extract it.
Arguments:
flavor: flavor of the sdk to download
base_url: base url to download toolchain tarballs from
version: version directory to select tarballs from
Returns:
The URL of the SDK archive
"""
# gsutil.py ships with depot_tools, which should be in PATH
gsutil = [sys.executable, naclports.util.FindInPath('gsutil.py')]
path = flavor + '.tar.bz2'
def GSList(path):
"""Run gsutil 'ls' on a path and return just the basenames of the
elements within.
"""
cmd = gsutil + ['ls', base_url + path]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
p_stdout = p.communicate()[0]
if p.returncode:
raise naclports.Error('gsutil command failed: %s' % str(cmd))
elements = p_stdout.splitlines()
return [os.path.basename(os.path.normpath(elem)) for elem in elements]
if version == 'latest':
naclports.Log('Looking for latest SDK build...')
# List the top level of the nacl_sdk folder
versions = GSList('')
# Find all trunk revision
versions = [v for v in versions if v.startswith('trunk')]
# Look backwards through all trunk revisions
# Only look back HISTORY_SIZE revisions so this script doesn't take
# forever.
versions = list(reversed(sorted(versions)))
for version_dir in versions[:HISTORY_SIZE]:
contents = GSList(version_dir)
if path in contents:
version = version_dir.rsplit('.', 1)[1]
break
else:
raise naclports.Error(
'No SDK build (%s) found in last %d trunk builds' %
(path, HISTORY_SIZE))
return '%strunk.%s/%s' % (GSTORE, version, path)
def main(argv):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-v',
'--version',
default='latest',
help='which version of the SDK to download')
parser.add_argument(
'--bionic',
action='store_true',
help='download bionic version of the SDK (linux only).')
options = parser.parse_args(argv)
if options.bionic:
flavor = 'naclsdk_bionic'
else:
flavor = 'naclsdk_' + PLATFORM_COLLAPSE[sys.platform]
url = DetermineSDKURL(flavor,
base_url=GS_URL_BASE,
version=options.version)
stamp_file = os.path.join(TARGET_DIR, 'stamp')
if os.path.exists(stamp_file):
with open(stamp_file) as f:
installed_url = f.read().strip()
if installed_url == url:
naclports.Log('SDK already installed: %s' % url)
return 0
else:
naclports.Log('Ignoring currently installed SDK: %s' %
installed_url)
DownloadAndInstallSDK(url, TARGET_DIR)
with open(stamp_file, 'w') as f:
f.write(url + '\n')
return 0
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
naclports.SetVerbose(True)
count = 0
package_names = [os.path.basename(p.root)
for p in naclports.source_package.SourcePackageIterator()]
for package in naclports.source_package.SourcePackageIterator():
if not package.CheckDeps(package_names):
return 1
count += 1
naclports.Log("Verfied dependencies for %d packages" % count)
return 0
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
naclports.SetVerbose(True)
count = 0
for package in naclports.source_package.SourcePackageIterator():
package.Download()
if not package.Verify():
return 1
count += 1
naclports.Log("Verfied checksums for %d packages" % count)
return 0
def CheckMirror(options, package, mirror_listing):
"""Check that is package has is archive mirrors on Google cloud storage"""
naclports.Trace('Checking %s' % package.NAME)
basename = package.GetArchiveFilename()
if not basename:
return
if basename in mirror_listing:
# already mirrored
return
if options.check:
naclports.Log('update_mirror: Archive missing from mirror: %s' %
basename)
sys.exit(1)
# Download upstream URL
package.Download(force_mirror=False)
url = '%s/%s' % (MIRROR_GS, basename)
GsUpload(options, package.DownloadLocation(), url)
def Install(self):
"""Install binary package into toolchain directory."""
dest = naclports.GetInstallRoot(self.config)
dest_tmp = os.path.join(dest, 'install_tmp')
if os.path.exists(dest_tmp):
shutil.rmtree(dest_tmp)
naclports.Log("Installing '%s' [%s]" % (self.NAME, self.config))
os.makedirs(dest_tmp)
try:
with tarfile.open(self.filename) as tar:
names = []
for info in tar:
if info.isdir():
continue
name = posixpath.normpath(info.name)
if name == 'pkg_info':
continue
if not name.startswith(PAYLOAD_DIR + '/'):
raise naclports.PkgFormatError('invalid file in package: %s' % name)
name = name[len(PAYLOAD_DIR) + 1:]
names.append(name)
tar.extractall(dest_tmp)
payload_tree = os.path.join(dest_tmp, PAYLOAD_DIR)
for name in names:
self.InstallFile(name, payload_tree, dest)
for name in names:
self.RelocateFile(name, dest)
self.WriteFileList(names)
self.WriteStamp()
finally:
shutil.rmtree(dest_tmp)
def rmtree(path):
naclports.Log('removing %s' % path)
if os.path.exists(path):
shutil.rmtree(path)
