def Build(self, build_deps, force=None):
self.CheckBuildable()
if build_deps:
self.InstallDeps(force)
if not force and self.IsBuilt():
Log("Already built '%s' [%s]" % (self.NAME, self.config))
return
log_root = os.path.join(OUT_DIR, 'logs')
if not os.path.isdir(log_root):
os.makedirs(log_root)
stdout = os.path.join(log_root, '%s.log' % self.NAME)
if os.path.exists(stdout):
os.remove(stdout)
if naclports.verbose:
prefix = '*** '
else:
prefix = ''
Log("%sBuilding '%s' [%s]" % (prefix, self.NAME, self.config))
start = time.time()
self.RunBuildSh(stdout)
duration = FormatTimeDelta(time.time() - start)
Log("Build complete '%s' [%s] [took %s]" %
(self.NAME, self.config, duration))
def Uninstall(self, ignore_missing):
if not os.path.exists(naclports.GetInstallStamp(
self.NAME, self.config)):
if ignore_missing:
return
raise Error('Package not installed: %s [%s]' %
(self.NAME, self.config))
Log("Uninstalling '%s' [%s]" % (self.NAME, self.config))
file_list = naclports.GetFileList(self.NAME, self.config)
if not os.path.exists(file_list):
Log('No files to uninstall')
else:
root = naclports.GetInstallRoot(self.config)
with open(file_list) as f:
for line in f:
filename = os.path.join(root, line.strip())
if not os.path.lexists(filename):
Warn('File not found while uninstalling: %s' %
filename)
continue
Trace('rm %s' % filename)
os.remove(filename)
RemoveEmptyDirs(os.path.dirname(filename))
os.remove(file_list)
os.remove(naclports.GetInstallStamp(self.NAME, self.config))
def Clean(self):
pkg = self.PackageFile()
Log('removing %s' % pkg)
if os.path.exists(pkg):
os.remove(pkg)
stamp_dir = os.path.join(naclports.STAMP_DIR, self.NAME)
Log('removing %s' % stamp_dir)
if os.path.exists(stamp_dir):
shutil.rmtree(stamp_dir)
def CheckDeps(self, valid_packages):
for package in self.DEPENDS:
if package not in valid_packages:
Log('%s: Invalid dependency: %s' % (self.info, package))
return False
for package in self.CONFLICTS:
if package not in valid_packages:
Log('%s: Invalid conflict: %s' % (self.info, package))
return False
return True
def ExtractInto(self, output_path):
"""Extract the package archive into the given location.
This method assumes the package has already been downloaded.
"""
archive = self.DownloadLocation()
if not archive:
return
if not os.path.exists(output_path):
os.makedirs(output_path)
new_foldername = os.path.basename(self.GetBuildLocation())
dest = os.path.join(output_path, new_foldername)
if os.path.exists(dest):
Trace('Already exists: %s' % dest)
return
tmp_output_path = tempfile.mkdtemp(dir=OUT_DIR)
try:
ext = os.path.splitext(archive)[1]
if ext in ('.gz', '.tgz', '.bz2'):
cmd = ['tar', 'xf', archive, '-C', tmp_output_path]
elif ext in ('.zip', ):
cmd = ['unzip', '-q', '-d', tmp_output_path, archive]
else:
raise Error('unhandled extension: %s' % ext)
Log("Extracting '%s'" % self.NAME)
Trace(cmd)
subprocess.check_call(cmd)
src = os.path.join(tmp_output_path, new_foldername)
os.rename(src, dest)
finally:
shutil.rmtree(tmp_output_path)
def Install(self, build_deps, force=None, from_source=False):
self.CheckInstallable()
if force is None and self.IsInstalled():
Log("Already installed '%s' [%s]" % (self.NAME, self.config))
return
if build_deps:
self.InstallDeps(force, from_source)
if force:
from_source = True
package_file = self.PackageFile()
if not self.IsBuilt() and not from_source:
index = package_index.GetCurrentIndex()
if index.Installable(self.NAME, self.config):
package_file = index.Download(self.NAME, self.config)
else:
from_source = True
if from_source:
self.Build(build_deps, force)
binary_package.BinaryPackage(package_file).Install()
def InstallDeps(self, force, from_source=False):
for dep in self.DEPENDS:
if not naclports.IsInstalled(dep, self.config) or force == 'all':
dep = CreatePackage(dep, self.config)
try:
dep.Install(True, force, from_source)
except DisabledError as e:
Log(str(e))
def DownloadFiles(files, check_hashes=True):
"""Download one of more files to the local disk.
Args:
files: List of FileInfo objects to download.
check_hashes: When False assume local files have the correct
hash otherwise always check the hashes match the onces in the
FileInfo ojects.
Returns:
List of (filename, url) tuples.
"""
files_to_download = []
filenames = []
download_dir = naclports.package_index.PREBUILT_ROOT
if not os.path.exists(download_dir):
os.makedirs(download_dir)
for file_info in files:
basename = os.path.basename(file_info.url)
fullname = os.path.join(download_dir, basename)
filenames.append((fullname, file_info.url))
if os.path.exists(fullname):
if not check_hashes or CheckHash(fullname, file_info.etag):
Log('Up-to-date: %s' % file_info.name)
continue
files_to_download.append(FileInfo(fullname, file_info.size, file_info.url,
file_info.etag))
if not files_to_download:
Log('All files up-to-date')
else:
total_size = sum(f[1] for f in files_to_download)
Log('Need to download %d/%d files [%s]' % (len(files_to_download),
len(files), FormatSize(total_size)))
for file_info in files_to_download:
naclports.DownloadFile(file_info.name, file_info.url)
if check_hashes and not CheckHash(file_info.name, file_info.etag):
raise naclports.Error('Checksum failed: %s' % file_info.name)
return filenames
def Verify(self):
"""Download upstream source and verify hash."""
archive = self.DownloadLocation()
if not archive:
Log("no archive: %s" % self.NAME)
return True
if self.SHA1 is None:
Log("missing SHA1 attribute: %s" % self.info)
return False
self.Download()
try:
sha1check.VerifyHash(archive, self.SHA1)
Log("verified: %s" % archive)
except sha1check.Error as e:
Log("verification failed: %s: %s" % (archive, str(e)))
return False
return True
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('revision', metavar='REVISION',
help='naclports revision to to scan for.')
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
parser.add_argument('-l', '--cache-listing', action='store_true',
help='Cached output of gsutil -le (for testing).')
parser.add_argument('--skip-md5', action='store_true',
help='Assume on-disk files are up-to-date (for testing).')
args = parser.parse_args(args)
if args.verbose:
naclports.SetVerbose(True)
sdk_version = naclports.GetSDKVersion()
Log('Scanning packages built for pepper_%s at revsion %s' %
(sdk_version, args.revision))
base_path = '%s/builds/pepper_%s/%s/packages' % (naclports.GS_BUCKET,
sdk_version,
args.revision)
gs_url = 'gs://' + base_path
listing_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'listing.txt')
if args.cache_listing and os.path.exists(listing_file):
Log('Using pre-cached gs listing: %s' % listing_file)
with open(listing_file) as f:
listing = f.read()
else:
Log("Searching for packages at: %s" % gs_url)
cmd = ['gsutil', 'ls', '-le', gs_url]
Trace("Running: %s" % str(cmd))
try:
listing = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
naclports.Error(e)
return 1
all_files = ParseGsUtilLs(listing)
if args.cache_listing and not os.path.exists(listing_file):
with open(listing_file, 'w') as f:
f.write(listing)
Log('Found %d packages [%s]' % (len(all_files),
FormatSize(sum(f.size for f in all_files))))
binaries = DownloadFiles(all_files, not args.skip_md5)
index_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt')
Log('Generating %s' % index_file)
naclports.package_index.WriteIndex(index_file, binaries)
Log('Done')
return 0