def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-n',
'--dry-run',
action='store_true',
help="Don't actually upload anything")
parser.add_argument('--check',
action='store_true',
help='Verify that the mirror is up-to-date.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Enable verbose output.')
options = parser.parse_args(args)
naclports.SetVerbose(options.verbose)
# Ensure gsutil is in the PATH.
bot_gsutil = '/b/build/scripts/slave/gsutil'
if os.path.exists(bot_gsutil):
gsutil = bot_gsutil
else:
gsutil = 'gsutil'
naclports.util.FindInPath(gsutil)
options.gsutil = gsutil
listing = GetMirrorListing(options, MIRROR_GS)
source_packages = naclports.source_package.SourcePackageIterator()
return CheckPackages(options, source_packages, listing)
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('revision',
metavar='REVISION',
help='naclports revision to to scan for.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Output extra information.')
parser.add_argument('-l',
'--cache-listing',
action='store_true',
help='Cached output of gsutil -le (for testing).')
parser.add_argument(
'--skip-md5',
action='store_true',
help='Assume on-disk files are up-to-date (for testing).')
args = parser.parse_args(args)
if args.verbose:
naclports.SetVerbose(True)
sdk_version = naclports.util.GetSDKVersion()
Log('Scanning packages built for pepper_%s at revsion %s' %
(sdk_version, args.revision))
base_path = '%s/builds/pepper_%s/%s/packages' % (
naclports.GS_BUCKET, sdk_version, args.revision)
gs_url = 'gs://' + base_path
gsutil = naclports.util.FindInPath('gsutil.py')
listing_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'listing.txt')
if args.cache_listing and os.path.exists(listing_file):
Log('Using pre-cached gs listing: %s' % listing_file)
with open(listing_file) as f:
listing = f.read()
else:
Log('Searching for packages at: %s' % gs_url)
cmd = [sys.executable, gsutil, 'ls', '-le', gs_url]
LogVerbose('Running: %s' % str(cmd))
try:
listing = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
naclports.Error(e)
return 1
all_files = ParseGsUtilLs(listing)
if args.cache_listing and not os.path.exists(listing_file):
with open(listing_file, 'w') as f:
f.write(listing)
Log('Found %d packages [%s]' %
(len(all_files), FormatSize(sum(f.size for f in all_files))))
binaries = DownloadFiles(all_files, not args.skip_md5)
index_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt')
Log('Generating %s' % index_file)
naclports.package_index.WriteIndex(index_file, binaries)
Log('Done')
return 0
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--check', action='store_true',
help='check canned partition information is up-to-date.')
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
parser.add_argument('-t', '--print-canned', type=int,
help='Print a the canned partition list and exit.')
parser.add_argument('-b', '--bot-prefix', help='builder name prefix.',
default='linux-newlib-')
parser.add_argument('-n', '--num-bots',
help='Number of builders on the waterfall to collect '
'data from or to print a canned partition for.',
type=int, default=3)
parser.add_argument('-p', '--num-parts',
help='Number of parts to partition things into '
'(this will differ from --num-bots when changing the '
'number of shards).',
type=int, default=3)
parser.add_argument('--build-number', help='Builder number to look at for '
'historical data on build times.', type=int, default=-1)
options = parser.parse_args(args)
naclports.SetVerbose(options.verbose)
if options.check:
for num_bots in xrange(1, 6):
print('Checking partioning with %d bot(s)' % (num_bots))
# GetCanned with raise an Error if the canned partition information is
# bad, which in turn will trigger a non-zero return from this script.
GetCanned(0, num_bots)
return
if options.print_canned is not None:
PrintCanned(options.print_canned, options.num_bots)
return
projects = Projects()
for bot in range(options.num_bots):
bot_name = '%s%d' % (options.bot_prefix, bot)
Trace('Attempting to add data from "%s"' % bot_name)
projects.AddDataFromBuilder(bot_name, options.build_number)
projects.PostProcessDeps()
parts = Partition(projects, options.num_parts)
for i, project_times in enumerate(parts):
print('builder %d (total: %d)' % (i, project_times.total_time))
project_names = project_times.TopologicallySortedProjectNames(projects)
print(' %s' % '\n '.join(project_names))
times = list(sorted(part.total_time for part in parts))
difference = 0
for i in range(1, len(times)):
difference += times[i] - times[i - 1]
print('Difference between total time of builders: %d' % difference)
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
naclports.SetVerbose(True)
count = 0
for package in naclports.source_package.SourcePackageIterator():
package.Download()
if not package.Verify():
return 1
count += 1
naclports.Log("Verfied checksums for %d packages" % count)
return 0
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
naclports.SetVerbose(True)
count = 0
package_names = [os.path.basename(p.root)
for p in naclports.source_package.SourcePackageIterator()]
for package in naclports.source_package.SourcePackageIterator():
if not package.CheckDeps(package_names):
return 1
count += 1
naclports.Log("Verfied dependencies for %d packages" % count)
return 0
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('--deps',
action='store_true',
help='include dependencies of effected packages.')
parser.add_argument('files', nargs='+', help='Changes files.')
options = parser.parse_args(args)
naclports.SetVerbose(options.verbose)
if options.deps:
package_filter = sys.stdin.read().split()
else:
package_filter = None
effected_packages = find_effected_packages(options.files, options.deps,
package_filter)
print '\n'.join(effected_packages)
return 0
def main(args):
global options
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
naclports.SetVerbose(True)
rtn = False
count = 0
for package in naclports.source_package.SourcePackageIterator():
rtn |= CheckLicense(package)
count += 1
if not rtn:
print("Verfied licenses for %d packages" % count)
return rtn
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-n',
'--dry-run',
action='store_true',
help="Don't actually upload anything")
parser.add_argument('--check',
action='store_true',
help='Verify that the mirror is up-to-date.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Enable verbose output.')
options = parser.parse_args(args)
naclports.SetVerbose(options.verbose)
# gsutil.py should be in PATH since its part of depot_tools.
options.gsutil = [sys.executable, naclports.util.FindInPath('gsutil.py')]
listing = GetMirrorListing(options, MIRROR_GS)
source_packages = naclports.source_package.SourcePackageIterator()
return CheckPackages(options, source_packages, listing)
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('revision',
metavar='REVISION',
help='naclports revision to to scan for.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Output extra information.')
parser.add_argument('-p',
'--parallel',
action='store_true',
help='Download packages in parallel.')
parser.add_argument('-l',
'--cache-listing',
action='store_true',
help='Cached output of gsutil -L (for testing).')
parser.add_argument(
'--skip-md5',
action='store_true',
help='Assume on-disk files are up-to-date (for testing).')
args = parser.parse_args(args)
if args.verbose:
naclports.SetVerbose(True)
sdk_version = naclports.util.GetSDKVersion()
Log('Scanning packages built for pepper_%s at revsion %s' %
(sdk_version, args.revision))
base_path = '%s/builds/pepper_%s/%s/publish' % (naclports.GS_BUCKET,
sdk_version, args.revision)
gs_base_url = 'gs://' + base_path
cmd = FindGsutil() + ['ls', gs_base_url]
LogVerbose('Running: %s' % str(cmd))
try:
all_published = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise naclports.Error("Command '%s' failed: %s" % (cmd, e))
pkg_dir = re.findall(r'pkg_[\w-]+', all_published)
for pkg in pkg_dir:
listing_file = os.path.join(naclports.NACLPORTS_ROOT, 'lib',
pkg + '_' + 'listing.txt')
if args.cache_listing and os.path.exists(listing_file):
Log('Using pre-cached gs listing: %s' % listing_file)
with open(listing_file) as f:
listing = f.read()
else:
gs_url = gs_base_url + '/' + pkg + '/*'
Log('Searching for packages at: %s' % gs_url)
cmd = FindGsutil() + ['stat', gs_url]
LogVerbose('Running: %s' % str(cmd))
try:
listing = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise naclports.Error("Command '%s' failed: %s" % (cmd, e))
if args.cache_listing:
with open(listing_file, 'w') as f:
f.write(listing)
all_files = ParseGsUtilOutput(listing)
Log('Found %d packages [%s] for %s' %
(len(all_files), FormatSize(sum(f.size for f in all_files)), pkg))
DownloadFiles(pkg, all_files, not args.skip_md5, args.parallel)
Log('Done')
return 0