def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('revision',
metavar='REVISION',
help='webports revision to to scan for.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Output extra information.')
parser.add_argument('-p',
'--parallel',
action='store_true',
help='Download packages in parallel.')
parser.add_argument('-l',
'--cache-listing',
action='store_true',
help='Cached output of gsutil -L (for testing).')
parser.add_argument(
'--skip-md5',
action='store_true',
help='Assume on-disk files are up-to-date (for testing).')
args = parser.parse_args(args)
if args.verbose:
webports.set_verbose(True)
sdk_version = webports.util.get_sdk_version()
log('Scanning packages built for pepper_%s at revsion %s' %
(sdk_version, args.revision))
base_path = '%s/builds/pepper_%s/%s/packages' % (
webports.GS_BUCKET, sdk_version, args.revision)
gs_url = 'gs://' + base_path + '/*'
listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', 'listing.txt')
if args.cache_listing and os.path.exists(listing_file):
log('Using pre-cached gs listing: %s' % listing_file)
with open(listing_file) as f:
listing = f.read()
else:
log('Searching for packages at: %s' % gs_url)
cmd = find_gsutil() + ['stat', gs_url]
log_verbose('Running: %s' % str(cmd))
try:
listing = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise webports.Error("Command '%s' failed: %s" % (cmd, e))
if args.cache_listing:
with open(listing_file, 'w') as f:
f.write(listing)
all_files = parse_gs_util_output(listing)
log('Found %d packages [%s]' %
(len(all_files), format_size(sum(f.size for f in all_files))))
binaries = download_files(all_files, not args.skip_md5, args.parallel)
index_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt')
log('Generating %s' % index_file)
webports.package_index.write_index(index_file, binaries)
log('Done')
return 0
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('revision', metavar='REVISION',
help='webports revision to to scan for.')
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
parser.add_argument('-p', '--parallel', action='store_true',
help='Download packages in parallel.')
parser.add_argument('-l', '--cache-listing', action='store_true',
help='Cached output of gsutil -L (for testing).')
parser.add_argument('--skip-md5', action='store_true',
help='Assume on-disk files are up-to-date (for testing).')
args = parser.parse_args(args)
if args.verbose:
webports.set_verbose(True)
sdk_version = webports.util.get_sdk_version()
log('Scanning packages built for pepper_%s at revsion %s' %
(sdk_version, args.revision))
base_path = '%s/builds/pepper_%s/%s/publish' % (webports.GS_BUCKET,
sdk_version, args.revision)
gs_base_url = 'gs://' + base_path
cmd = find_gsutil() + ['ls', gs_base_url]
log_verbose('Running: %s' % str(cmd))
try:
all_published = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise webports.Error("Command '%s' failed: %s" % (cmd, e))
pkg_dir = re.findall(r'pkg_[\w-]+', all_published)
for pkg in pkg_dir:
listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib',
pkg + '_' + 'listing.txt')
if args.cache_listing and os.path.exists(listing_file):
log('Using pre-cached gs listing: %s' % listing_file)
with open(listing_file) as f:
listing = f.read()
else:
gs_url = gs_base_url + '/' + pkg + '/*'
log('Searching for packages at: %s' % gs_url)
cmd = find_gsutil() + ['stat', gs_url]
log_verbose('Running: %s' % str(cmd))
try:
listing = subprocess.check_output(cmd)
except subprocess.CalledProcessError as e:
raise webports.Error("Command '%s' failed: %s" % (cmd, e))
if args.cache_listing:
with open(listing_file, 'w') as f:
f.write(listing)
all_files = parse_gs_util_output(listing)
log('Found %d packages [%s] for %s' %
(len(all_files), format_size(sum(f.size for f in all_files)), pkg))
download_files(pkg, all_files, not args.skip_md5, args.parallel)
log('Done')
return 0
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('--check', action='store_true',
help='check canned partition information is up-to-date.')
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
parser.add_argument('-t', '--print-canned', type=int,
help='Print a the canned partition list and exit.')
parser.add_argument('-b', '--bot-prefix', help='builder name prefix.',
default='linux-clang-')
parser.add_argument('-n', '--num-bots',
help='Number of builders on the waterfall to collect '
'data from or to print a canned partition for.',
type=int, default=5)
parser.add_argument('-p', '--num-parts',
help='Number of parts to partition things into '
'(this will differ from --num-bots when changing the '
'number of shards).',
type=int, default=5)
parser.add_argument('--build-number', help='Builder number to look at for '
'historical data on build times.', type=int, default=-1)
options = parser.parse_args(args)
webports.set_verbose(options.verbose)
if options.check:
for num_bots in xrange(1, 7):
print('Checking partioning with %d bot(s)' % (num_bots))
# GetCanned with raise an Error if the canned partition information is
# bad, which in turn will trigger a non-zero return from this script.
get_canned(0, num_bots)
return
if options.print_canned is not None:
print_canned(options.print_canned, options.num_bots)
return
projects = Projects()
for bot in range(options.num_bots):
bot_name = '%s%d' % (options.bot_prefix, bot)
log_verbose('Attempting to add data from "%s"' % bot_name)
projects.add_data_from_builder(bot_name, options.build_number)
projects.post_process_deps()
parts = get_partition(projects, options.num_parts)
for i, project_times in enumerate(parts):
print('builder %d (total: %d)' % (i, project_times.total_time))
project_names = project_times.topologically_sorted_project_names(projects)
print(' %s' % '\n '.join(project_names))
times = list(sorted(part.total_time for part in parts))
difference = 0
for i in range(1, len(times)):
difference += times[i] - times[i - 1]
print('Difference between total time of builders: %d' % difference)
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
webports.set_verbose(True)
count = 0
for package in webports.source_package.source_package_iterator():
package.download()
count += 1
webports.log("Verfied checksums for %d packages" % count)
return 0
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
webports.set_verbose(True)
count = 0
for package in webports.source_package.source_package_iterator():
package.download()
count += 1
webports.log("Verfied checksums for %d packages" % count)
return 0
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
webports.set_verbose(True)
count = 0
package_names = [os.path.basename(p.root)
for p in webports.source_package.source_package_iterator()]
for package in webports.source_package.source_package_iterator():
if not package.check_deps(package_names):
return 1
count += 1
webports.log("Verfied dependencies for %d packages" % count)
return 0
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-n', '--dry-run', action='store_true',
help="Don't actually upload anything")
parser.add_argument('--check', action='store_true',
help='Verify that the mirror is up-to-date.')
parser.add_argument('-v', '--verbose', action='store_true',
help='Enable verbose output.')
options = parser.parse_args(args)
webports.set_verbose(options.verbose)
# gsutil.py should be in PATH since its part of depot_tools.
options.gsutil = [sys.executable, webports.util.find_in_path('gsutil.py')]
listing = get_mirror_listing(options, MIRROR_GS)
source_packages = webports.source_package.source_package_iterator()
return check_packages(options, source_packages, listing)
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-n', '--dry-run', action='store_true',
help="Don't actually upload anything")
parser.add_argument('--check', action='store_true',
help='Verify that the mirror is up-to-date.')
parser.add_argument('-v', '--verbose', action='store_true',
help='Enable verbose output.')
options = parser.parse_args(args)
webports.set_verbose(options.verbose)
# gsutil.py should be in PATH since its part of depot_tools.
options.gsutil = [sys.executable, webports.util.find_in_path('gsutil.py')]
listing = get_mirror_listing(options, MIRROR_GS)
source_packages = webports.source_package.source_package_iterator()
return check_packages(options, source_packages, listing)
def main(args):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('-v', '--verbose', action='store_true')
parser.add_argument('--deps', action='store_true',
help='include dependencies of effected packages.')
parser.add_argument('files', nargs='+', help='Changes files.')
options = parser.parse_args(args)
webports.set_verbose(options.verbose)
if options.deps:
package_filter = sys.stdin.read().split()
else:
package_filter = None
effected_packages = find_effected_packages(options.files, options.deps,
package_filter)
print '\n'.join(effected_packages)
return 0
def main(args):
global options
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
webports.set_verbose(True)
rtn = False
count = 0
for package in webports.source_package.source_package_iterator():
rtn |= check_license(package)
count += 1
if not rtn:
print("Verfied licenses for %d packages" % count)
return rtn
def main(args):
global options
parser = argparse.ArgumentParser()
parser.add_argument('-v', '--verbose', action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
webports.set_verbose(True)
rtn = False
count = 0
for package in webports.source_package.source_package_iterator():
rtn |= check_license(package)
count += 1
if not rtn:
print("Verfied licenses for %d packages" % count)
return rtn
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Output extra information.')
options = parser.parse_args(args)
if options.verbose:
webports.set_verbose(True)
count = 0
package_names = [
os.path.basename(p.root)
for p in webports.source_package.source_package_iterator()
]
for package in webports.source_package.source_package_iterator():
if not package.check_deps(package_names):
return 1
count += 1
webports.log("Verfied dependencies for %d packages" % count)
return 0
def main(args):
parser = argparse.ArgumentParser()
parser.add_argument(
'--check',
action='store_true',
help='check canned partition information is up-to-date.')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Output extra information.')
parser.add_argument('-t',
'--print-canned',
type=int,
help='Print a the canned partition list and exit.')
parser.add_argument('-b',
'--bot-prefix',
help='builder name prefix.',
default='linux-clang-')
parser.add_argument('-n',
'--num-bots',
help='Number of builders on the waterfall to collect '
'data from or to print a canned partition for.',
type=int,
default=5)
parser.add_argument('-p',
'--num-parts',
help='Number of parts to partition things into '
'(this will differ from --num-bots when changing the '
'number of shards).',
type=int,
default=5)
parser.add_argument('--build-number',
help='Builder number to look at for '
'historical data on build times.',
type=int,
default=-1)
options = parser.parse_args(args)
webports.set_verbose(options.verbose)
if options.check:
for num_bots in xrange(1, 7):
print('Checking partioning with %d bot(s)' % (num_bots))
# GetCanned with raise an Error if the canned partition information is
# bad, which in turn will trigger a non-zero return from this script.
get_canned(0, num_bots)
return
if options.print_canned is not None:
print_canned(options.print_canned, options.num_bots)
return
projects = Projects()
for bot in range(options.num_bots):
bot_name = '%s%d' % (options.bot_prefix, bot)
log_verbose('Attempting to add data from "%s"' % bot_name)
projects.add_data_from_builder(bot_name, options.build_number)
projects.post_process_deps()
parts = get_partition(projects, options.num_parts)
for i, project_times in enumerate(parts):
print('builder %d (total: %d)' % (i, project_times.total_time))
project_names = project_times.topologically_sorted_project_names(
projects)
print(' %s' % '\n '.join(project_names))
times = list(sorted(part.total_time for part in parts))
difference = 0
for i in range(1, len(times)):
difference += times[i] - times[i - 1]
print('Difference between total time of builders: %d' % difference)
