def main(args): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('revision', metavar='REVISION', help='webports revision to to scan for.') parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') parser.add_argument('-p', '--parallel', action='store_true', help='Download packages in parallel.') parser.add_argument('-l', '--cache-listing', action='store_true', help='Cached output of gsutil -L (for testing).') parser.add_argument( '--skip-md5', action='store_true', help='Assume on-disk files are up-to-date (for testing).') args = parser.parse_args(args) if args.verbose: webports.set_verbose(True) sdk_version = webports.util.get_sdk_version() log('Scanning packages built for pepper_%s at revsion %s' % (sdk_version, args.revision)) base_path = '%s/builds/pepper_%s/%s/packages' % ( webports.GS_BUCKET, sdk_version, args.revision) gs_url = 'gs://' + base_path + '/*' listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', 'listing.txt') if args.cache_listing and os.path.exists(listing_file): log('Using pre-cached gs listing: %s' % listing_file) with open(listing_file) as f: listing = f.read() else: log('Searching for packages at: %s' % gs_url) cmd = find_gsutil() + ['stat', gs_url] log_verbose('Running: %s' % str(cmd)) try: listing = subprocess.check_output(cmd) except subprocess.CalledProcessError as e: raise webports.Error("Command '%s' failed: %s" % (cmd, e)) if args.cache_listing: with open(listing_file, 'w') as f: f.write(listing) all_files = parse_gs_util_output(listing) log('Found %d packages [%s]' % (len(all_files), format_size(sum(f.size for f in all_files)))) binaries = download_files(all_files, not args.skip_md5, args.parallel) index_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', 'prebuilt.txt') log('Generating %s' % index_file) webports.package_index.write_index(index_file, binaries) log('Done') return 0
def main(args): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('revision', metavar='REVISION', help='webports revision to to scan for.') parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') parser.add_argument('-p', '--parallel', action='store_true', help='Download packages in parallel.') parser.add_argument('-l', '--cache-listing', action='store_true', help='Cached output of gsutil -L (for testing).') parser.add_argument('--skip-md5', action='store_true', help='Assume on-disk files are up-to-date (for testing).') args = parser.parse_args(args) if args.verbose: webports.set_verbose(True) sdk_version = webports.util.get_sdk_version() log('Scanning packages built for pepper_%s at revsion %s' % (sdk_version, args.revision)) base_path = '%s/builds/pepper_%s/%s/publish' % (webports.GS_BUCKET, sdk_version, args.revision) gs_base_url = 'gs://' + base_path cmd = find_gsutil() + ['ls', gs_base_url] log_verbose('Running: %s' % str(cmd)) try: all_published = subprocess.check_output(cmd) except subprocess.CalledProcessError as e: raise webports.Error("Command '%s' failed: %s" % (cmd, e)) pkg_dir = re.findall(r'pkg_[\w-]+', all_published) for pkg in pkg_dir: listing_file = os.path.join(webports.NACLPORTS_ROOT, 'lib', pkg + '_' + 'listing.txt') if args.cache_listing and os.path.exists(listing_file): log('Using pre-cached gs listing: %s' % listing_file) with open(listing_file) as f: listing = f.read() else: gs_url = gs_base_url + '/' + pkg + '/*' log('Searching for packages at: %s' % gs_url) cmd = find_gsutil() + ['stat', gs_url] log_verbose('Running: %s' % str(cmd)) try: listing = subprocess.check_output(cmd) except subprocess.CalledProcessError as e: raise webports.Error("Command '%s' failed: %s" % (cmd, e)) if args.cache_listing: with open(listing_file, 'w') as f: f.write(listing) all_files = parse_gs_util_output(listing) log('Found %d packages [%s] for %s' % (len(all_files), format_size(sum(f.size for f in all_files)), pkg)) download_files(pkg, all_files, not args.skip_md5, args.parallel) log('Done') return 0
def main(args): parser = argparse.ArgumentParser() parser.add_argument('--check', action='store_true', help='check canned partition information is up-to-date.') parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') parser.add_argument('-t', '--print-canned', type=int, help='Print a the canned partition list and exit.') parser.add_argument('-b', '--bot-prefix', help='builder name prefix.', default='linux-clang-') parser.add_argument('-n', '--num-bots', help='Number of builders on the waterfall to collect ' 'data from or to print a canned partition for.', type=int, default=5) parser.add_argument('-p', '--num-parts', help='Number of parts to partition things into ' '(this will differ from --num-bots when changing the ' 'number of shards).', type=int, default=5) parser.add_argument('--build-number', help='Builder number to look at for ' 'historical data on build times.', type=int, default=-1) options = parser.parse_args(args) webports.set_verbose(options.verbose) if options.check: for num_bots in xrange(1, 7): print('Checking partioning with %d bot(s)' % (num_bots)) # GetCanned with raise an Error if the canned partition information is # bad, which in turn will trigger a non-zero return from this script. get_canned(0, num_bots) return if options.print_canned is not None: print_canned(options.print_canned, options.num_bots) return projects = Projects() for bot in range(options.num_bots): bot_name = '%s%d' % (options.bot_prefix, bot) log_verbose('Attempting to add data from "%s"' % bot_name) projects.add_data_from_builder(bot_name, options.build_number) projects.post_process_deps() parts = get_partition(projects, options.num_parts) for i, project_times in enumerate(parts): print('builder %d (total: %d)' % (i, project_times.total_time)) project_names = project_times.topologically_sorted_project_names(projects) print(' %s' % '\n '.join(project_names)) times = list(sorted(part.total_time for part in parts)) difference = 0 for i in range(1, len(times)): difference += times[i] - times[i - 1] print('Difference between total time of builders: %d' % difference)
def main(args): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') options = parser.parse_args(args) if options.verbose: webports.set_verbose(True) count = 0 for package in webports.source_package.source_package_iterator(): package.download() count += 1 webports.log("Verfied checksums for %d packages" % count) return 0
def main(args): parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') options = parser.parse_args(args) if options.verbose: webports.set_verbose(True) count = 0 package_names = [os.path.basename(p.root) for p in webports.source_package.source_package_iterator()] for package in webports.source_package.source_package_iterator(): if not package.check_deps(package_names): return 1 count += 1 webports.log("Verfied dependencies for %d packages" % count) return 0
def main(args): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-n', '--dry-run', action='store_true', help="Don't actually upload anything") parser.add_argument('--check', action='store_true', help='Verify that the mirror is up-to-date.') parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output.') options = parser.parse_args(args) webports.set_verbose(options.verbose) # gsutil.py should be in PATH since its part of depot_tools. options.gsutil = [sys.executable, webports.util.find_in_path('gsutil.py')] listing = get_mirror_listing(options, MIRROR_GS) source_packages = webports.source_package.source_package_iterator() return check_packages(options, source_packages, listing)
def main(args): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-v', '--verbose', action='store_true') parser.add_argument('--deps', action='store_true', help='include dependencies of effected packages.') parser.add_argument('files', nargs='+', help='Changes files.') options = parser.parse_args(args) webports.set_verbose(options.verbose) if options.deps: package_filter = sys.stdin.read().split() else: package_filter = None effected_packages = find_effected_packages(options.files, options.deps, package_filter) print '\n'.join(effected_packages) return 0
def main(args): global options parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') options = parser.parse_args(args) if options.verbose: webports.set_verbose(True) rtn = False count = 0 for package in webports.source_package.source_package_iterator(): rtn |= check_license(package) count += 1 if not rtn: print("Verfied licenses for %d packages" % count) return rtn
def main(args): parser = argparse.ArgumentParser() parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') options = parser.parse_args(args) if options.verbose: webports.set_verbose(True) count = 0 package_names = [ os.path.basename(p.root) for p in webports.source_package.source_package_iterator() ] for package in webports.source_package.source_package_iterator(): if not package.check_deps(package_names): return 1 count += 1 webports.log("Verfied dependencies for %d packages" % count) return 0
def main(args): parser = argparse.ArgumentParser() parser.add_argument( '--check', action='store_true', help='check canned partition information is up-to-date.') parser.add_argument('-v', '--verbose', action='store_true', help='Output extra information.') parser.add_argument('-t', '--print-canned', type=int, help='Print a the canned partition list and exit.') parser.add_argument('-b', '--bot-prefix', help='builder name prefix.', default='linux-clang-') parser.add_argument('-n', '--num-bots', help='Number of builders on the waterfall to collect ' 'data from or to print a canned partition for.', type=int, default=5) parser.add_argument('-p', '--num-parts', help='Number of parts to partition things into ' '(this will differ from --num-bots when changing the ' 'number of shards).', type=int, default=5) parser.add_argument('--build-number', help='Builder number to look at for ' 'historical data on build times.', type=int, default=-1) options = parser.parse_args(args) webports.set_verbose(options.verbose) if options.check: for num_bots in xrange(1, 7): print('Checking partioning with %d bot(s)' % (num_bots)) # GetCanned with raise an Error if the canned partition information is # bad, which in turn will trigger a non-zero return from this script. get_canned(0, num_bots) return if options.print_canned is not None: print_canned(options.print_canned, options.num_bots) return projects = Projects() for bot in range(options.num_bots): bot_name = '%s%d' % (options.bot_prefix, bot) log_verbose('Attempting to add data from "%s"' % bot_name) projects.add_data_from_builder(bot_name, options.build_number) projects.post_process_deps() parts = get_partition(projects, options.num_parts) for i, project_times in enumerate(parts): print('builder %d (total: %d)' % (i, project_times.total_time)) project_names = project_times.topologically_sorted_project_names( projects) print(' %s' % '\n '.join(project_names)) times = list(sorted(part.total_time for part in parts)) difference = 0 for i in range(1, len(times)): difference += times[i] - times[i - 1] print('Difference between total time of builders: %d' % difference)