def archive_repo(languages): """Archives local version of repo including submodules.""" # Directory contains symlinks that can't be correctly untarred on Windows # so we just skip them as a workaround. # See https://github.com/grpc/grpc/issues/16334 bad_symlinks_dir = '../grpc/third_party/libcxx/test/std/experimental/filesystem/Inputs/static_test_env' cmdline = [ 'tar', '--exclude', bad_symlinks_dir, '-cf', '../grpc.tar', '../grpc/' ] if 'java' in languages: cmdline.append('../grpc-java') if 'go' in languages: cmdline.append('../grpc-go') if 'node' in languages or 'node_purejs' in languages: cmdline.append('../grpc-node') archive_job = jobset.JobSpec( cmdline=cmdline, shortname='archive_repo', timeout_seconds=3 * 60) jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run( [archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message( 'SUCCESS', 'Archive with local repository created successfully.', do_newline=True) else: jobset.message( 'FAILED', 'Failed to archive local repository.', do_newline=True) sys.exit(1)
def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), build_local=False): """Builds performance worker on remote hosts (and maybe also locally).""" build_timeout = 15*60 build_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) build_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/remote_host_build.sh'] + languages, shortname='remote_host_build.%s' % host, environ = {'USER_AT_HOST': user_at_host, 'CONFIG': 'opt'}, timeout_seconds=build_timeout)) if build_local: # Build locally as well build_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/build_performance.sh'] + languages, shortname='local_build', environ = {'CONFIG': 'opt'}, timeout_seconds=build_timeout)) jobset.message('START', 'Building.', do_newline=True) num_failures, _ = jobset.run( build_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Built successfully.', do_newline=True) else: jobset.message('FAILED', 'Build failed.', do_newline=True) sys.exit(1)
def prepare_remote_hosts(hosts, prepare_local=False): """Prepares remote hosts (and maybe prepare localhost as well).""" prepare_timeout = 5*60 prepare_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) prepare_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/remote_host_prepare.sh'], shortname='remote_host_prepare.%s' % host, environ = {'USER_AT_HOST': user_at_host}, timeout_seconds=prepare_timeout)) if prepare_local: # Prepare localhost as well prepare_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/kill_workers.sh'], shortname='local_prepare', timeout_seconds=prepare_timeout)) jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Prepare step completed successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to prepare remote hosts.', do_newline=True) sys.exit(1)
def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name): perf_report_jobs = [] global profile_output_files for host_and_port in hosts_and_base_names: perf_base_name = hosts_and_base_names[host_and_port] output_filename = '%s-%s' % (scenario_name, perf_base_name) # from the base filename, create .svg output filename host = host_and_port.split(':')[0] profile_output_files.append('%s.svg' % output_filename) perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename)) jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1) jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) return failures
def archive_repo(languages): """Archives local version of repo including submodules.""" cmdline = ['tar', '-cf', '../grpc.tar', '../grpc/'] if 'java' in languages: cmdline.append('../grpc-java') if 'go' in languages: cmdline.append('../grpc-go') archive_job = jobset.JobSpec( cmdline=cmdline, shortname='archive_repo', timeout_seconds=3 * 60) jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run( [archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message( 'SUCCESS', 'Archive with local repository created successfully.', do_newline=True) else: jobset.message( 'FAILED', 'Failed to archive local repository.', do_newline=True) sys.exit(1)
qps_workers_killed = 0 merged_resultset = {} perf_report_failures = 0 for scenario in scenarios: if args.dry_run: print(scenario.name) else: scenario_failures = 0 try: for worker in scenario.workers: worker.start() jobs = [scenario.jobspec] if scenario.workers: jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset), six.iteritems(resultset))) finally: # Consider qps workers that need to be killed as failures qps_workers_killed += finish_qps_workers(scenario.workers) if perf_cmd and scenario_failures == 0 and not args.skip_generate_flamegraphs: workers_and_base_names = {} for worker in scenario.workers: if not worker.perf_file_base_name: raise Exception('using perf buf perf report filename is unspecified') workers_and_base_names[worker.host_and_port] = worker.perf_file_base_name perf_report_failures += run_collect_perf_profile_jobs(workers_and_base_names, scenario.name)
def collect_latency(bm_name, args): """generate latency profiles""" benchmarks = [] profile_analysis = [] cleanup = [] heading('Latency Profiles: %s' % bm_name) subprocess.check_call( ['make', bm_name, 'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()]) for line in subprocess.check_output(['bins/basicprof/%s' % bm_name, '--benchmark_list_tests']).splitlines(): link(line, '%s.txt' % fnize(line)) benchmarks.append( jobset.JobSpec(['bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=0.05'], environ={'LATENCY_TRACE': '%s.trace' % fnize(line)})) profile_analysis.append( jobset.JobSpec([sys.executable, 'tools/profiling/latency_profile/profile_analyzer.py', '--source', '%s.trace' % fnize(line), '--fmt', 'simple', '--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None)) cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)])) # periodically flush out the list of jobs: profile_analysis jobs at least # consume upwards of five gigabytes of ram in some cases, and so analysing # hundreds of them at once is impractical -- but we want at least some # concurrency or the work takes too long if len(benchmarks) >= min(16, multiprocessing.cpu_count()): # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2)) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) benchmarks = [] profile_analysis = [] cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2)) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
def collect_perf(bm_name, args): """generate flamegraphs""" heading('Flamegraphs: %s' % bm_name) subprocess.check_call( ['make', bm_name, 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()]) benchmarks = [] profile_analysis = [] cleanup = [] for line in subprocess.check_output(['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines(): link(line, '%s.svg' % fnize(line)) benchmarks.append( jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line), '-g', '-F', '997', 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10'])) profile_analysis.append( jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'], environ = { 'PERF_BASE_NAME': fnize(line), 'OUTPUT_DIR': 'reports', 'OUTPUT_FILENAME': fnize(line), })) cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)])) cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)])) # periodically flush out the list of jobs: temporary space required for this # processing is large if len(benchmarks) >= 20: # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) jobset.run(benchmarks, maxjobs=1) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) benchmarks = [] profile_analysis = [] cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): jobset.run(benchmarks, maxjobs=1) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
build_jobs = [] for l in languages_to_build: if str(l) == 'objc': # we don't need to build a docker image for objc continue job = build_interop_image_jobspec(l) docker_images[str(l)] = job.tag build_jobs.append(job) if build_jobs: jobset.message('START', 'Building interop docker images.', do_newline=True) if args.verbose: print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) num_failures, _ = jobset.run( build_jobs, newline_on_success=True, maxjobs=args.jobs) if num_failures == 0: jobset.message('SUCCESS', 'All docker images built successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) for image in six.itervalues(docker_images): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) server_manual_cmd_log = [] if args.manual_run else None client_manual_cmd_log = [] if args.manual_run else None # Start interop servers. server_jobs = {}
def main(): argp = argparse.ArgumentParser(description='Run performance tests.') argp.add_argument('-l', '--language', choices=['all'] + sorted(scenario_config.LANGUAGES.keys()), nargs='+', required=True, help='Languages to benchmark.') argp.add_argument('--remote_driver_host', default=None, help='Run QPS driver on given host. By default, QPS driver is run locally.') argp.add_argument('--remote_worker_host', nargs='+', default=[], help='Worker hosts where to start QPS workers.') argp.add_argument('--dry_run', default=False, action='store_const', const=True, help='Just list scenarios to be run, but don\'t run them.') argp.add_argument('-r', '--regex', default='.*', type=str, help='Regex to select scenarios to run.') argp.add_argument('--bq_result_table', default=None, type=str, help='Bigquery "dataset.table" to upload results to.') argp.add_argument('--category', choices=['smoketest','all','scalable','sweep'], default='all', help='Select a category of tests to run.') argp.add_argument('--netperf', default=False, action='store_const', const=True, help='Run netperf benchmark as one of the scenarios.') argp.add_argument('--server_cpu_load', default=0, type=int, help='Select a targeted server cpu load to run. 0 means ignore this flag') argp.add_argument('-x', '--xml_report', default='report.xml', type=str, help='Name of XML report file to generate.') argp.add_argument('--perf_args', help=('Example usage: "--perf_args=record -F 99 -g". ' 'Wrap QPS workers in a perf command ' 'with the arguments to perf specified here. ' '".svg" flame graph profiles will be ' 'created for each Qps Worker on each scenario. ' 'Files will output to "<repo_root>/<args.flame_graph_reports>" ' 'directory. Output files from running the worker ' 'under perf are saved in the repo root where its ran. ' 'Note that the perf "-g" flag is necessary for ' 'flame graphs generation to work (assuming the binary ' 'being profiled uses frame pointers, check out ' '"--call-graph dwarf" option using libunwind otherwise.) ' 'Also note that the entire "--perf_args=<arg(s)>" must ' 'be wrapped in quotes as in the example usage. ' 'If the "--perg_args" is unspecified, "perf" will ' 'not be used at all. ' 'See http://www.brendangregg.com/perf.html ' 'for more general perf examples.')) argp.add_argument('--skip_generate_flamegraphs', default=False, action='store_const', const=True, help=('Turn flame graph generation off. ' 'May be useful if "perf_args" arguments do not make sense for ' 'generating flamegraphs (e.g., "--perf_args=stat ...")')) argp.add_argument('-f', '--flame_graph_reports', default='perf_reports', type=str, help='Name of directory to output flame graph profiles to, if any are created.') args = argp.parse_args() languages = set(scenario_config.LANGUAGES[l] for l in itertools.chain.from_iterable( six.iterkeys(scenario_config.LANGUAGES) if x == 'all' else [x] for x in args.language)) # Put together set of remote hosts where to run and build remote_hosts = set() if args.remote_worker_host: for host in args.remote_worker_host: remote_hosts.add(host) if args.remote_driver_host: remote_hosts.add(args.remote_driver_host) if not args.dry_run: if remote_hosts: archive_repo(languages=[str(l) for l in languages]) prepare_remote_hosts(remote_hosts, prepare_local=True) else: prepare_remote_hosts([], prepare_local=True) build_local = False if not args.remote_driver_host: build_local = True if not args.dry_run: build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build_local=build_local) perf_cmd = None if args.perf_args: print('Running workers under perf profiler') # Expect /usr/bin/perf to be installed here, as is usual perf_cmd = ['/usr/bin/perf'] perf_cmd.extend(re.split('\s+', args.perf_args)) qpsworker_jobs = create_qpsworkers(languages, args.remote_worker_host, perf_cmd=perf_cmd) # get list of worker addresses for each language. workers_by_lang = dict([(str(language), []) for language in languages]) for job in qpsworker_jobs: workers_by_lang[str(job.language)].append(job) scenarios = create_scenarios(languages, workers_by_lang=workers_by_lang, remote_host=args.remote_driver_host, regex=args.regex, category=args.category, bq_result_table=args.bq_result_table, netperf=args.netperf, netperf_hosts=args.remote_worker_host, server_cpu_load=args.server_cpu_load) if not scenarios: raise Exception('No scenarios to run') total_scenario_failures = 0 qps_workers_killed = 0 merged_resultset = {} perf_report_failures = 0 for scenario in scenarios: if args.dry_run: print(scenario.name) else: scenario_failures = 0 try: for worker in scenario.workers: worker.start() jobs = [scenario.jobspec] if scenario.workers: jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict(itertools.chain(six.iteritems(merged_resultset), six.iteritems(resultset))) finally: # Consider qps workers that need to be killed as failures qps_workers_killed += finish_qps_workers(scenario.workers, qpsworker_jobs) if perf_cmd and scenario_failures == 0 and not args.skip_generate_flamegraphs: workers_and_base_names = {} for worker in scenario.workers: if not worker.perf_file_base_name: raise Exception('using perf buf perf report filename is unspecified') workers_and_base_names[worker.host_and_port] = worker.perf_file_base_name perf_report_failures += run_collect_perf_profile_jobs(workers_and_base_names, scenario.name, args.flame_graph_reports) # Still write the index.html even if some scenarios failed. # 'profile_output_files' will only have names for scenarios that passed if perf_cmd and not args.skip_generate_flamegraphs: # write the index fil to the output dir, with all profiles from all scenarios/workers report_utils.render_perf_profiling_results('%s/index.html' % args.flame_graph_reports, profile_output_files) report_utils.render_junit_xml_report(merged_resultset, args.xml_report, suite_name='benchmarks') if total_scenario_failures > 0 or qps_workers_killed > 0: print('%s scenarios failed and %s qps worker jobs killed' % (total_scenario_failures, qps_workers_killed)) sys.exit(1) if perf_report_failures > 0: print('%s perf profile collection jobs failed' % perf_report_failures) sys.exit(1)
print('Will run these tests:') for job in jobs: if args.dry_run: print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))) else: print(' %s' % job.shortname) print if args.dry_run: print('--dry_run was used, exiting') sys.exit(1) jobset.message('START', 'Running test matrix.', do_newline=True) num_failures, resultset = jobset.run(jobs, newline_on_success=True, travis=True, maxjobs=args.jobs) # Merge skipped tests into results to show skipped tests on report.xml if skipped_jobs: ignored_num_skipped_failures, skipped_results = jobset.run( skipped_jobs, skip_jobs=True) resultset.update(skipped_results) report_utils.render_junit_xml_report(resultset, 'report_%s' % _REPORT_SUFFIX, suite_name='aggregate_tests') if num_failures == 0: jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.', do_newline=True) else: jobset.message('FAILED', 'Some run_tests.py instance have failed.', do_newline=True)
def collect_perf(bm_name, args): """generate flamegraphs""" heading('Flamegraphs: %s' % bm_name) subprocess.check_call([ 'make', bm_name, 'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count() ]) benchmarks = [] profile_analysis = [] cleanup = [] for line in subprocess.check_output( ['bins/mutrace/%s' % bm_name, '--benchmark_list_tests']).splitlines(): link(line, '%s.svg' % fnize(line)) benchmarks.append( jobset.JobSpec([ 'perf', 'record', '-o', '%s-perf.data' % fnize(line), '-g', '-F', '997', 'bins/mutrace/%s' % bm_name, '--benchmark_filter=^%s$' % line, '--benchmark_min_time=10' ])) profile_analysis.append( jobset.JobSpec( [ 'tools/run_tests/performance/process_local_perf_flamegraphs.sh' ], environ={ 'PERF_BASE_NAME': fnize(line), 'OUTPUT_DIR': 'reports', 'OUTPUT_FILENAME': fnize(line), })) cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)])) cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)])) # periodically flush out the list of jobs: temporary space required for this # processing is large if len(benchmarks) >= 20: # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) jobset.run(benchmarks, maxjobs=1, add_env={ 'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port }) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) benchmarks = [] profile_analysis = [] cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): jobset.run(benchmarks, maxjobs=1, add_env={ 'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port }) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
# Figure out which targets to build targets = [] for label in args.build: targets += _BUILD_MAP[label] # Among targets selected by -b, filter out those that don't match the filter targets = [t for t in targets if all(f in t.labels for f in args.filter)] targets = sorted(set(targets), key=lambda target: target.name) # Execute pre-build phase prebuild_jobs = [] for target in targets: prebuild_jobs += target.pre_build_jobspecs() if prebuild_jobs: num_failures, _ = jobset.run(prebuild_jobs, newline_on_success=True, maxjobs=args.jobs) if num_failures != 0: jobset.message('FAILED', 'Pre-build phase failed.', do_newline=True) sys.exit(1) build_jobs = [] for target in targets: build_jobs.append(target.build_jobspec()) if not build_jobs: print('Nothing to build.') sys.exit(1) jobset.message('START', 'Building targets.', do_newline=True) num_failures, resultset = jobset.run(build_jobs, newline_on_success=True,
# we don't need to build a docker image for objc continue job = build_interop_image_jobspec(l) docker_images[str(l)] = job.tag build_jobs.append(job) if build_jobs: jobset.message('START', 'Building interop docker images.', do_newline=True) if args.verbose: print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) num_failures, _ = jobset.run(build_jobs, newline_on_success=True, maxjobs=args.jobs) if num_failures == 0: jobset.message('SUCCESS', 'All docker images built successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) for image in six.itervalues(docker_images): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) server_manual_cmd_log = [] if args.manual_run else None client_manual_cmd_log = [] if args.manual_run else None
print('Will run these tests:') for job in jobs: if args.dry_run: print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))) else: print(' %s' % job.shortname) print if args.dry_run: print('--dry_run was used, exiting') sys.exit(1) jobset.message('START', 'Running test matrix.', do_newline=True) num_failures, resultset = jobset.run(jobs, newline_on_success=True, travis=True, maxjobs=args.jobs) # Merge skipped tests into results to show skipped tests on report.xml if skipped_jobs: ignored_num_skipped_failures, skipped_results = jobset.run( skipped_jobs, skip_jobs=True) resultset.update(skipped_results) report_utils.render_junit_xml_report(resultset, _report_filename('aggregate_tests'), suite_name='aggregate_tests') if num_failures == 0: jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.', do_newline=True) else: jobset.message('FAILED', 'Some run_tests.py instance have failed.', do_newline=True)
def run_one_scenario(scenario_config): jobset.message('START', 'Run scenario: %s' % scenario_config['name']) server_jobs = {} server_addresses = {} suppress_server_logs = True try: backend_addrs = [] fallback_ips = [] grpclb_ips = [] shortname_prefix = scenario_config['name'] # Start backends for i in xrange(len(scenario_config['backend_configs'])): backend_config = scenario_config['backend_configs'][i] backend_shortname = shortname(shortname_prefix, 'backend_server', i) backend_spec = backend_server_jobspec( backend_config['transport_sec'], backend_shortname) backend_job = dockerjob.DockerJob(backend_spec) server_jobs[backend_shortname] = backend_job backend_addrs.append('%s:%d' % (backend_job.ip_address(), _BACKEND_SERVER_PORT)) # Start fallbacks for i in xrange(len(scenario_config['fallback_configs'])): fallback_config = scenario_config['fallback_configs'][i] fallback_shortname = shortname(shortname_prefix, 'fallback_server', i) fallback_spec = fallback_server_jobspec( fallback_config['transport_sec'], fallback_shortname) fallback_job = dockerjob.DockerJob(fallback_spec) server_jobs[fallback_shortname] = fallback_job fallback_ips.append(fallback_job.ip_address()) # Start balancers for i in xrange(len(scenario_config['balancer_configs'])): balancer_config = scenario_config['balancer_configs'][i] grpclb_shortname = shortname(shortname_prefix, 'grpclb_server', i) grpclb_spec = grpclb_jobspec(balancer_config['transport_sec'], balancer_config['short_stream'], backend_addrs, grpclb_shortname) grpclb_job = dockerjob.DockerJob(grpclb_spec) server_jobs[grpclb_shortname] = grpclb_job grpclb_ips.append(grpclb_job.ip_address()) # Start DNS server dns_server_shortname = shortname(shortname_prefix, 'dns_server', 0) dns_server_spec = dns_server_in_docker_jobspec( grpclb_ips, fallback_ips, dns_server_shortname, scenario_config['cause_no_error_no_data_for_balancer_a_record']) dns_server_job = dockerjob.DockerJob(dns_server_spec) server_jobs[dns_server_shortname] = dns_server_job # Get the IP address of the docker container running the DNS server. # The DNS server is running on port 53 of that IP address. Note we will # point the DNS resolvers of grpc clients under test to our controlled # DNS server by effectively modifying the /etc/resolve.conf "nameserver" # lists of their docker containers. dns_server_ip = dns_server_job.ip_address() wait_until_dns_server_is_up(dns_server_ip) # Run clients jobs = [] for lang_name in languages: # Skip languages that are known to not currently # work for this test. if not args.no_skips and lang_name in scenario_config.get( 'skip_langs', []): jobset.message('IDLE', 'Skipping scenario: %s for language: %s\n' % (scenario_config['name'], lang_name)) continue lang = _LANGUAGES[lang_name] test_job = lb_client_interop_jobspec( lang, dns_server_ip, docker_image=docker_images.get(lang.safename), transport_security=scenario_config['transport_sec']) jobs.append(test_job) jobset.message('IDLE', 'Jobs to run: \n%s\n' % '\n'.join( str(job) for job in jobs)) num_failures, resultset = jobset.run( jobs, newline_on_success=True, maxjobs=args.jobs) report_utils.render_junit_xml_report(resultset, 'sponge_log.xml') if num_failures: suppress_server_logs = False jobset.message( 'FAILED', 'Scenario: %s. Some tests failed' % scenario_config['name'], do_newline=True) else: jobset.message( 'SUCCESS', 'Scenario: %s. All tests passed' % scenario_config['name'], do_newline=True) return num_failures finally: # Check if servers are still running. for server, job in server_jobs.items(): if not job.is_running(): print('Server "%s" has exited prematurely.' % server) suppress_failure = suppress_server_logs and not args.verbose dockerjob.finish_jobs( [j for j in six.itervalues(server_jobs)], suppress_failure=suppress_failure)
def collect_latency(bm_name, args): """generate latency profiles""" benchmarks = [] profile_analysis = [] cleanup = [] heading('Latency Profiles: %s' % bm_name) subprocess.check_call([ 'make', bm_name, 'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count() ]) for line in subprocess.check_output( ['bins/basicprof/%s' % bm_name, '--benchmark_list_tests']).splitlines(): link(line, '%s.txt' % fnize(line)) benchmarks.append( jobset.JobSpec([ 'bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' % line ], environ={'LATENCY_TRACE': '%s.trace' % fnize(line)})) profile_analysis.append( jobset.JobSpec([ sys.executable, 'tools/profiling/latency_profile/profile_analyzer.py', '--source', '%s.trace' % fnize(line), '--fmt', 'simple', '--out', 'reports/%s.txt' % fnize(line) ], timeout_seconds=None)) cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)])) # periodically flush out the list of jobs: profile_analysis jobs at least # consume upwards of five gigabytes of ram in some cases, and so analysing # hundreds of them at once is impractical -- but we want at least some # concurrency or the work takes too long if len(benchmarks) >= min(4, multiprocessing.cpu_count()): # run up to half the cpu count: each benchmark can use up to two cores # (one for the microbenchmark, one for the data flush) jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2), add_env={ 'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port }) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count()) benchmarks = [] profile_analysis = [] cleanup = [] # run the remaining benchmarks that weren't flushed if len(benchmarks): jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count() / 2), add_env={ 'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port }) jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count()) jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
def main(): argp = argparse.ArgumentParser(description='Run performance tests.') argp.add_argument('-l', '--language', choices=['all'] + sorted(scenario_config.LANGUAGES.keys()), nargs='+', required=True, help='Languages to benchmark.') argp.add_argument( '--remote_driver_host', default=None, help= 'Run QPS driver on given host. By default, QPS driver is run locally.') argp.add_argument('--remote_worker_host', nargs='+', default=[], help='Worker hosts where to start QPS workers.') argp.add_argument( '--dry_run', default=False, action='store_const', const=True, help='Just list scenarios to be run, but don\'t run them.') argp.add_argument('-r', '--regex', default='.*', type=str, help='Regex to select scenarios to run.') argp.add_argument('--bq_result_table', default=None, type=str, help='Bigquery "dataset.table" to upload results to.') argp.add_argument('--category', choices=['smoketest', 'all', 'scalable', 'sweep'], default='all', help='Select a category of tests to run.') argp.add_argument('--netperf', default=False, action='store_const', const=True, help='Run netperf benchmark as one of the scenarios.') argp.add_argument( '--server_cpu_load', default=0, type=int, help= 'Select a targeted server cpu load to run. 0 means ignore this flag') argp.add_argument('-x', '--xml_report', default='report.xml', type=str, help='Name of XML report file to generate.') argp.add_argument( '--perf_args', help=('Example usage: "--perf_args=record -F 99 -g". ' 'Wrap QPS workers in a perf command ' 'with the arguments to perf specified here. ' '".svg" flame graph profiles will be ' 'created for each Qps Worker on each scenario. ' 'Files will output to "<repo_root>/<args.flame_graph_reports>" ' 'directory. Output files from running the worker ' 'under perf are saved in the repo root where its ran. ' 'Note that the perf "-g" flag is necessary for ' 'flame graphs generation to work (assuming the binary ' 'being profiled uses frame pointers, check out ' '"--call-graph dwarf" option using libunwind otherwise.) ' 'Also note that the entire "--perf_args=<arg(s)>" must ' 'be wrapped in quotes as in the example usage. ' 'If the "--perg_args" is unspecified, "perf" will ' 'not be used at all. ' 'See http://www.brendangregg.com/perf.html ' 'for more general perf examples.')) argp.add_argument( '--skip_generate_flamegraphs', default=False, action='store_const', const=True, help=('Turn flame graph generation off. ' 'May be useful if "perf_args" arguments do not make sense for ' 'generating flamegraphs (e.g., "--perf_args=stat ...")')) argp.add_argument( '-f', '--flame_graph_reports', default='perf_reports', type=str, help= 'Name of directory to output flame graph profiles to, if any are created.' ) argp.add_argument( '-u', '--remote_host_username', default='', type=str, help='Use a username that isn\'t "Jenkins" to SSH into remote workers.' ) args = argp.parse_args() global _REMOTE_HOST_USERNAME if args.remote_host_username: _REMOTE_HOST_USERNAME = args.remote_host_username languages = set( scenario_config.LANGUAGES[l] for l in itertools.chain.from_iterable( six.iterkeys(scenario_config.LANGUAGES) if x == 'all' else [x] for x in args.language)) # Put together set of remote hosts where to run and build remote_hosts = set() if args.remote_worker_host: for host in args.remote_worker_host: remote_hosts.add(host) if args.remote_driver_host: remote_hosts.add(args.remote_driver_host) if not args.dry_run: if remote_hosts: archive_repo(languages=[str(l) for l in languages]) prepare_remote_hosts(remote_hosts, prepare_local=True) else: prepare_remote_hosts([], prepare_local=True) build_local = False if not args.remote_driver_host: build_local = True if not args.dry_run: build_on_remote_hosts(remote_hosts, languages=[str(l) for l in languages], build_local=build_local) perf_cmd = None if args.perf_args: print('Running workers under perf profiler') # Expect /usr/bin/perf to be installed here, as is usual perf_cmd = ['/usr/bin/perf'] perf_cmd.extend(re.split('\s+', args.perf_args)) qpsworker_jobs = create_qpsworkers(languages, args.remote_worker_host, perf_cmd=perf_cmd) # get list of worker addresses for each language. workers_by_lang = dict([(str(language), []) for language in languages]) for job in qpsworker_jobs: workers_by_lang[str(job.language)].append(job) scenarios = create_scenarios(languages, workers_by_lang=workers_by_lang, remote_host=args.remote_driver_host, regex=args.regex, category=args.category, bq_result_table=args.bq_result_table, netperf=args.netperf, netperf_hosts=args.remote_worker_host, server_cpu_load=args.server_cpu_load) if not scenarios: raise Exception('No scenarios to run') total_scenario_failures = 0 qps_workers_killed = 0 merged_resultset = {} perf_report_failures = 0 for scenario in scenarios: if args.dry_run: print(scenario.name) else: scenario_failures = 0 try: for worker in scenario.workers: worker.start() jobs = [scenario.jobspec] if scenario.workers: # TODO(jtattermusch): ideally the "quit" job won't show up # in the report jobs.append( create_quit_jobspec( scenario.workers, remote_host=args.remote_driver_host)) scenario_failures, resultset = jobset.run( jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict( itertools.chain(six.iteritems(merged_resultset), six.iteritems(resultset))) finally: # Consider qps workers that need to be killed as failures qps_workers_killed += finish_qps_workers( scenario.workers, qpsworker_jobs) if perf_cmd and scenario_failures == 0 and not args.skip_generate_flamegraphs: workers_and_base_names = {} for worker in scenario.workers: if not worker.perf_file_base_name: raise Exception( 'using perf buf perf report filename is unspecified' ) workers_and_base_names[ worker.host_and_port] = worker.perf_file_base_name perf_report_failures += run_collect_perf_profile_jobs( workers_and_base_names, scenario.name, args.flame_graph_reports) # Still write the index.html even if some scenarios failed. # 'profile_output_files' will only have names for scenarios that passed if perf_cmd and not args.skip_generate_flamegraphs: # write the index fil to the output dir, with all profiles from all scenarios/workers report_utils.render_perf_profiling_results( '%s/index.html' % args.flame_graph_reports, profile_output_files) report_utils.render_junit_xml_report(merged_resultset, args.xml_report, suite_name='benchmarks', multi_target=True) if total_scenario_failures > 0 or qps_workers_killed > 0: print('%s scenarios failed and %s qps worker jobs killed' % (total_scenario_failures, qps_workers_killed)) sys.exit(1) if perf_report_failures > 0: print('%s perf profile collection jobs failed' % perf_report_failures) sys.exit(1)
def run_one_scenario(scenario_config): jobset.message('START', 'Run scenario: %s' % scenario_config['name']) server_jobs = {} server_addresses = {} suppress_server_logs = True try: backend_addrs = [] fallback_ips = [] grpclb_ips = [] shortname_prefix = scenario_config['name'] # Start backends for i in range(len(scenario_config['backend_configs'])): backend_config = scenario_config['backend_configs'][i] backend_shortname = shortname(shortname_prefix, 'backend_server', i) backend_spec = backend_server_jobspec( backend_config['transport_sec'], backend_shortname) backend_job = dockerjob.DockerJob(backend_spec) server_jobs[backend_shortname] = backend_job backend_addrs.append( '%s:%d' % (backend_job.ip_address(), _BACKEND_SERVER_PORT)) # Start fallbacks for i in range(len(scenario_config['fallback_configs'])): fallback_config = scenario_config['fallback_configs'][i] fallback_shortname = shortname(shortname_prefix, 'fallback_server', i) fallback_spec = fallback_server_jobspec( fallback_config['transport_sec'], fallback_shortname) fallback_job = dockerjob.DockerJob(fallback_spec) server_jobs[fallback_shortname] = fallback_job fallback_ips.append(fallback_job.ip_address()) # Start balancers for i in range(len(scenario_config['balancer_configs'])): balancer_config = scenario_config['balancer_configs'][i] grpclb_shortname = shortname(shortname_prefix, 'grpclb_server', i) grpclb_spec = grpclb_jobspec(balancer_config['transport_sec'], balancer_config['short_stream'], backend_addrs, grpclb_shortname) grpclb_job = dockerjob.DockerJob(grpclb_spec) server_jobs[grpclb_shortname] = grpclb_job grpclb_ips.append(grpclb_job.ip_address()) # Start DNS server dns_server_shortname = shortname(shortname_prefix, 'dns_server', 0) dns_server_spec = dns_server_in_docker_jobspec( grpclb_ips, fallback_ips, dns_server_shortname, scenario_config['cause_no_error_no_data_for_balancer_a_record']) dns_server_job = dockerjob.DockerJob(dns_server_spec) server_jobs[dns_server_shortname] = dns_server_job # Get the IP address of the docker container running the DNS server. # The DNS server is running on port 53 of that IP address. Note we will # point the DNS resolvers of grpc clients under test to our controlled # DNS server by effectively modifying the /etc/resolve.conf "nameserver" # lists of their docker containers. dns_server_ip = dns_server_job.ip_address() wait_until_dns_server_is_up(dns_server_ip) # Run clients jobs = [] for lang_name in languages: # Skip languages that are known to not currently # work for this test. if not args.no_skips and lang_name in scenario_config.get( 'skip_langs', []): jobset.message( 'IDLE', 'Skipping scenario: %s for language: %s\n' % (scenario_config['name'], lang_name)) continue lang = _LANGUAGES[lang_name] test_job = lb_client_interop_jobspec( lang, dns_server_ip, docker_image=docker_images.get(lang.safename), transport_security=scenario_config['transport_sec']) jobs.append(test_job) jobset.message( 'IDLE', 'Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs)) num_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=args.jobs) report_utils.render_junit_xml_report(resultset, 'sponge_log.xml') if num_failures: suppress_server_logs = False jobset.message('FAILED', 'Scenario: %s. Some tests failed' % scenario_config['name'], do_newline=True) else: jobset.message('SUCCESS', 'Scenario: %s. All tests passed' % scenario_config['name'], do_newline=True) return num_failures finally: # Check if servers are still running. for server, job in list(server_jobs.items()): if not job.is_running(): print('Server "%s" has exited prematurely.' % server) suppress_failure = suppress_server_logs and not args.verbose dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)], suppress_failure=suppress_failure)
# Figure out which targets to build targets = [] for label in args.build: targets += _BUILD_MAP[label] # Among targets selected by -b, filter out those that don't match the filter targets = [t for t in targets if all(f in t.labels for f in args.filter)] targets = sorted(set(targets)) # Execute pre-build phase prebuild_jobs = [] for target in targets: prebuild_jobs += target.pre_build_jobspecs() if prebuild_jobs: num_failures, _ = jobset.run( prebuild_jobs, newline_on_success=True, maxjobs=args.jobs) if num_failures != 0: jobset.message('FAILED', 'Pre-build phase failed.', do_newline=True) sys.exit(1) build_jobs = [] for target in targets: build_jobs.append(target.build_jobspec()) if not build_jobs: print('Nothing to build.') sys.exit(1) jobset.message('START', 'Building targets.', do_newline=True) num_failures, _ = jobset.run( build_jobs, newline_on_success=True, maxjobs=args.jobs) if num_failures == 0: