def _build_and_run(check_cancelled, newline_on_success, travis, cache): """Do one pass of building & running tests.""" # build latest sequentially if not jobset.run(build_steps, maxjobs=1, newline_on_success=newline_on_success, travis=travis): return 1 # start antagonists antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py']) for _ in range(0, args.antagonists)] try: # run all the tests all_runs = itertools.chain.from_iterable( itertools.repeat(one_run, runs_per_test)) if not jobset.run(all_runs, check_cancelled, newline_on_success=newline_on_success, travis=travis, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, cache=cache): return 2 finally: for antagonist in antagonists: antagonist.kill() return 0
def _build_and_run( check_cancelled, newline_on_success, travis, cache, xml_report=None): """Do one pass of building & running tests.""" # build latest sequentially if not jobset.run(build_steps, maxjobs=1, newline_on_success=newline_on_success, travis=travis): return 1 # start antagonists antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py']) for _ in range(0, args.antagonists)] port_server_port = 9999 _start_port_server(port_server_port) try: infinite_runs = runs_per_test == 0 one_run = set( spec for config in run_configs for language in languages for spec in language.test_specs(config, args.travis) if re.search(args.regex, spec.shortname)) # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if travis: massaged_one_run = sorted(one_run, key=lambda x: x.shortname) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. massaged_one_run = list(one_run) # random.shuffle needs an indexable seq. random.shuffle(massaged_one_run) # which it modifies in-place. if infinite_runs: assert len(massaged_one_run) > 0, 'Must have at least one test for a -n inf run' runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs else itertools.repeat(massaged_one_run, runs_per_test)) all_runs = itertools.chain.from_iterable(runs_sequence) root = ET.Element('testsuites') if xml_report else None testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') if xml_report else None if not jobset.run(all_runs, check_cancelled, newline_on_success=newline_on_success, travis=travis, infinite_runs=infinite_runs, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, cache=cache if not xml_report else None, xml_report=testsuite, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}): return 2 finally: for antagonist in antagonists: antagonist.kill() if xml_report: tree = ET.ElementTree(root) tree.write(xml_report, encoding='UTF-8') if cache: cache.save() return 0
def run(name, benchmarks, jobs, loops, regex, counters): jobs_list = [] for loop in range(0, loops): for bm in benchmarks: jobs_list += _collect_bm_data(bm, 'opt', name, regex, loop, loops) if counters: jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop, loops) random.shuffle(jobs_list, random.SystemRandom().random) jobset.run(jobs_list, maxjobs=jobs)
def _build_and_run(check_cancelled, newline_on_success, travis, cache, xml_report=None): """Do one pass of building & running tests.""" # build latest sequentially if not jobset.run(build_steps, maxjobs=1, newline_on_success=newline_on_success, travis=travis): return 1 # start antagonists antagonists = [subprocess.Popen(["tools/run_tests/antagonist.py"]) for _ in range(0, args.antagonists)] try: infinite_runs = runs_per_test == 0 # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if travis: massaged_one_run = sorted(one_run, key=lambda x: x.shortname) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. massaged_one_run = list(one_run) # random.shuffle needs an indexable seq. random.shuffle(massaged_one_run) # which it modifies in-place. if infinite_runs: assert len(massaged_one_run) > 0, "Must have at least one test for a -n inf run" runs_sequence = ( itertools.repeat(massaged_one_run) if infinite_runs else itertools.repeat(massaged_one_run, runs_per_test) ) all_runs = itertools.chain.from_iterable(runs_sequence) root = ET.Element("testsuites") if xml_report else None testsuite = ET.SubElement(root, "testsuite", id="1", package="grpc", name="tests") if xml_report else None if not jobset.run( all_runs, check_cancelled, newline_on_success=newline_on_success, travis=travis, infinite_runs=infinite_runs, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, cache=cache if not xml_report else None, xml_report=testsuite, ): return 2 finally: for antagonist in antagonists: antagonist.kill() if xml_report: tree = ET.ElementTree(root) tree.write(xml_report, encoding="UTF-8") if cache: cache.save() return 0
def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), build_local=False): """Builds performance worker on remote hosts (and maybe also locally).""" build_timeout = 15 * 60 build_jobs = [] for host in hosts: user_at_host = "%s@%s" % (_REMOTE_HOST_USERNAME, host) build_jobs.append( jobset.JobSpec( cmdline=["tools/run_tests/performance/remote_host_build.sh"] + languages, shortname="remote_host_build.%s" % host, environ={"USER_AT_HOST": user_at_host, "CONFIG": "opt"}, timeout_seconds=build_timeout, ) ) if build_local: # Build locally as well build_jobs.append( jobset.JobSpec( cmdline=["tools/run_tests/performance/build_performance.sh"] + languages, shortname="local_build", environ={"CONFIG": "opt"}, timeout_seconds=build_timeout, ) ) jobset.message("START", "Building.", do_newline=True) num_failures, _ = jobset.run(build_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message("SUCCESS", "Built successfully.", do_newline=True) else: jobset.message("FAILED", "Build failed.", do_newline=True) sys.exit(1)
def _pull_images_for_lang(lang, images): """Pull all images for given lang from container registry.""" jobset.message('START', 'Downloading images for language "%s"' % lang, do_newline=True) download_specs = [] for release, image in images: # Pull the image and warm it up. # First time we use an image with "docker run", it takes time to unpack # the image and later this delay would fail our test cases. cmdline = [ 'time gcloud docker -- pull %s && time docker run --rm=true %s /bin/true' % (image, image) ] spec = jobset.JobSpec(cmdline=cmdline, shortname='pull_image_%s' % (image), timeout_seconds=_PULL_IMAGE_TIMEOUT_SECONDS, shell=True) download_specs.append(spec) # too many image downloads at once tend to get stuck max_pull_jobs = min(args.jobs, _MAX_PARALLEL_DOWNLOADS) num_failures, resultset = jobset.run(download_specs, newline_on_success=True, maxjobs=max_pull_jobs) if num_failures: jobset.message('FAILED', 'Failed to download some images', do_newline=True) return False else: jobset.message('SUCCESS', 'All images downloaded successfully.', do_newline=True) return True
def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), build_local=False): """Builds performance worker on remote hosts (and maybe also locally).""" build_timeout = 15*60 build_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) build_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/remote_host_build.sh'] + languages, shortname='remote_host_build.%s' % host, environ = {'USER_AT_HOST': user_at_host, 'CONFIG': 'opt'}, timeout_seconds=build_timeout)) if build_local: # Build locally as well build_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/build_performance.sh'] + languages, shortname='local_build', environ = {'CONFIG': 'opt'}, timeout_seconds=build_timeout)) jobset.message('START', 'Building.', do_newline=True) num_failures, _ = jobset.run( build_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Built successfully.', do_newline=True) else: jobset.message('FAILED', 'Build failed.', do_newline=True) sys.exit(1)
def run_tests_for_lang(lang, runtime, images): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ total_num_failures = 0 for image_tuple in images: release, image = image_tuple jobset.message('START', 'Testing %s' % image, do_newline=True) # Download the docker image before running each test case. subprocess.check_call(['gcloud', 'docker', '--', 'pull', image]) _docker_images_cleanup.append(image) suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = find_test_cases(lang, release, suite_name) num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table, args) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results(_xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) return total_num_failures
def prepare_remote_hosts(hosts, prepare_local=False): """Prepares remote hosts (and maybe prepare localhost as well).""" prepare_timeout = 5 * 60 prepare_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) prepare_jobs.append( jobset.JobSpec(cmdline=[ '../grpc_benchmark/scripts/performance/remote_host_prepare.sh' ], shortname='remote_host_prepare.%s' % host, environ={'USER_AT_HOST': user_at_host}, timeout_seconds=prepare_timeout)) if prepare_local: # Prepare localhost as well prepare_jobs.append( jobset.JobSpec(cmdline=[ '../grpc_benchmark/scripts/performance/kill_workers.sh' ], shortname='local_prepare', timeout_seconds=prepare_timeout)) jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run(prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Prepare step completed successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to prepare remote hosts.', do_newline=True) sys.exit(1)
def archive_repo(languages): """Archives local version of repo including submodules.""" cmdline = ['tar', '-cf', '../grpc.tar', '../grpc/'] if 'java' in languages: cmdline.append('../grpc-java') if 'go' in languages: cmdline.append('../grpc-go') archive_job = jobset.JobSpec(cmdline=cmdline, shortname='archive_repo', timeout_seconds=3 * 60) jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run([archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', 'Archive with local repository created successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to archive local repository.', do_newline=True) sys.exit(1)
def run_tests_for_lang(lang, runtime, images): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ for image_tuple in images: release, image = image_tuple jobset.message('START', 'Testing %s' % image, do_newline=True) # Download the docker image before running each test case. subprocess.check_call(['gcloud', 'docker', '--', 'pull', image]) _docker_images_cleanup.append(image) job_spec_list = find_test_cases(lang,release) num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image':image}, maxjobs=args.jobs) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results( _xml_report_tree, resultset, 'grpc_interop_matrix', '%s__%s %s'%(lang,runtime,release), str(uuid.uuid4()))
def build_on_remote_hosts(hosts, build_local=False): """Builds performance worker on remote hosts.""" build_timeout = 15*60 build_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) build_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/remote_host_build.sh'], shortname='remote_host_build.%s' % host, environ = {'USER_AT_HOST': user_at_host, 'CONFIG': 'opt'}, timeout_seconds=build_timeout)) if build_local: # Build locally as well build_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/build_performance.sh'], shortname='local_build', environ = {'CONFIG': 'opt'}, timeout_seconds=build_timeout)) jobset.message('START', 'Building on remote hosts.', do_newline=True) num_failures, _ = jobset.run( build_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Build on remote hosts was successful.', do_newline=True) else: jobset.message('FAILED', 'Failed to build on remote hosts.', do_newline=True) sys.exit(1)
def _test_release(lang, runtime, release, image, xml_report_tree, skip_tests): total_num_failures = 0 suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = _generate_test_case_jobspecs(lang, runtime, release, suite_name) if not job_spec_list: jobset.message('FAILED', 'No test cases were found.', do_newline=True) total_num_failures += 1 else: num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs, skip_jobs=skip_tests) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table) if skip_tests: jobset.message('FAILED', 'Tests were skipped', do_newline=True) total_num_failures += 1 if num_failures: total_num_failures += num_failures report_utils.append_junit_xml_results(xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) return total_num_failures
def prepare_remote_hosts(hosts, prepare_local=False): """Prepares remote hosts (and maybe prepare localhost as well).""" prepare_timeout = 5*60 prepare_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) prepare_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/remote_host_prepare.sh'], shortname='remote_host_prepare.%s' % host, environ = {'USER_AT_HOST': user_at_host}, timeout_seconds=prepare_timeout)) if prepare_local: # Prepare localhost as well prepare_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/kill_workers.sh'], shortname='local_prepare', timeout_seconds=prepare_timeout)) jobset.message('START', 'Preparing hosts.', do_newline=True) num_failures, _ = jobset.run( prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Prepare step completed successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to prepare remote hosts.', do_newline=True) sys.exit(1)
def _pull_images_for_lang(lang, images): """Pull all images for given lang from container registry.""" jobset.message( 'START', 'Downloading images for language "%s"' % lang, do_newline=True) download_specs = [] for release, image in images: # Pull the image and warm it up. # First time we use an image with "docker run", it takes time to unpack # the image and later this delay would fail our test cases. cmdline = [ 'time gcloud docker -- pull %s && time docker run --rm=true %s /bin/true' % (image, image) ] spec = jobset.JobSpec( cmdline=cmdline, shortname='pull_image_%s' % (image), timeout_seconds=_PULL_IMAGE_TIMEOUT_SECONDS, shell=True) download_specs.append(spec) # too many image downloads at once tend to get stuck max_pull_jobs = min(args.jobs, _MAX_PARALLEL_DOWNLOADS) num_failures, resultset = jobset.run( download_specs, newline_on_success=True, maxjobs=max_pull_jobs) if num_failures: jobset.message( 'FAILED', 'Failed to download some images', do_newline=True) return False else: jobset.message( 'SUCCESS', 'All images downloaded successfully.', do_newline=True) return True
def _build_and_run(check_cancelled, newline_on_success, travis, cache): """Do one pass of building & running tests.""" # build latest sequentially if not jobset.run(build_steps, maxjobs=1, newline_on_success=newline_on_success, travis=travis): return 1 # run all the tests all_runs = itertools.chain.from_iterable( itertools.repeat(one_run, runs_per_test)) if not jobset.run(all_runs, check_cancelled, newline_on_success=newline_on_success, travis=travis, maxjobs=min(args.jobs, min(c.maxjobs for c in run_configs)), cache=cache): return 2 return 0
def _build_and_run(check_cancelled, newline_on_success, travis, cache): """Do one pass of building & running tests.""" # build latest sequentially if not jobset.run(build_steps, maxjobs=1, newline_on_success=newline_on_success, travis=travis): return 1 # start antagonists antagonists = [ subprocess.Popen(['tools/run_tests/antagonist.py']) for _ in range(0, args.antagonists) ] try: infinite_runs = runs_per_test == 0 # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if travis: massaged_one_run = sorted(one_run, key=lambda x: x.shortname) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. massaged_one_run = list( one_run) # random.shuffle needs an indexable seq. random.shuffle(massaged_one_run) # which it modifies in-place. runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs else itertools.repeat( massaged_one_run, runs_per_test)) all_runs = itertools.chain.from_iterable(runs_sequence) if not jobset.run(all_runs, check_cancelled, newline_on_success=newline_on_success, travis=travis, infinite_runs=infinite_runs, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, cache=cache): return 2 finally: for antagonist in antagonists: antagonist.kill() return 0
def _run_tests_for_lang(lang, runtime, images, xml_report_tree): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ skip_tests = False if not _pull_images_for_lang(lang, images): jobset.message( 'FAILED', 'Image download failed. Skipping tests for language "%s"' % lang, do_newline=True) skip_tests = True total_num_failures = 0 for release, image in images: suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = _generate_test_case_jobspecs(lang, runtime, release, suite_name) if not job_spec_list: jobset.message( 'FAILED', 'No test cases were found.', do_newline=True) total_num_failures += 1 continue num_failures, resultset = jobset.run( job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs, skip_jobs=skip_tests) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table) if skip_tests: jobset.message('FAILED', 'Tests were skipped', do_newline=True) total_num_failures += 1 elif num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results(xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) # cleanup all downloaded docker images for _, image in images: if not args.keep: _cleanup_docker_image(image) return total_num_failures
def _run_tests_for_lang(lang, runtime, images, xml_report_tree): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ skip_tests = False if not _pull_images_for_lang(lang, images): jobset.message( 'FAILED', 'Image download failed. Skipping tests for language "%s"' % lang, do_newline=True) skip_tests = True total_num_failures = 0 for release, image in images: suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = _generate_test_case_jobspecs(lang, runtime, release, suite_name) if not job_spec_list: jobset.message('FAILED', 'No test cases were found.', do_newline=True) total_num_failures += 1 continue num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs, skip_jobs=skip_tests) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table) if skip_tests: jobset.message('FAILED', 'Tests were skipped', do_newline=True) total_num_failures += 1 elif num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results(xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) # cleanup all downloaded docker images for _, image in images: if not args.keep: _cleanup_docker_image(image) return total_num_failures
def main(args): bm_build.build('new', args.benchmarks, args.jobs, args.counters) old = args.old if args.diff_base: old = 'old' where_am_i = subprocess.check_output( ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_base]) try: bm_build.build(old, args.benchmarks, args.jobs, args.counters) finally: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) jobs_list = [] jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops, args.regex, args.counters) jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops, args.regex, args.counters) # shuffle all jobs to eliminate noise from GCE CPU drift random.shuffle(jobs_list, random.SystemRandom().random) jobset.run(jobs_list, maxjobs=args.jobs) diff, note, significance = bm_diff.diff(args.benchmarks, args.loops, args.regex, args.track, old, 'new', args.counters) if diff: text = '[%s] Performance differences noted:\n%s' % ( args.pr_comment_name, diff) else: text = '[%s] No significant performance differences' % args.pr_comment_name if note: text = note + '\n\n' + text print('%s' % text) check_on_pr.check_on_pr('Benchmark', '```\n%s\n```' % text) check_on_pr.label_significance_on_pr('perf-change', significance)
def _build_and_run(check_cancelled, newline_on_success, travis, cache): """Do one pass of building & running tests.""" # build latest sequentially if not jobset.run(build_steps, maxjobs=1, newline_on_success=newline_on_success, travis=travis): return 1 # start antagonists antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py']) for _ in range(0, args.antagonists)] try: infinite_runs = runs_per_test == 0 # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if travis: massaged_one_run = sorted(one_run, key=lambda x: x.shortname) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. massaged_one_run = list(one_run) # random.shuffle needs an indexable seq. random.shuffle(massaged_one_run) # which it modifies in-place. runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs else itertools.repeat(massaged_one_run, runs_per_test)) all_runs = itertools.chain.from_iterable(runs_sequence) if not jobset.run(all_runs, check_cancelled, newline_on_success=newline_on_success, travis=travis, infinite_runs=infinite_runs, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, cache=cache): return 2 finally: for antagonist in antagonists: antagonist.kill() if cache: cache.save() return 0
def main(args): bm_build.build('new', args.benchmarks, args.jobs, args.counters) old = args.old if args.diff_base: old = 'old' where_am_i = subprocess.check_output( ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip() subprocess.check_call(['git', 'checkout', args.diff_base]) try: bm_build.build(old, args.benchmarks, args.jobs, args.counters) finally: subprocess.check_call(['git', 'checkout', where_am_i]) subprocess.check_call(['git', 'submodule', 'update']) jobs_list = [] jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops, args.regex, args.counters) jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops, args.regex, args.counters) # shuffle all jobs to eliminate noise from GCE CPU drift random.shuffle(jobs_list, random.SystemRandom().random) jobset.run(jobs_list, maxjobs=args.jobs) diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex, args.track, old, 'new', args.counters) if diff: text = '[%s] Performance differences noted:\n%s' % ( args.pr_comment_name, diff) else: text = '[%s] No significant performance differences' % args.pr_comment_name if note: text = note + '\n\n' + text print('%s' % text) comment_on_pr.comment_on_pr('```\n%s\n```' % text)
def _run_tests_for_lang(lang, runtime, images, xml_report_tree): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ skip_tests = False total_num_failures = 0 max_pull_jobs = min(args.jobs, _MAX_PARALLEL_DOWNLOADS) max_chunk_size = max_pull_jobs chunk_count = (len(images) + max_chunk_size) // max_chunk_size for chunk_index in range(chunk_count): chunk_start = chunk_index * max_chunk_size chunk_size = min(max_chunk_size, len(images) - chunk_start) chunk_end = chunk_start + chunk_size pull_specs = [] if not skip_tests: for release, image in images[chunk_start:chunk_end]: pull_specs.append(_pull_image_for_lang(lang, image, release)) # NOTE(rbellevi): We batch docker pull operations to maximize # parallelism, without letting the disk usage grow unbounded. pull_failures, _ = jobset.run(pull_specs, newline_on_success=True, maxjobs=max_pull_jobs) if pull_failures: jobset.message( 'FAILED', 'Image download failed. Skipping tests for language "%s"' % lang, do_newline=True) skip_tests = True for release, image in images[chunk_start:chunk_end]: total_num_failures += _test_release(lang, runtime, release, image, xml_report_tree, skip_tests) if not args.keep: for _, image in images[chunk_start:chunk_end]: _cleanup_docker_image(image) if not total_num_failures: jobset.message('SUCCESS', 'All {} tests passed'.format(lang), do_newline=True) else: jobset.message('FAILED', 'Some {} tests failed'.format(lang), do_newline=True) return total_num_failures
def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name): perf_report_jobs = [] global profile_output_files for host_and_port in hosts_and_base_names: perf_base_name = hosts_and_base_names[host_and_port] output_filename = '%s-%s' % (scenario_name, perf_base_name) # from the base filename, create .svg output filename host = host_and_port.split(':')[0] profile_output_files.append('%s.svg' % output_filename) perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename)) jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1) jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) return failures
def build_all_images_for_release(lang, release): """Build all docker images for a release across all runtimes.""" docker_images = [] build_jobs = [] env = {} # If we not using current tree or the sibling for grpc stack, do checkout. stack_base = '' if args.git_checkout: stack_base = checkout_grpc_stack(lang, release) var = { 'go': 'GRPC_GO_ROOT', 'java': 'GRPC_JAVA_ROOT', 'node': 'GRPC_NODE_ROOT' }.get(lang, 'GRPC_ROOT') env[var] = stack_base for runtime in client_matrix.get_runtimes_for_lang_release(lang, release): job = build_image_jobspec(runtime, env, release, stack_base) docker_images.append(job.tag) build_jobs.append(job) jobset.message('START', 'Building interop docker images.', do_newline=True) print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) num_failures, _ = jobset.run( build_jobs, newline_on_success=True, maxjobs=multiprocessing.cpu_count()) if num_failures: jobset.message( 'FAILED', 'Failed to build interop docker images.', do_newline=True) docker_images_cleanup.extend(docker_images) sys.exit(1) jobset.message( 'SUCCESS', 'All docker images built for %s at %s.' % (lang, release), do_newline=True) if release != 'master': commit_log = os.path.join(stack_base, 'commit_log') if os.path.exists(commit_log): for image in docker_images: add_files_to_image(image, [commit_log], 'release=%s' % release) return docker_images
def archive_repo(languages): """Archives local version of repo including submodules.""" cmdline = ["tar", "-cf", "../grpc.tar", "../grpc/"] if "java" in languages: cmdline.append("../grpc-java") if "go" in languages: cmdline.append("../grpc-go") archive_job = jobset.JobSpec(cmdline=cmdline, shortname="archive_repo", timeout_seconds=3 * 60) jobset.message("START", "Archiving local repository.", do_newline=True) num_failures, _ = jobset.run([archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message("SUCCESS", "Archive with local repository created successfully.", do_newline=True) else: jobset.message("FAILED", "Failed to archive local repository.", do_newline=True) sys.exit(1)
def build_all_images_for_release(lang, release): """Build all docker images for a release across all runtimes.""" docker_images = [] build_jobs = [] env = {} # If we not using current tree or the sibling for grpc stack, do checkout. stack_base = '' if args.git_checkout: stack_base = checkout_grpc_stack(lang, release) var = { 'go': 'GRPC_GO_ROOT', 'java': 'GRPC_JAVA_ROOT', 'node': 'GRPC_NODE_ROOT' }.get(lang, 'GRPC_ROOT') env[var] = stack_base for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]: job = build_image_jobspec(runtime, env, release, stack_base) docker_images.append(job.tag) build_jobs.append(job) jobset.message('START', 'Building interop docker images.', do_newline=True) print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs)) num_failures, _ = jobset.run( build_jobs, newline_on_success=True, maxjobs=multiprocessing.cpu_count()) if num_failures: jobset.message( 'FAILED', 'Failed to build interop docker images.', do_newline=True) docker_images_cleanup.extend(docker_images) sys.exit(1) jobset.message( 'SUCCESS', 'All docker images built for %s at %s.' % (lang, release), do_newline=True) if release != 'master': commit_log = os.path.join(stack_base, 'commit_log') if os.path.exists(commit_log): for image in docker_images: add_files_to_image(image, [commit_log], 'release=%s' % release) return docker_images
def run_tests_for_lang(lang, runtime, images): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ # Fine to ignore return value as failure to download will result in test failure # later anyway. pull_images_for_lang(lang, images) total_num_failures = 0 for release, image in images: jobset.message('START', 'Testing %s' % image, do_newline=True) suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = find_test_cases(lang, runtime, release, suite_name) if not job_spec_list: jobset.message('FAILED', 'No test cases were found.', do_newline=True) return 1 num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table, args) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results(_xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) if not args.keep: cleanup(image) return total_num_failures
def archive_repo(): """Archives local version of repo including submodules.""" # TODO: also archive grpc-go and grpc-java repos archive_job = jobset.JobSpec( cmdline=['tar', '-cf', '../grpc.tar', '../grpc/'], shortname='archive_repo', timeout_seconds=3*60) jobset.message('START', 'Archiving local repository.', do_newline=True) num_failures, _ = jobset.run( [archive_job], newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', 'Archive with local repository create successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to archive local repository.', do_newline=True) sys.exit(1)
def run_tests_for_lang(lang, runtime, images): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ total_num_failures = 0 for image_tuple in images: release, image = image_tuple jobset.message('START', 'Testing %s' % image, do_newline=True) # Download the docker image before running each test case. subprocess.check_call(['gcloud', 'docker', '--', 'pull', image]) suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = find_test_cases(lang, runtime, release, suite_name) if not job_spec_list: jobset.message( 'FAILED', 'No test cases were found.', do_newline=True) return 1 num_failures, resultset = jobset.run( job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table, args) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results(_xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) if not args.keep: cleanup(image) return total_num_failures
def prepare_remote_hosts(hosts): """Prepares remote hosts.""" prepare_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) prepare_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/remote_host_prepare.sh'], shortname='remote_host_prepare.%s' % host, environ = {'USER_AT_HOST': user_at_host}, timeout_seconds=5*60)) jobset.message('START', 'Preparing remote hosts.', do_newline=True) num_failures, _ = jobset.run( prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Remote hosts ready to start build.', do_newline=True) else: jobset.message('FAILED', 'Failed to prepare remote hosts.', do_newline=True) sys.exit(1)
def run_tests_for_lang(lang, runtime, images): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ for image_tuple in images: release, image = image_tuple jobset.message('START', 'Testing %s' % image, do_newline=True) _docker_images_cleanup.append(image) job_spec_list = find_test_cases(lang, release) num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results( _xml_report_tree, resultset, 'grpc_interop_matrix', '%s__%s:%s' % (lang, runtime, release), str(uuid.uuid4()))
def prepare_remote_hosts(hosts): """Prepares remote hosts.""" prepare_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) prepare_jobs.append( jobset.JobSpec( cmdline=['tools/run_tests/performance/remote_host_prepare.sh'], shortname='remote_host_prepare.%s' % host, environ = {'USER_AT_HOST': user_at_host}, timeout_seconds=3*60)) jobset.message('START', 'Preparing remote hosts.', do_newline=True) num_failures, _ = jobset.run( prepare_jobs, newline_on_success=True, maxjobs=10) if num_failures == 0: jobset.message('SUCCESS', 'Remote hosts ready to start build.', do_newline=True) else: jobset.message('FAILED', 'Failed to prepare remote hosts.', do_newline=True) sys.exit(1)
def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name): perf_report_jobs = [] global profile_output_files for host_and_port in hosts_and_base_names: perf_base_name = hosts_and_base_names[host_and_port] output_filename = '%s-%s' % (scenario_name, perf_base_name) # from the base filename, create .svg output filename host = host_and_port.split(':')[0] profile_output_files.append('%s.svg' % output_filename) perf_report_jobs.append( perf_report_processor_job(host, perf_base_name, output_filename)) jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True) failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1) jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True) return failures
default=multiprocessing.cpu_count(), help='Number of CPUs to use') argp.set_defaults(fix=False) args = argp.parse_args() # Explicitly passing the .clang-tidy config by reading it. # This is required because source files in the compilation database are # in a different source tree so clang-tidy cannot find the right config file # by seeking their parent directories. with open(".clang-tidy") as f: config = f.read() cmdline = [ clang_tidy, '--config=' + config, ] if args.fix: cmdline.append('--fix') jobs = [] for filename in args.files: jobs.append( jobset.JobSpec( cmdline + [filename], shortname=filename, timeout_seconds=15 * 60, )) num_fails, res_set = jobset.run(jobs, maxjobs=args.jobs) sys.exit(num_fails)
docker_images={} if args.use_docker: # languages for which to build docker images languages_to_build = set(_LANGUAGES[k] for k in set([str(l) for l in languages] + [s for s in servers])) build_jobs = [] for l in languages_to_build: job = build_interop_image_jobspec(l) docker_images[str(l)] = job.tag build_jobs.append(job) if build_jobs: jobset.message('START', 'Building interop docker images.', do_newline=True) if jobset.run(build_jobs, newline_on_success=True, maxjobs=args.jobs): jobset.message('SUCCESS', 'All docker images built successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) for image in docker_images.itervalues(): dockerjob.remove_image(image, skip_nonexistent=True) exit(1); # Start interop servers. server_jobs={} server_addresses={} try: for s in servers: lang = str(s) spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang)) job = dockerjob.DockerJob(spec)
# languages for which to build docker images languages_to_build = set(_LANGUAGES[k] for k in set([str(l) for l in languages] + [s for s in servers])) build_jobs = [] for l in languages_to_build: job = build_interop_image_jobspec(l) docker_images[str(l)] = job.tag build_jobs.append(job) if build_jobs: jobset.message('START', 'Building interop docker images.', do_newline=True) if jobset.run(build_jobs, newline_on_success=True, maxjobs=args.jobs): jobset.message('SUCCESS', 'All docker images built successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) for image in docker_images.itervalues(): dockerjob.remove_image(image, skip_nonexistent=True) exit(1) # Start interop servers. server_jobs = {} server_addresses = {} try:
netperf_hosts=args.remote_worker_host) if not scenarios: raise Exception('No scenarios to run') total_scenario_failures = 0 qps_workers_killed = 0 for scenario in scenarios: if args.dry_run: print(scenario.name) else: try: for worker in scenario.workers: worker.start() scenario_failures, _ = jobset.run([ scenario.jobspec, create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host) ], newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures finally: # Consider qps workers that need to be killed as failures qps_workers_killed += finish_qps_workers(scenario.workers) if total_scenario_failures > 0 or qps_workers_killed > 0: print("%s scenarios failed and %s qps worker jobs killed" % (total_scenario_failures, qps_workers_killed)) sys.exit(1)
build_local = False if not args.remote_driver_host: build_local = True build_on_remote_hosts(remote_hosts, build_local=build_local) qpsworker_jobs = start_qpsworkers(args.remote_worker_host) worker_addresses = [job.host_and_port for job in qpsworker_jobs] try: scenarios = create_scenarios(languages=[CXXLanguage()], workers=worker_addresses, remote_host=args.remote_driver_host) if not scenarios: raise Exception('No scenarios to run') jobset.message('START', 'Running scenarios.', do_newline=True) num_failures, _ = jobset.run( scenarios, newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', 'All scenarios finished successfully.', do_newline=True) else: jobset.message('FAILED', 'Some of the scenarios failed.', do_newline=True) sys.exit(1) finally: finish_qps_workers(qpsworker_jobs)
def _build_and_run( check_cancelled, newline_on_success, cache, xml_report=None, build_only=False): """Do one pass of building & running tests.""" # build latest sequentially num_failures, resultset = jobset.run( build_steps, maxjobs=1, stop_on_failure=True, newline_on_success=newline_on_success, travis=args.travis) if num_failures: return [BuildAndRunError.BUILD] if build_only: if xml_report: report_utils.render_junit_xml_report(resultset, xml_report) return [] # start antagonists antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py']) for _ in range(0, args.antagonists)] port_server_port = 32767 _start_port_server(port_server_port) resultset = None num_test_failures = 0 try: infinite_runs = runs_per_test == 0 one_run = set( spec for language in languages for spec in language.test_specs() if re.search(args.regex, spec.shortname)) # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if args.travis: massaged_one_run = sorted(one_run, key=lambda x: x.shortname) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. massaged_one_run = list(one_run) # random.shuffle needs an indexable seq. random.shuffle(massaged_one_run) # which it modifies in-place. if infinite_runs: assert len(massaged_one_run) > 0, 'Must have at least one test for a -n inf run' runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs else itertools.repeat(massaged_one_run, runs_per_test)) all_runs = itertools.chain.from_iterable(runs_sequence) num_test_failures, resultset = jobset.run( all_runs, check_cancelled, newline_on_success=newline_on_success, travis=args.travis, infinite_runs=infinite_runs, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, cache=cache if not xml_report else None, add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}) if resultset: for k, v in resultset.iteritems(): num_runs, num_failures = _calculate_num_runs_failures(v) if num_failures == num_runs: # what about infinite_runs??? jobset.message('FAILED', k, do_newline=True) elif num_failures > 0: jobset.message( 'FLAKE', '%s [%d/%d runs flaked]' % (k, num_failures, num_runs), do_newline=True) else: jobset.message('PASSED', k, do_newline=True) finally: for antagonist in antagonists: antagonist.kill() if xml_report and resultset: report_utils.render_junit_xml_report(resultset, xml_report) number_failures, _ = jobset.run( post_tests_steps, maxjobs=1, stop_on_failure=True, newline_on_success=newline_on_success, travis=args.travis) out = [] if number_failures: out.append(BuildAndRunError.POST_TEST) if num_test_failures: out.append(BuildAndRunError.TEST) if cache: cache.save() return out
def _build_and_run(check_cancelled, newline_on_success, travis, cache, xml_report=None): """Do one pass of building & running tests.""" # build latest sequentially if not jobset.run(build_steps, maxjobs=1, stop_on_failure=True, newline_on_success=newline_on_success, travis=travis): return 1 # start antagonists antagonists = [ subprocess.Popen(['tools/run_tests/antagonist.py']) for _ in range(0, args.antagonists) ] port_server_port = 32767 _start_port_server(port_server_port) try: infinite_runs = runs_per_test == 0 one_run = set(spec for config in run_configs for language in languages for spec in language.test_specs(config, args.travis) if re.search(args.regex, spec.shortname)) # When running on travis, we want out test runs to be as similar as possible # for reproducibility purposes. if travis: massaged_one_run = sorted(one_run, key=lambda x: x.shortname) else: # whereas otherwise, we want to shuffle things up to give all tests a # chance to run. massaged_one_run = list( one_run) # random.shuffle needs an indexable seq. random.shuffle(massaged_one_run) # which it modifies in-place. if infinite_runs: assert len(massaged_one_run ) > 0, 'Must have at least one test for a -n inf run' runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs else itertools.repeat( massaged_one_run, runs_per_test)) all_runs = itertools.chain.from_iterable(runs_sequence) root = ET.Element('testsuites') if xml_report else None testsuite = ET.SubElement( root, 'testsuite', id='1', package='grpc', name='tests') if xml_report else None if not jobset.run(all_runs, check_cancelled, newline_on_success=newline_on_success, travis=travis, infinite_runs=infinite_runs, maxjobs=args.jobs, stop_on_failure=args.stop_on_failure, cache=cache if not xml_report else None, xml_report=testsuite, add_env={ 'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port }): return 2 finally: for antagonist in antagonists: antagonist.kill() if xml_report: tree = ET.ElementTree(root) tree.write(xml_report, encoding='UTF-8') if cache: cache.save() return 0
print if len(relevant_jobs) == len(jobs): print '(TESTING) No tests will be skipped.' else: print '(TESTING) These tests will be skipped:' for job in list(set(jobs) - set(relevant_jobs)): print ' %s' % job.shortname print if args.dry_run: print '--dry_run was used, exiting' sys.exit(1) jobset.message('START', 'Running test matrix.', do_newline=True) num_failures, resultset = jobset.run(jobs, newline_on_success=True, travis=True, maxjobs=args.jobs) report_utils.render_junit_xml_report(resultset, 'report.xml', suite_name='aggregate_tests') if num_failures == 0: jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.', do_newline=True) else: jobset.message('FAILED', 'Some run_tests.py instance have failed.', do_newline=True) sys.exit(1)
relevant_jobs = filter_tests(jobs, args.base_branch) # todo(mattkwong): add skipped tests to report.xml print if len(relevant_jobs) == len(jobs): print '(TESTING) No tests will be skipped.' else: print '(TESTING) These tests will be skipped:' for job in list(set(jobs) - set(relevant_jobs)): print ' %s' % job.shortname print if args.dry_run: print '--dry_run was used, exiting' sys.exit(1) jobset.message('START', 'Running test matrix.', do_newline=True) num_failures, resultset = jobset.run(jobs, newline_on_success=True, travis=True, maxjobs=args.jobs) report_utils.render_junit_xml_report(resultset, 'report.xml', suite_name='aggregate_tests') if num_failures == 0: jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.', do_newline=True) else: jobset.message('FAILED', 'Some run_tests.py instance have failed.', do_newline=True) sys.exit(1)
if os.path.splitext(f)[1] == '.template': out_dir = '.' + root[len('templates'):] out = out_dir + '/' + os.path.splitext(f)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) cmd = base_cmd[:] cmd.append('-P') cmd.append(preprocessed_build) cmd.append('-o') if test is None: cmd.append(out) else: tf = tempfile.mkstemp() test[out] = tf[1] os.close(tf[0]) cmd.append(test[out]) cmd.append(root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) jobset.run(pre_jobs, maxjobs=args.jobs) jobset.run(jobs, maxjobs=args.jobs) if test is not None: for s, g in test.iteritems(): if os.path.isfile(g): assert 0 == os.system('diff %s %s' % (s, g)), s os.unlink(g) else: assert 0 == os.system('diff -r %s %s' % (s, g)), s shutil.rmtree(g, ignore_errors=True)
'--config=' + config, ] if args.fix: cmdline.append('--fix') if args.only_changed: orig_files = set(args.files) actual_files = [] output = subprocess.check_output( ['git', 'diff', 'origin/master', 'HEAD', '--name-only']) for line in output.decode('ascii').splitlines(False): if line in orig_files: print("check: %s" % line) actual_files.append(line) else: print("skip: %s - not in the build" % line) args.files = actual_files jobs = [] for filename in args.files: jobs.append( jobset.JobSpec( cmdline + [filename], shortname=filename, timeout_seconds=15 * 60, )) num_fails, res_set = jobset.run(jobs, maxjobs=args.jobs, quiet_success=True) sys.exit(num_fails)
if args.use_docker: # languages for which to build docker images languages_to_build = set(_LANGUAGES[k] for k in set([str(l) for l in languages] + [s for s in servers])) if args.http2_interop: languages_to_build.add(http2Interop) build_jobs = [] for l in languages_to_build: job = build_interop_image_jobspec(l) docker_images[str(l)] = job.tag build_jobs.append(job) if build_jobs: jobset.message('START', 'Building interop docker images.', do_newline=True) num_failures, _ = jobset.run( build_jobs, newline_on_success=True, maxjobs=args.jobs) if num_failures == 0: jobset.message('SUCCESS', 'All docker images built successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) for image in docker_images.itervalues(): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) # Start interop servers. server_jobs={} server_addresses={} try: for s in servers:
print('Will run these tests:') for job in jobs: if args.dry_run: print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))) else: print(' %s' % job.shortname) print if args.dry_run: print('--dry_run was used, exiting') sys.exit(1) jobset.message('START', 'Running test matrix.', do_newline=True) num_failures, resultset = jobset.run(jobs, newline_on_success=True, travis=True, maxjobs=args.jobs) # Merge skipped tests into results to show skipped tests on report.xml if skipped_jobs: skipped_results = jobset.run(skipped_jobs, skip_jobs=True) resultset.update(skipped_results) report_utils.render_junit_xml_report(resultset, 'report.xml', suite_name='aggregate_tests') if num_failures == 0: jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.', do_newline=True) else: jobset.message('FAILED',
argp.add_argument("-l", "--language", default="c++") args = argp.parse_args() # build job build_job = jobset.JobSpec( cmdline=["tools/run_tests/run_interops_build.sh", "%s" % args.language], shortname="build", timeout_seconds=30 * 60 ) # test jobs, each test is a separate job to run in parallel _TESTS = ["large_unary", "empty_unary", "ping_pong", "client_streaming", "server_streaming"] jobs = [] jobNumber = 0 for test in _TESTS: test_job = jobset.JobSpec( cmdline=["tools/run_tests/run_interops_test.sh", "%s" % args.language, "%s" % test], shortname=test, timeout_seconds=15 * 60, ) jobs.append(test_job) jobNumber += 1 root = ET.Element("testsuites") testsuite = ET.SubElement(root, "testsuite", id="1", package="grpc", name="tests") # always do the build of docker first, and then all the tests can run in parallel jobset.run([build_job], maxjobs=1, xml_report=testsuite) jobset.run(jobs, maxjobs=jobNumber, xml_report=testsuite) tree = ET.ElementTree(root) tree.write("report.xml", encoding="UTF-8")
(bm, stripped_line, cfg, name, idx), '--benchmark_out_format=json', ] jobs_list.append( jobset.JobSpec(cmd, shortname='%s %s %s %s %d/%d' % (bm, line, cfg, name, idx + 1, loops), verbose_success=True, cpu_cost=2, timeout_seconds=60 * 60)) # one hour return jobs_list def create_jobs(name, benchmarks, loops, regex, counters): jobs_list = [] for loop in range(0, loops): for bm in benchmarks: jobs_list += _collect_bm_data(bm, 'opt', name, regex, loop, loops) if counters: jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop, loops) random.shuffle(jobs_list, random.SystemRandom().random) return jobs_list if __name__ == '__main__': args = _args() jobs_list = create_jobs(args.name, args.benchmarks, args.loops, args.regex, args.counters) jobset.run(jobs_list, maxjobs=args.jobs)
print('Will run these tests:') for job in jobs: if args.dry_run: print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline))) else: print(' %s' % job.shortname) print if args.dry_run: print('--dry_run was used, exiting') sys.exit(1) jobset.message('START', 'Running test matrix.', do_newline=True) num_failures, resultset = jobset.run(jobs, newline_on_success=True, travis=True, maxjobs=args.jobs) # Merge skipped tests into results to show skipped tests on report.xml if skipped_jobs: skipped_results = jobset.run(skipped_jobs, skip_jobs=True) resultset.update(skipped_results) report_utils.render_junit_xml_report(resultset, 'report.xml', suite_name='aggregate_tests') if num_failures == 0: jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.', do_newline=True) else: jobset.message('FAILED', 'Some run_tests.py instance have failed.', do_newline=True)
if os.path.splitext(f)[1] == '.template': out_dir = args.base + root[len('templates'):] out = out_dir + '/' + os.path.splitext(f)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) cmd = base_cmd[:] cmd.append('-P') cmd.append(preprocessed_build) cmd.append('-o') if test is None: cmd.append(out) else: tf = tempfile.mkstemp() test[out] = tf[1] os.close(tf[0]) cmd.append(test[out]) cmd.append(args.base + '/' + root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) jobset.run(pre_jobs, maxjobs=args.jobs) jobset.run(jobs, maxjobs=args.jobs) if test is not None: for s, g in test.iteritems(): if os.path.isfile(g): assert 0 == os.system('diff %s %s' % (s, g)), s os.unlink(g) else: assert 0 == os.system('diff -r %s %s' % (s, g)), s shutil.rmtree(g, ignore_errors=True)
# get list of worker addresses for each language. worker_addresses = dict([(str(language), []) for language in languages]) for job in qpsworker_jobs: worker_addresses[str(job.language)].append(job.host_and_port) try: scenarios = create_scenarios(languages, workers_by_lang=worker_addresses, remote_host=args.remote_driver_host, regex=args.regex, bq_result_table=args.bq_result_table) if not scenarios: raise Exception('No scenarios to run') jobset.message('START', 'Running scenarios.', do_newline=True) num_failures, _ = jobset.run(scenarios, newline_on_success=True, maxjobs=1) if num_failures == 0: jobset.message('SUCCESS', 'All scenarios finished successfully.', do_newline=True) else: jobset.message('FAILED', 'Some of the scenarios failed.', do_newline=True) sys.exit(1) except: traceback.print_exc() raise finally: finish_qps_workers(qpsworker_jobs)
if not scenarios: raise Exception('No scenarios to run') total_scenario_failures = 0 qps_workers_killed = 0 merged_resultset = {} for scenario in scenarios: if args.dry_run: print(scenario.name) else: try: for worker in scenario.workers: worker.start() scenario_failures, resultset = jobset.run([scenario.jobspec, create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)], newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict(itertools.chain(merged_resultset.iteritems(), resultset.iteritems())) finally: # Consider qps workers that need to be killed as failures qps_workers_killed += finish_qps_workers(scenario.workers) report_utils.render_junit_xml_report(merged_resultset, 'report.xml', suite_name='benchmarks') if total_scenario_failures > 0 or qps_workers_killed > 0: print ("%s scenarios failed and %s qps worker jobs killed" % (total_scenario_failures, qps_workers_killed)) sys.exit(1)
for scenario in scenarios: if args.dry_run: print(scenario.name) else: scenario_failures = 0 try: for worker in scenario.workers: worker.start() jobs = [scenario.jobspec] if scenario.workers: jobs.append( create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict( itertools.chain(merged_resultset.iteritems(), resultset.iteritems())) finally: # Consider qps workers that need to be killed as failures qps_workers_killed += finish_qps_workers(scenario.workers) if perf_cmd and scenario_failures == 0 and not args.skip_generate_flamegraphs: workers_and_base_names = {} for worker in scenario.workers: if not worker.perf_file_base_name: raise Exception( 'using perf buf perf report filename is unspecified')
for f in files: if os.path.splitext(f)[1] == '.template': out_dir = '.' + root[len('templates'):] out = out_dir + '/' + os.path.splitext(f)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) cmd = ['python', 'tools/buildgen/mako_renderer.py'] for plugin in plugins: cmd.append('-p') cmd.append(plugin) for js in json: cmd.append('-d') cmd.append(js) cmd.append('-o') if test is None: cmd.append(out) else: tf = tempfile.mkstemp() test[out] = tf[1] os.close(tf[0]) cmd.append(test[out]) cmd.append(root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out)) jobset.run(jobs) if test is not None: for s, g in test.iteritems(): assert (0 == os.system('diff %s %s' % (s, g))) os.unlink(g)
[s for s in servers])) if args.http2_interop: languages_to_build.add(http2Interop) build_jobs = [] for l in languages_to_build: job = build_interop_image_jobspec(l) docker_images[str(l)] = job.tag build_jobs.append(job) if build_jobs: jobset.message('START', 'Building interop docker images.', do_newline=True) num_failures, _ = jobset.run(build_jobs, newline_on_success=True, maxjobs=args.jobs) if num_failures == 0: jobset.message('SUCCESS', 'All docker images built successfully.', do_newline=True) else: jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True) for image in docker_images.itervalues(): dockerjob.remove_image(image, skip_nonexistent=True) sys.exit(1) # Start interop servers. server_jobs = {}
qps_workers_killed = 0 merged_resultset = {} perf_report_failures = 0 for scenario in scenarios: if args.dry_run: print(scenario.name) else: scenario_failures = 0 try: for worker in scenario.workers: worker.start() jobs = [scenario.jobspec] if scenario.workers: jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)) scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1) total_scenario_failures += scenario_failures merged_resultset = dict(itertools.chain(merged_resultset.iteritems(), resultset.iteritems())) finally: # Consider qps workers that need to be killed as failures qps_workers_killed += finish_qps_workers(scenario.workers) if perf_cmd and scenario_failures == 0 and not args.skip_generate_flamegraphs: workers_and_base_names = {} for worker in scenario.workers: if not worker.perf_file_base_name: raise Exception('using perf buf perf report filename is unspecified') workers_and_base_names[worker.host_and_port] = worker.perf_file_base_name perf_report_failures += run_collect_perf_profile_jobs(workers_and_base_names, scenario.name)
jobs = [] for root, dirs, files in os.walk('templates'): for f in files: if os.path.splitext(f)[1] == '.template': out = '.' + root[len('templates'):] + '/' + os.path.splitext(f)[0] cmd = ['tools/buildgen/mako_renderer.py'] for plugin in plugins: cmd.append('-p') cmd.append(plugin) for js in json: cmd.append('-d') cmd.append(js) cmd.append('-o') if test is None: cmd.append(out) else: tf = tempfile.mkstemp() test[out] = tf[1] os.close(tf[0]) cmd.append(test[out]) cmd.append(root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out)) jobset.run(jobs) if test is not None: for s, g in test.iteritems(): assert(0 == os.system('diff %s %s' % (s, g))) os.unlink(g)