def _build_and_run(check_cancelled, newline_on_success, travis, cache):
"""Do one pass of building & running tests."""
# build latest sequentially
if not jobset.run(build_steps, maxjobs=1,
newline_on_success=newline_on_success, travis=travis):
return 1
# start antagonists
antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py'])
for _ in range(0, args.antagonists)]
try:
# run all the tests
all_runs = itertools.chain.from_iterable(
itertools.repeat(one_run, runs_per_test))
if not jobset.run(all_runs, check_cancelled,
newline_on_success=newline_on_success, travis=travis,
maxjobs=args.jobs,
stop_on_failure=args.stop_on_failure,
cache=cache):
return 2
finally:
for antagonist in antagonists:
antagonist.kill()
return 0
def _build_and_run(
check_cancelled, newline_on_success, travis, cache, xml_report=None):
"""Do one pass of building & running tests."""
# build latest sequentially
if not jobset.run(build_steps, maxjobs=1,
newline_on_success=newline_on_success, travis=travis):
return 1
# start antagonists
antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py'])
for _ in range(0, args.antagonists)]
port_server_port = 9999
_start_port_server(port_server_port)
try:
infinite_runs = runs_per_test == 0
one_run = set(
spec
for config in run_configs
for language in languages
for spec in language.test_specs(config, args.travis)
if re.search(args.regex, spec.shortname))
# When running on travis, we want out test runs to be as similar as possible
# for reproducibility purposes.
if travis:
massaged_one_run = sorted(one_run, key=lambda x: x.shortname)
else:
# whereas otherwise, we want to shuffle things up to give all tests a
# chance to run.
massaged_one_run = list(one_run) # random.shuffle needs an indexable seq.
random.shuffle(massaged_one_run) # which it modifies in-place.
if infinite_runs:
assert len(massaged_one_run) > 0, 'Must have at least one test for a -n inf run'
runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs
else itertools.repeat(massaged_one_run, runs_per_test))
all_runs = itertools.chain.from_iterable(runs_sequence)
root = ET.Element('testsuites') if xml_report else None
testsuite = ET.SubElement(root, 'testsuite', id='1', package='grpc', name='tests') if xml_report else None
if not jobset.run(all_runs, check_cancelled,
newline_on_success=newline_on_success, travis=travis,
infinite_runs=infinite_runs,
maxjobs=args.jobs,
stop_on_failure=args.stop_on_failure,
cache=cache if not xml_report else None,
xml_report=testsuite,
add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port}):
return 2
finally:
for antagonist in antagonists:
antagonist.kill()
if xml_report:
tree = ET.ElementTree(root)
tree.write(xml_report, encoding='UTF-8')
if cache: cache.save()
return 0
def run(name, benchmarks, jobs, loops, regex, counters):
jobs_list = []
for loop in range(0, loops):
for bm in benchmarks:
jobs_list += _collect_bm_data(bm, 'opt', name, regex, loop, loops)
if counters:
jobs_list += _collect_bm_data(bm, 'counters', name, regex, loop,
loops)
random.shuffle(jobs_list, random.SystemRandom().random)
jobset.run(jobs_list, maxjobs=jobs)
def _build_and_run(check_cancelled, newline_on_success, travis, cache, xml_report=None):
"""Do one pass of building & running tests."""
# build latest sequentially
if not jobset.run(build_steps, maxjobs=1, newline_on_success=newline_on_success, travis=travis):
return 1
# start antagonists
antagonists = [subprocess.Popen(["tools/run_tests/antagonist.py"]) for _ in range(0, args.antagonists)]
try:
infinite_runs = runs_per_test == 0
# When running on travis, we want out test runs to be as similar as possible
# for reproducibility purposes.
if travis:
massaged_one_run = sorted(one_run, key=lambda x: x.shortname)
else:
# whereas otherwise, we want to shuffle things up to give all tests a
# chance to run.
massaged_one_run = list(one_run) # random.shuffle needs an indexable seq.
random.shuffle(massaged_one_run) # which it modifies in-place.
if infinite_runs:
assert len(massaged_one_run) > 0, "Must have at least one test for a -n inf run"
runs_sequence = (
itertools.repeat(massaged_one_run) if infinite_runs else itertools.repeat(massaged_one_run, runs_per_test)
)
all_runs = itertools.chain.from_iterable(runs_sequence)
root = ET.Element("testsuites") if xml_report else None
testsuite = ET.SubElement(root, "testsuite", id="1", package="grpc", name="tests") if xml_report else None
if not jobset.run(
all_runs,
check_cancelled,
newline_on_success=newline_on_success,
travis=travis,
infinite_runs=infinite_runs,
maxjobs=args.jobs,
stop_on_failure=args.stop_on_failure,
cache=cache if not xml_report else None,
xml_report=testsuite,
):
return 2
finally:
for antagonist in antagonists:
antagonist.kill()
if xml_report:
tree = ET.ElementTree(root)
tree.write(xml_report, encoding="UTF-8")
if cache:
cache.save()
return 0
def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), build_local=False):
"""Builds performance worker on remote hosts (and maybe also locally)."""
build_timeout = 15 * 60
build_jobs = []
for host in hosts:
user_at_host = "%s@%s" % (_REMOTE_HOST_USERNAME, host)
build_jobs.append(
jobset.JobSpec(
cmdline=["tools/run_tests/performance/remote_host_build.sh"] + languages,
shortname="remote_host_build.%s" % host,
environ={"USER_AT_HOST": user_at_host, "CONFIG": "opt"},
timeout_seconds=build_timeout,
)
)
if build_local:
# Build locally as well
build_jobs.append(
jobset.JobSpec(
cmdline=["tools/run_tests/performance/build_performance.sh"] + languages,
shortname="local_build",
environ={"CONFIG": "opt"},
timeout_seconds=build_timeout,
)
)
jobset.message("START", "Building.", do_newline=True)
num_failures, _ = jobset.run(build_jobs, newline_on_success=True, maxjobs=10)
if num_failures == 0:
jobset.message("SUCCESS", "Built successfully.", do_newline=True)
else:
jobset.message("FAILED", "Build failed.", do_newline=True)
sys.exit(1)
def _pull_images_for_lang(lang, images):
"""Pull all images for given lang from container registry."""
jobset.message('START',
'Downloading images for language "%s"' % lang,
do_newline=True)
download_specs = []
for release, image in images:
# Pull the image and warm it up.
# First time we use an image with "docker run", it takes time to unpack
# the image and later this delay would fail our test cases.
cmdline = [
'time gcloud docker -- pull %s && time docker run --rm=true %s /bin/true'
% (image, image)
]
spec = jobset.JobSpec(cmdline=cmdline,
shortname='pull_image_%s' % (image),
timeout_seconds=_PULL_IMAGE_TIMEOUT_SECONDS,
shell=True)
download_specs.append(spec)
# too many image downloads at once tend to get stuck
max_pull_jobs = min(args.jobs, _MAX_PARALLEL_DOWNLOADS)
num_failures, resultset = jobset.run(download_specs,
newline_on_success=True,
maxjobs=max_pull_jobs)
if num_failures:
jobset.message('FAILED',
'Failed to download some images',
do_newline=True)
return False
else:
jobset.message('SUCCESS',
'All images downloaded successfully.',
do_newline=True)
return True
def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), build_local=False):
"""Builds performance worker on remote hosts (and maybe also locally)."""
build_timeout = 15*60
build_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_build.sh'] + languages,
shortname='remote_host_build.%s' % host,
environ = {'USER_AT_HOST': user_at_host, 'CONFIG': 'opt'},
timeout_seconds=build_timeout))
if build_local:
# Build locally as well
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/build_performance.sh'] + languages,
shortname='local_build',
environ = {'CONFIG': 'opt'},
timeout_seconds=build_timeout))
jobset.message('START', 'Building.', do_newline=True)
num_failures, _ = jobset.run(
build_jobs, newline_on_success=True, maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS',
'Built successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Build failed.',
do_newline=True)
sys.exit(1)
def run_tests_for_lang(lang, runtime, images):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
total_num_failures = 0
for image_tuple in images:
release, image = image_tuple
jobset.message('START', 'Testing %s' % image, do_newline=True)
# Download the docker image before running each test case.
subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
_docker_images_cleanup.append(image)
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = find_test_cases(lang, release, suite_name)
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table, args)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(_xml_report_tree, resultset,
'grpc_interop_matrix',
suite_name, str(uuid.uuid4()))
return total_num_failures
def prepare_remote_hosts(hosts, prepare_local=False):
"""Prepares remote hosts (and maybe prepare localhost as well)."""
prepare_timeout = 5 * 60
prepare_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
prepare_jobs.append(
jobset.JobSpec(cmdline=[
'../grpc_benchmark/scripts/performance/remote_host_prepare.sh'
],
shortname='remote_host_prepare.%s' % host,
environ={'USER_AT_HOST': user_at_host},
timeout_seconds=prepare_timeout))
if prepare_local:
# Prepare localhost as well
prepare_jobs.append(
jobset.JobSpec(cmdline=[
'../grpc_benchmark/scripts/performance/kill_workers.sh'
],
shortname='local_prepare',
timeout_seconds=prepare_timeout))
jobset.message('START', 'Preparing hosts.', do_newline=True)
num_failures, _ = jobset.run(prepare_jobs,
newline_on_success=True,
maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS',
'Prepare step completed successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Failed to prepare remote hosts.',
do_newline=True)
sys.exit(1)
def archive_repo(languages):
"""Archives local version of repo including submodules."""
cmdline = ['tar', '-cf', '../grpc.tar', '../grpc/']
if 'java' in languages:
cmdline.append('../grpc-java')
if 'go' in languages:
cmdline.append('../grpc-go')
archive_job = jobset.JobSpec(cmdline=cmdline,
shortname='archive_repo',
timeout_seconds=3 * 60)
jobset.message('START', 'Archiving local repository.', do_newline=True)
num_failures, _ = jobset.run([archive_job],
newline_on_success=True,
maxjobs=1)
if num_failures == 0:
jobset.message('SUCCESS',
'Archive with local repository created successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Failed to archive local repository.',
do_newline=True)
sys.exit(1)
def run_tests_for_lang(lang, runtime, images):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
for image_tuple in images:
release, image = image_tuple
jobset.message('START', 'Testing %s' % image, do_newline=True)
# Download the docker image before running each test case.
subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
_docker_images_cleanup.append(image)
job_spec_list = find_test_cases(lang,release)
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image':image},
maxjobs=args.jobs)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(
_xml_report_tree,
resultset,
'grpc_interop_matrix',
'%s__%s %s'%(lang,runtime,release),
str(uuid.uuid4()))
def build_on_remote_hosts(hosts, build_local=False):
"""Builds performance worker on remote hosts."""
build_timeout = 15*60
build_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_build.sh'],
shortname='remote_host_build.%s' % host,
environ = {'USER_AT_HOST': user_at_host, 'CONFIG': 'opt'},
timeout_seconds=build_timeout))
if build_local:
# Build locally as well
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/build_performance.sh'],
shortname='local_build',
environ = {'CONFIG': 'opt'},
timeout_seconds=build_timeout))
jobset.message('START', 'Building on remote hosts.', do_newline=True)
num_failures, _ = jobset.run(
build_jobs, newline_on_success=True, maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS',
'Build on remote hosts was successful.',
do_newline=True)
else:
jobset.message('FAILED', 'Failed to build on remote hosts.',
do_newline=True)
sys.exit(1)
def _test_release(lang, runtime, release, image, xml_report_tree, skip_tests):
total_num_failures = 0
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = _generate_test_case_jobspecs(lang, runtime, release,
suite_name)
if not job_spec_list:
jobset.message('FAILED', 'No test cases were found.', do_newline=True)
total_num_failures += 1
else:
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs,
skip_jobs=skip_tests)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table)
if skip_tests:
jobset.message('FAILED', 'Tests were skipped', do_newline=True)
total_num_failures += 1
if num_failures:
total_num_failures += num_failures
report_utils.append_junit_xml_results(xml_report_tree, resultset,
'grpc_interop_matrix',
suite_name, str(uuid.uuid4()))
return total_num_failures
def prepare_remote_hosts(hosts, prepare_local=False):
"""Prepares remote hosts (and maybe prepare localhost as well)."""
prepare_timeout = 5*60
prepare_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
prepare_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_prepare.sh'],
shortname='remote_host_prepare.%s' % host,
environ = {'USER_AT_HOST': user_at_host},
timeout_seconds=prepare_timeout))
if prepare_local:
# Prepare localhost as well
prepare_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/kill_workers.sh'],
shortname='local_prepare',
timeout_seconds=prepare_timeout))
jobset.message('START', 'Preparing hosts.', do_newline=True)
num_failures, _ = jobset.run(
prepare_jobs, newline_on_success=True, maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS',
'Prepare step completed successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Failed to prepare remote hosts.',
do_newline=True)
sys.exit(1)
def _pull_images_for_lang(lang, images):
"""Pull all images for given lang from container registry."""
jobset.message(
'START', 'Downloading images for language "%s"' % lang, do_newline=True)
download_specs = []
for release, image in images:
# Pull the image and warm it up.
# First time we use an image with "docker run", it takes time to unpack
# the image and later this delay would fail our test cases.
cmdline = [
'time gcloud docker -- pull %s && time docker run --rm=true %s /bin/true'
% (image, image)
]
spec = jobset.JobSpec(
cmdline=cmdline,
shortname='pull_image_%s' % (image),
timeout_seconds=_PULL_IMAGE_TIMEOUT_SECONDS,
shell=True)
download_specs.append(spec)
# too many image downloads at once tend to get stuck
max_pull_jobs = min(args.jobs, _MAX_PARALLEL_DOWNLOADS)
num_failures, resultset = jobset.run(
download_specs, newline_on_success=True, maxjobs=max_pull_jobs)
if num_failures:
jobset.message(
'FAILED', 'Failed to download some images', do_newline=True)
return False
else:
jobset.message(
'SUCCESS', 'All images downloaded successfully.', do_newline=True)
return True
def _build_and_run(check_cancelled, newline_on_success, travis, cache):
"""Do one pass of building & running tests."""
# build latest sequentially
if not jobset.run(build_steps, maxjobs=1,
newline_on_success=newline_on_success, travis=travis):
return 1
# run all the tests
all_runs = itertools.chain.from_iterable(
itertools.repeat(one_run, runs_per_test))
if not jobset.run(all_runs, check_cancelled,
newline_on_success=newline_on_success, travis=travis,
maxjobs=min(args.jobs, min(c.maxjobs for c in run_configs)),
cache=cache):
return 2
return 0
def _build_and_run(check_cancelled, newline_on_success, travis, cache):
"""Do one pass of building & running tests."""
# build latest sequentially
if not jobset.run(build_steps, maxjobs=1,
newline_on_success=newline_on_success, travis=travis):
return 1
# run all the tests
all_runs = itertools.chain.from_iterable(
itertools.repeat(one_run, runs_per_test))
if not jobset.run(all_runs, check_cancelled,
newline_on_success=newline_on_success, travis=travis,
maxjobs=min(args.jobs, min(c.maxjobs for c in run_configs)),
cache=cache):
return 2
return 0
def _build_and_run(check_cancelled, newline_on_success, travis, cache):
"""Do one pass of building & running tests."""
# build latest sequentially
if not jobset.run(build_steps,
maxjobs=1,
newline_on_success=newline_on_success,
travis=travis):
return 1
# start antagonists
antagonists = [
subprocess.Popen(['tools/run_tests/antagonist.py'])
for _ in range(0, args.antagonists)
]
try:
infinite_runs = runs_per_test == 0
# When running on travis, we want out test runs to be as similar as possible
# for reproducibility purposes.
if travis:
massaged_one_run = sorted(one_run, key=lambda x: x.shortname)
else:
# whereas otherwise, we want to shuffle things up to give all tests a
# chance to run.
massaged_one_run = list(
one_run) # random.shuffle needs an indexable seq.
random.shuffle(massaged_one_run) # which it modifies in-place.
runs_sequence = (itertools.repeat(massaged_one_run)
if infinite_runs else itertools.repeat(
massaged_one_run, runs_per_test))
all_runs = itertools.chain.from_iterable(runs_sequence)
if not jobset.run(all_runs,
check_cancelled,
newline_on_success=newline_on_success,
travis=travis,
infinite_runs=infinite_runs,
maxjobs=args.jobs,
stop_on_failure=args.stop_on_failure,
cache=cache):
return 2
finally:
for antagonist in antagonists:
antagonist.kill()
return 0
def _run_tests_for_lang(lang, runtime, images, xml_report_tree):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
skip_tests = False
if not _pull_images_for_lang(lang, images):
jobset.message(
'FAILED',
'Image download failed. Skipping tests for language "%s"' % lang,
do_newline=True)
skip_tests = True
total_num_failures = 0
for release, image in images:
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = _generate_test_case_jobspecs(lang, runtime, release,
suite_name)
if not job_spec_list:
jobset.message(
'FAILED', 'No test cases were found.', do_newline=True)
total_num_failures += 1
continue
num_failures, resultset = jobset.run(
job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs,
skip_jobs=skip_tests)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table)
if skip_tests:
jobset.message('FAILED', 'Tests were skipped', do_newline=True)
total_num_failures += 1
elif num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(xml_report_tree, resultset,
'grpc_interop_matrix', suite_name,
str(uuid.uuid4()))
# cleanup all downloaded docker images
for _, image in images:
if not args.keep:
_cleanup_docker_image(image)
return total_num_failures
def _run_tests_for_lang(lang, runtime, images, xml_report_tree):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
skip_tests = False
if not _pull_images_for_lang(lang, images):
jobset.message(
'FAILED',
'Image download failed. Skipping tests for language "%s"' % lang,
do_newline=True)
skip_tests = True
total_num_failures = 0
for release, image in images:
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = _generate_test_case_jobspecs(lang, runtime, release,
suite_name)
if not job_spec_list:
jobset.message('FAILED',
'No test cases were found.',
do_newline=True)
total_num_failures += 1
continue
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs,
skip_jobs=skip_tests)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table)
if skip_tests:
jobset.message('FAILED', 'Tests were skipped', do_newline=True)
total_num_failures += 1
elif num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(xml_report_tree, resultset,
'grpc_interop_matrix',
suite_name, str(uuid.uuid4()))
# cleanup all downloaded docker images
for _, image in images:
if not args.keep:
_cleanup_docker_image(image)
return total_num_failures
def main(args):
bm_build.build('new', args.benchmarks, args.jobs, args.counters)
old = args.old
if args.diff_base:
old = 'old'
where_am_i = subprocess.check_output(
['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
subprocess.check_call(['git', 'checkout', args.diff_base])
try:
bm_build.build(old, args.benchmarks, args.jobs, args.counters)
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
jobs_list = []
jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops,
args.regex, args.counters)
jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops,
args.regex, args.counters)
# shuffle all jobs to eliminate noise from GCE CPU drift
random.shuffle(jobs_list, random.SystemRandom().random)
jobset.run(jobs_list, maxjobs=args.jobs)
diff, note, significance = bm_diff.diff(args.benchmarks, args.loops,
args.regex, args.track, old, 'new',
args.counters)
if diff:
text = '[%s] Performance differences noted:\n%s' % (
args.pr_comment_name, diff)
else:
text = '[%s] No significant performance differences' % args.pr_comment_name
if note:
text = note + '\n\n' + text
print('%s' % text)
check_on_pr.check_on_pr('Benchmark', '```\n%s\n```' % text)
check_on_pr.label_significance_on_pr('perf-change', significance)
def _build_and_run(check_cancelled, newline_on_success, travis, cache):
"""Do one pass of building & running tests."""
# build latest sequentially
if not jobset.run(build_steps, maxjobs=1,
newline_on_success=newline_on_success, travis=travis):
return 1
# start antagonists
antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py'])
for _ in range(0, args.antagonists)]
try:
infinite_runs = runs_per_test == 0
# When running on travis, we want out test runs to be as similar as possible
# for reproducibility purposes.
if travis:
massaged_one_run = sorted(one_run, key=lambda x: x.shortname)
else:
# whereas otherwise, we want to shuffle things up to give all tests a
# chance to run.
massaged_one_run = list(one_run) # random.shuffle needs an indexable seq.
random.shuffle(massaged_one_run) # which it modifies in-place.
runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs
else itertools.repeat(massaged_one_run, runs_per_test))
all_runs = itertools.chain.from_iterable(runs_sequence)
if not jobset.run(all_runs, check_cancelled,
newline_on_success=newline_on_success, travis=travis,
infinite_runs=infinite_runs,
maxjobs=args.jobs,
stop_on_failure=args.stop_on_failure,
cache=cache):
return 2
finally:
for antagonist in antagonists:
antagonist.kill()
if cache: cache.save()
return 0
def main(args):
bm_build.build('new', args.benchmarks, args.jobs, args.counters)
old = args.old
if args.diff_base:
old = 'old'
where_am_i = subprocess.check_output(
['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
subprocess.check_call(['git', 'checkout', args.diff_base])
try:
bm_build.build(old, args.benchmarks, args.jobs, args.counters)
finally:
subprocess.check_call(['git', 'checkout', where_am_i])
subprocess.check_call(['git', 'submodule', 'update'])
jobs_list = []
jobs_list += bm_run.create_jobs('new', args.benchmarks, args.loops,
args.regex, args.counters)
jobs_list += bm_run.create_jobs(old, args.benchmarks, args.loops,
args.regex, args.counters)
# shuffle all jobs to eliminate noise from GCE CPU drift
random.shuffle(jobs_list, random.SystemRandom().random)
jobset.run(jobs_list, maxjobs=args.jobs)
diff, note = bm_diff.diff(args.benchmarks, args.loops, args.regex,
args.track, old, 'new', args.counters)
if diff:
text = '[%s] Performance differences noted:\n%s' % (
args.pr_comment_name, diff)
else:
text = '[%s] No significant performance differences' % args.pr_comment_name
if note:
text = note + '\n\n' + text
print('%s' % text)
comment_on_pr.comment_on_pr('```\n%s\n```' % text)
def _run_tests_for_lang(lang, runtime, images, xml_report_tree):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
skip_tests = False
total_num_failures = 0
max_pull_jobs = min(args.jobs, _MAX_PARALLEL_DOWNLOADS)
max_chunk_size = max_pull_jobs
chunk_count = (len(images) + max_chunk_size) // max_chunk_size
for chunk_index in range(chunk_count):
chunk_start = chunk_index * max_chunk_size
chunk_size = min(max_chunk_size, len(images) - chunk_start)
chunk_end = chunk_start + chunk_size
pull_specs = []
if not skip_tests:
for release, image in images[chunk_start:chunk_end]:
pull_specs.append(_pull_image_for_lang(lang, image, release))
# NOTE(rbellevi): We batch docker pull operations to maximize
# parallelism, without letting the disk usage grow unbounded.
pull_failures, _ = jobset.run(pull_specs,
newline_on_success=True,
maxjobs=max_pull_jobs)
if pull_failures:
jobset.message(
'FAILED',
'Image download failed. Skipping tests for language "%s"' %
lang,
do_newline=True)
skip_tests = True
for release, image in images[chunk_start:chunk_end]:
total_num_failures += _test_release(lang, runtime, release, image,
xml_report_tree, skip_tests)
if not args.keep:
for _, image in images[chunk_start:chunk_end]:
_cleanup_docker_image(image)
if not total_num_failures:
jobset.message('SUCCESS',
'All {} tests passed'.format(lang),
do_newline=True)
else:
jobset.message('FAILED',
'Some {} tests failed'.format(lang),
do_newline=True)
return total_num_failures
def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name):
perf_report_jobs = []
global profile_output_files
for host_and_port in hosts_and_base_names:
perf_base_name = hosts_and_base_names[host_and_port]
output_filename = '%s-%s' % (scenario_name, perf_base_name)
# from the base filename, create .svg output filename
host = host_and_port.split(':')[0]
profile_output_files.append('%s.svg' % output_filename)
perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename))
jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True)
failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1)
jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True)
return failures
def build_all_images_for_release(lang, release):
"""Build all docker images for a release across all runtimes."""
docker_images = []
build_jobs = []
env = {}
# If we not using current tree or the sibling for grpc stack, do checkout.
stack_base = ''
if args.git_checkout:
stack_base = checkout_grpc_stack(lang, release)
var = {
'go': 'GRPC_GO_ROOT',
'java': 'GRPC_JAVA_ROOT',
'node': 'GRPC_NODE_ROOT'
}.get(lang, 'GRPC_ROOT')
env[var] = stack_base
for runtime in client_matrix.get_runtimes_for_lang_release(lang, release):
job = build_image_jobspec(runtime, env, release, stack_base)
docker_images.append(job.tag)
build_jobs.append(job)
jobset.message('START', 'Building interop docker images.', do_newline=True)
print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
num_failures, _ = jobset.run(
build_jobs,
newline_on_success=True,
maxjobs=multiprocessing.cpu_count())
if num_failures:
jobset.message(
'FAILED', 'Failed to build interop docker images.', do_newline=True)
docker_images_cleanup.extend(docker_images)
sys.exit(1)
jobset.message(
'SUCCESS',
'All docker images built for %s at %s.' % (lang, release),
do_newline=True)
if release != 'master':
commit_log = os.path.join(stack_base, 'commit_log')
if os.path.exists(commit_log):
for image in docker_images:
add_files_to_image(image, [commit_log], 'release=%s' % release)
return docker_images
def archive_repo(languages):
"""Archives local version of repo including submodules."""
cmdline = ["tar", "-cf", "../grpc.tar", "../grpc/"]
if "java" in languages:
cmdline.append("../grpc-java")
if "go" in languages:
cmdline.append("../grpc-go")
archive_job = jobset.JobSpec(cmdline=cmdline, shortname="archive_repo", timeout_seconds=3 * 60)
jobset.message("START", "Archiving local repository.", do_newline=True)
num_failures, _ = jobset.run([archive_job], newline_on_success=True, maxjobs=1)
if num_failures == 0:
jobset.message("SUCCESS", "Archive with local repository created successfully.", do_newline=True)
else:
jobset.message("FAILED", "Failed to archive local repository.", do_newline=True)
sys.exit(1)
def build_all_images_for_release(lang, release):
"""Build all docker images for a release across all runtimes."""
docker_images = []
build_jobs = []
env = {}
# If we not using current tree or the sibling for grpc stack, do checkout.
stack_base = ''
if args.git_checkout:
stack_base = checkout_grpc_stack(lang, release)
var = {
'go': 'GRPC_GO_ROOT',
'java': 'GRPC_JAVA_ROOT',
'node': 'GRPC_NODE_ROOT'
}.get(lang, 'GRPC_ROOT')
env[var] = stack_base
for runtime in client_matrix.LANG_RUNTIME_MATRIX[lang]:
job = build_image_jobspec(runtime, env, release, stack_base)
docker_images.append(job.tag)
build_jobs.append(job)
jobset.message('START', 'Building interop docker images.', do_newline=True)
print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
num_failures, _ = jobset.run(
build_jobs,
newline_on_success=True,
maxjobs=multiprocessing.cpu_count())
if num_failures:
jobset.message(
'FAILED', 'Failed to build interop docker images.', do_newline=True)
docker_images_cleanup.extend(docker_images)
sys.exit(1)
jobset.message(
'SUCCESS',
'All docker images built for %s at %s.' % (lang, release),
do_newline=True)
if release != 'master':
commit_log = os.path.join(stack_base, 'commit_log')
if os.path.exists(commit_log):
for image in docker_images:
add_files_to_image(image, [commit_log], 'release=%s' % release)
return docker_images
def run_tests_for_lang(lang, runtime, images):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
# Fine to ignore return value as failure to download will result in test failure
# later anyway.
pull_images_for_lang(lang, images)
total_num_failures = 0
for release, image in images:
jobset.message('START', 'Testing %s' % image, do_newline=True)
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = find_test_cases(lang, runtime, release, suite_name)
if not job_spec_list:
jobset.message('FAILED',
'No test cases were found.',
do_newline=True)
return 1
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table, args)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(_xml_report_tree, resultset,
'grpc_interop_matrix',
suite_name, str(uuid.uuid4()))
if not args.keep:
cleanup(image)
return total_num_failures
def archive_repo():
"""Archives local version of repo including submodules."""
# TODO: also archive grpc-go and grpc-java repos
archive_job = jobset.JobSpec(
cmdline=['tar', '-cf', '../grpc.tar', '../grpc/'],
shortname='archive_repo',
timeout_seconds=3*60)
jobset.message('START', 'Archiving local repository.', do_newline=True)
num_failures, _ = jobset.run(
[archive_job], newline_on_success=True, maxjobs=1)
if num_failures == 0:
jobset.message('SUCCESS',
'Archive with local repository create successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Failed to archive local repository.',
do_newline=True)
sys.exit(1)
def run_tests_for_lang(lang, runtime, images):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
total_num_failures = 0
for image_tuple in images:
release, image = image_tuple
jobset.message('START', 'Testing %s' % image, do_newline=True)
# Download the docker image before running each test case.
subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = find_test_cases(lang, runtime, release, suite_name)
if not job_spec_list:
jobset.message(
'FAILED', 'No test cases were found.', do_newline=True)
return 1
num_failures, resultset = jobset.run(
job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table, args)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(_xml_report_tree, resultset,
'grpc_interop_matrix', suite_name,
str(uuid.uuid4()))
if not args.keep:
cleanup(image)
return total_num_failures
def prepare_remote_hosts(hosts):
"""Prepares remote hosts."""
prepare_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
prepare_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_prepare.sh'],
shortname='remote_host_prepare.%s' % host,
environ = {'USER_AT_HOST': user_at_host},
timeout_seconds=5*60))
jobset.message('START', 'Preparing remote hosts.', do_newline=True)
num_failures, _ = jobset.run(
prepare_jobs, newline_on_success=True, maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS',
'Remote hosts ready to start build.',
do_newline=True)
else:
jobset.message('FAILED', 'Failed to prepare remote hosts.',
do_newline=True)
sys.exit(1)
def run_tests_for_lang(lang, runtime, images):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
for image_tuple in images:
release, image = image_tuple
jobset.message('START', 'Testing %s' % image, do_newline=True)
_docker_images_cleanup.append(image)
job_spec_list = find_test_cases(lang, release)
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(
_xml_report_tree, resultset, 'grpc_interop_matrix',
'%s__%s:%s' % (lang, runtime, release), str(uuid.uuid4()))
def prepare_remote_hosts(hosts):
"""Prepares remote hosts."""
prepare_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
prepare_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_prepare.sh'],
shortname='remote_host_prepare.%s' % host,
environ = {'USER_AT_HOST': user_at_host},
timeout_seconds=3*60))
jobset.message('START', 'Preparing remote hosts.', do_newline=True)
num_failures, _ = jobset.run(
prepare_jobs, newline_on_success=True, maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS',
'Remote hosts ready to start build.',
do_newline=True)
else:
jobset.message('FAILED', 'Failed to prepare remote hosts.',
do_newline=True)
sys.exit(1)
def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name):
perf_report_jobs = []
global profile_output_files
for host_and_port in hosts_and_base_names:
perf_base_name = hosts_and_base_names[host_and_port]
output_filename = '%s-%s' % (scenario_name, perf_base_name)
# from the base filename, create .svg output filename
host = host_and_port.split(':')[0]
profile_output_files.append('%s.svg' % output_filename)
perf_report_jobs.append(
perf_report_processor_job(host, perf_base_name, output_filename))
jobset.message('START',
'Collecting perf reports from qps workers',
do_newline=True)
failures, _ = jobset.run(perf_report_jobs,
newline_on_success=True,
maxjobs=1)
jobset.message('END',
'Collecting perf reports from qps workers',
do_newline=True)
return failures
default=multiprocessing.cpu_count(),
help='Number of CPUs to use')
argp.set_defaults(fix=False)
args = argp.parse_args()
# Explicitly passing the .clang-tidy config by reading it.
# This is required because source files in the compilation database are
# in a different source tree so clang-tidy cannot find the right config file
# by seeking their parent directories.
with open(".clang-tidy") as f:
config = f.read()
cmdline = [
clang_tidy,
'--config=' + config,
]
if args.fix:
cmdline.append('--fix')
jobs = []
for filename in args.files:
jobs.append(
jobset.JobSpec(
cmdline + [filename],
shortname=filename,
timeout_seconds=15 * 60,
))
num_fails, res_set = jobset.run(jobs, maxjobs=args.jobs)
sys.exit(num_fails)
docker_images={}
if args.use_docker:
# languages for which to build docker images
languages_to_build = set(_LANGUAGES[k] for k in set([str(l) for l in languages] +
[s for s in servers]))
build_jobs = []
for l in languages_to_build:
job = build_interop_image_jobspec(l)
docker_images[str(l)] = job.tag
build_jobs.append(job)
if build_jobs:
jobset.message('START', 'Building interop docker images.', do_newline=True)
if jobset.run(build_jobs, newline_on_success=True, maxjobs=args.jobs):
jobset.message('SUCCESS', 'All docker images built successfully.', do_newline=True)
else:
jobset.message('FAILED', 'Failed to build interop docker images.', do_newline=True)
for image in docker_images.itervalues():
dockerjob.remove_image(image, skip_nonexistent=True)
exit(1);
# Start interop servers.
server_jobs={}
server_addresses={}
try:
for s in servers:
lang = str(s)
spec = server_jobspec(_LANGUAGES[lang], docker_images.get(lang))
job = dockerjob.DockerJob(spec)
# languages for which to build docker images
languages_to_build = set(_LANGUAGES[k]
for k in set([str(l) for l in languages] +
[s for s in servers]))
build_jobs = []
for l in languages_to_build:
job = build_interop_image_jobspec(l)
docker_images[str(l)] = job.tag
build_jobs.append(job)
if build_jobs:
jobset.message('START',
'Building interop docker images.',
do_newline=True)
if jobset.run(build_jobs, newline_on_success=True, maxjobs=args.jobs):
jobset.message('SUCCESS',
'All docker images built successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Failed to build interop docker images.',
do_newline=True)
for image in docker_images.itervalues():
dockerjob.remove_image(image, skip_nonexistent=True)
exit(1)
# Start interop servers.
server_jobs = {}
server_addresses = {}
try:
netperf_hosts=args.remote_worker_host)
if not scenarios:
raise Exception('No scenarios to run')
total_scenario_failures = 0
qps_workers_killed = 0
for scenario in scenarios:
if args.dry_run:
print(scenario.name)
else:
try:
for worker in scenario.workers:
worker.start()
scenario_failures, _ = jobset.run([
scenario.jobspec,
create_quit_jobspec(scenario.workers,
remote_host=args.remote_driver_host)
],
newline_on_success=True,
maxjobs=1)
total_scenario_failures += scenario_failures
finally:
# Consider qps workers that need to be killed as failures
qps_workers_killed += finish_qps_workers(scenario.workers)
if total_scenario_failures > 0 or qps_workers_killed > 0:
print("%s scenarios failed and %s qps worker jobs killed" %
(total_scenario_failures, qps_workers_killed))
sys.exit(1)
build_local = False
if not args.remote_driver_host:
build_local = True
build_on_remote_hosts(remote_hosts, build_local=build_local)
qpsworker_jobs = start_qpsworkers(args.remote_worker_host)
worker_addresses = [job.host_and_port for job in qpsworker_jobs]
try:
scenarios = create_scenarios(languages=[CXXLanguage()],
workers=worker_addresses,
remote_host=args.remote_driver_host)
if not scenarios:
raise Exception('No scenarios to run')
jobset.message('START', 'Running scenarios.', do_newline=True)
num_failures, _ = jobset.run(
scenarios, newline_on_success=True, maxjobs=1)
if num_failures == 0:
jobset.message('SUCCESS',
'All scenarios finished successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Some of the scenarios failed.',
do_newline=True)
sys.exit(1)
finally:
finish_qps_workers(qpsworker_jobs)
def _build_and_run(
check_cancelled, newline_on_success, cache, xml_report=None, build_only=False):
"""Do one pass of building & running tests."""
# build latest sequentially
num_failures, resultset = jobset.run(
build_steps, maxjobs=1, stop_on_failure=True,
newline_on_success=newline_on_success, travis=args.travis)
if num_failures:
return [BuildAndRunError.BUILD]
if build_only:
if xml_report:
report_utils.render_junit_xml_report(resultset, xml_report)
return []
# start antagonists
antagonists = [subprocess.Popen(['tools/run_tests/antagonist.py'])
for _ in range(0, args.antagonists)]
port_server_port = 32767
_start_port_server(port_server_port)
resultset = None
num_test_failures = 0
try:
infinite_runs = runs_per_test == 0
one_run = set(
spec
for language in languages
for spec in language.test_specs()
if re.search(args.regex, spec.shortname))
# When running on travis, we want out test runs to be as similar as possible
# for reproducibility purposes.
if args.travis:
massaged_one_run = sorted(one_run, key=lambda x: x.shortname)
else:
# whereas otherwise, we want to shuffle things up to give all tests a
# chance to run.
massaged_one_run = list(one_run) # random.shuffle needs an indexable seq.
random.shuffle(massaged_one_run) # which it modifies in-place.
if infinite_runs:
assert len(massaged_one_run) > 0, 'Must have at least one test for a -n inf run'
runs_sequence = (itertools.repeat(massaged_one_run) if infinite_runs
else itertools.repeat(massaged_one_run, runs_per_test))
all_runs = itertools.chain.from_iterable(runs_sequence)
num_test_failures, resultset = jobset.run(
all_runs, check_cancelled, newline_on_success=newline_on_success,
travis=args.travis, infinite_runs=infinite_runs, maxjobs=args.jobs,
stop_on_failure=args.stop_on_failure,
cache=cache if not xml_report else None,
add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
if resultset:
for k, v in resultset.iteritems():
num_runs, num_failures = _calculate_num_runs_failures(v)
if num_failures == num_runs: # what about infinite_runs???
jobset.message('FAILED', k, do_newline=True)
elif num_failures > 0:
jobset.message(
'FLAKE', '%s [%d/%d runs flaked]' % (k, num_failures, num_runs),
do_newline=True)
else:
jobset.message('PASSED', k, do_newline=True)
finally:
for antagonist in antagonists:
antagonist.kill()
if xml_report and resultset:
report_utils.render_junit_xml_report(resultset, xml_report)
number_failures, _ = jobset.run(
post_tests_steps, maxjobs=1, stop_on_failure=True,
newline_on_success=newline_on_success, travis=args.travis)
out = []
if number_failures:
out.append(BuildAndRunError.POST_TEST)
if num_test_failures:
out.append(BuildAndRunError.TEST)
if cache: cache.save()
return out
def _build_and_run(check_cancelled,
newline_on_success,
travis,
cache,
xml_report=None):
"""Do one pass of building & running tests."""
# build latest sequentially
if not jobset.run(build_steps,
maxjobs=1,
stop_on_failure=True,
newline_on_success=newline_on_success,
travis=travis):
return 1
# start antagonists
antagonists = [
subprocess.Popen(['tools/run_tests/antagonist.py'])
for _ in range(0, args.antagonists)
]
port_server_port = 32767
_start_port_server(port_server_port)
try:
infinite_runs = runs_per_test == 0
one_run = set(spec for config in run_configs for language in languages
for spec in language.test_specs(config, args.travis)
if re.search(args.regex, spec.shortname))
# When running on travis, we want out test runs to be as similar as possible
# for reproducibility purposes.
if travis:
massaged_one_run = sorted(one_run, key=lambda x: x.shortname)
else:
# whereas otherwise, we want to shuffle things up to give all tests a
# chance to run.
massaged_one_run = list(
one_run) # random.shuffle needs an indexable seq.
random.shuffle(massaged_one_run) # which it modifies in-place.
if infinite_runs:
assert len(massaged_one_run
) > 0, 'Must have at least one test for a -n inf run'
runs_sequence = (itertools.repeat(massaged_one_run)
if infinite_runs else itertools.repeat(
massaged_one_run, runs_per_test))
all_runs = itertools.chain.from_iterable(runs_sequence)
root = ET.Element('testsuites') if xml_report else None
testsuite = ET.SubElement(
root, 'testsuite', id='1', package='grpc',
name='tests') if xml_report else None
if not jobset.run(all_runs,
check_cancelled,
newline_on_success=newline_on_success,
travis=travis,
infinite_runs=infinite_runs,
maxjobs=args.jobs,
stop_on_failure=args.stop_on_failure,
cache=cache if not xml_report else None,
xml_report=testsuite,
add_env={
'GRPC_TEST_PORT_SERVER':
'localhost:%d' % port_server_port
}):
return 2
finally:
for antagonist in antagonists:
antagonist.kill()
if xml_report:
tree = ET.ElementTree(root)
tree.write(xml_report, encoding='UTF-8')
if cache: cache.save()
return 0
print
if len(relevant_jobs) == len(jobs):
print '(TESTING) No tests will be skipped.'
else:
print '(TESTING) These tests will be skipped:'
for job in list(set(jobs) - set(relevant_jobs)):
print ' %s' % job.shortname
print
if args.dry_run:
print '--dry_run was used, exiting'
sys.exit(1)
jobset.message('START', 'Running test matrix.', do_newline=True)
num_failures, resultset = jobset.run(jobs,
newline_on_success=True,
travis=True,
maxjobs=args.jobs)
report_utils.render_junit_xml_report(resultset,
'report.xml',
suite_name='aggregate_tests')
if num_failures == 0:
jobset.message('SUCCESS',
'All run_tests.py instance finished successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Some run_tests.py instance have failed.',
do_newline=True)
sys.exit(1)
relevant_jobs = filter_tests(jobs, args.base_branch)
# todo(mattkwong): add skipped tests to report.xml
print
if len(relevant_jobs) == len(jobs):
print '(TESTING) No tests will be skipped.'
else:
print '(TESTING) These tests will be skipped:'
for job in list(set(jobs) - set(relevant_jobs)):
print ' %s' % job.shortname
print
if args.dry_run:
print '--dry_run was used, exiting'
sys.exit(1)
jobset.message('START', 'Running test matrix.', do_newline=True)
num_failures, resultset = jobset.run(jobs,
newline_on_success=True,
travis=True,
maxjobs=args.jobs)
report_utils.render_junit_xml_report(resultset, 'report.xml',
suite_name='aggregate_tests')
if num_failures == 0:
jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Some run_tests.py instance have failed.',
do_newline=True)
sys.exit(1)
if os.path.splitext(f)[1] == '.template':
out_dir = '.' + root[len('templates'):]
out = out_dir + '/' + os.path.splitext(f)[0]
if not os.path.exists(out_dir):
os.makedirs(out_dir)
cmd = base_cmd[:]
cmd.append('-P')
cmd.append(preprocessed_build)
cmd.append('-o')
if test is None:
cmd.append(out)
else:
tf = tempfile.mkstemp()
test[out] = tf[1]
os.close(tf[0])
cmd.append(test[out])
cmd.append(root + '/' + f)
jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None))
jobset.run(pre_jobs, maxjobs=args.jobs)
jobset.run(jobs, maxjobs=args.jobs)
if test is not None:
for s, g in test.iteritems():
if os.path.isfile(g):
assert 0 == os.system('diff %s %s' % (s, g)), s
os.unlink(g)
else:
assert 0 == os.system('diff -r %s %s' % (s, g)), s
shutil.rmtree(g, ignore_errors=True)
'--config=' + config,
]
if args.fix:
cmdline.append('--fix')
if args.only_changed:
orig_files = set(args.files)
actual_files = []
output = subprocess.check_output(
['git', 'diff', 'origin/master', 'HEAD', '--name-only'])
for line in output.decode('ascii').splitlines(False):
if line in orig_files:
print("check: %s" % line)
actual_files.append(line)
else:
print("skip: %s - not in the build" % line)
args.files = actual_files
jobs = []
for filename in args.files:
jobs.append(
jobset.JobSpec(
cmdline + [filename],
shortname=filename,
timeout_seconds=15 * 60,
))
num_fails, res_set = jobset.run(jobs, maxjobs=args.jobs, quiet_success=True)
sys.exit(num_fails)
if args.use_docker:
# languages for which to build docker images
languages_to_build = set(_LANGUAGES[k] for k in set([str(l) for l in languages] +
[s for s in servers]))
if args.http2_interop:
languages_to_build.add(http2Interop)
build_jobs = []
for l in languages_to_build:
job = build_interop_image_jobspec(l)
docker_images[str(l)] = job.tag
build_jobs.append(job)
if build_jobs:
jobset.message('START', 'Building interop docker images.', do_newline=True)
num_failures, _ = jobset.run(
build_jobs, newline_on_success=True, maxjobs=args.jobs)
if num_failures == 0:
jobset.message('SUCCESS', 'All docker images built successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Failed to build interop docker images.',
do_newline=True)
for image in docker_images.itervalues():
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
# Start interop servers.
server_jobs={}
server_addresses={}
try:
for s in servers:
print('Will run these tests:')
for job in jobs:
if args.dry_run:
print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline)))
else:
print(' %s' % job.shortname)
print
if args.dry_run:
print('--dry_run was used, exiting')
sys.exit(1)
jobset.message('START', 'Running test matrix.', do_newline=True)
num_failures, resultset = jobset.run(jobs,
newline_on_success=True,
travis=True,
maxjobs=args.jobs)
# Merge skipped tests into results to show skipped tests on report.xml
if skipped_jobs:
skipped_results = jobset.run(skipped_jobs, skip_jobs=True)
resultset.update(skipped_results)
report_utils.render_junit_xml_report(resultset,
'report.xml',
suite_name='aggregate_tests')
if num_failures == 0:
jobset.message('SUCCESS',
'All run_tests.py instance finished successfully.',
do_newline=True)
else:
jobset.message('FAILED',
argp.add_argument("-l", "--language", default="c++")
args = argp.parse_args()
# build job
build_job = jobset.JobSpec(
cmdline=["tools/run_tests/run_interops_build.sh", "%s" % args.language], shortname="build", timeout_seconds=30 * 60
)
# test jobs, each test is a separate job to run in parallel
_TESTS = ["large_unary", "empty_unary", "ping_pong", "client_streaming", "server_streaming"]
jobs = []
jobNumber = 0
for test in _TESTS:
test_job = jobset.JobSpec(
cmdline=["tools/run_tests/run_interops_test.sh", "%s" % args.language, "%s" % test],
shortname=test,
timeout_seconds=15 * 60,
)
jobs.append(test_job)
jobNumber += 1
root = ET.Element("testsuites")
testsuite = ET.SubElement(root, "testsuite", id="1", package="grpc", name="tests")
# always do the build of docker first, and then all the tests can run in parallel
jobset.run([build_job], maxjobs=1, xml_report=testsuite)
jobset.run(jobs, maxjobs=jobNumber, xml_report=testsuite)
tree = ET.ElementTree(root)
tree.write("report.xml", encoding="UTF-8")
(bm, stripped_line, cfg, name, idx),
'--benchmark_out_format=json',
]
jobs_list.append(
jobset.JobSpec(cmd,
shortname='%s %s %s %s %d/%d' %
(bm, line, cfg, name, idx + 1, loops),
verbose_success=True,
cpu_cost=2,
timeout_seconds=60 * 60)) # one hour
return jobs_list
def create_jobs(name, benchmarks, loops, regex, counters):
jobs_list = []
for loop in range(0, loops):
for bm in benchmarks:
jobs_list += _collect_bm_data(bm, 'opt', name, regex, loop, loops)
if counters:
jobs_list += _collect_bm_data(bm, 'counters', name, regex,
loop, loops)
random.shuffle(jobs_list, random.SystemRandom().random)
return jobs_list
if __name__ == '__main__':
args = _args()
jobs_list = create_jobs(args.name, args.benchmarks, args.loops, args.regex,
args.counters)
jobset.run(jobs_list, maxjobs=args.jobs)
print('Will run these tests:')
for job in jobs:
if args.dry_run:
print(' %s: "%s"' % (job.shortname, ' '.join(job.cmdline)))
else:
print(' %s' % job.shortname)
print
if args.dry_run:
print('--dry_run was used, exiting')
sys.exit(1)
jobset.message('START', 'Running test matrix.', do_newline=True)
num_failures, resultset = jobset.run(jobs,
newline_on_success=True,
travis=True,
maxjobs=args.jobs)
# Merge skipped tests into results to show skipped tests on report.xml
if skipped_jobs:
skipped_results = jobset.run(skipped_jobs,
skip_jobs=True)
resultset.update(skipped_results)
report_utils.render_junit_xml_report(resultset, 'report.xml',
suite_name='aggregate_tests')
if num_failures == 0:
jobset.message('SUCCESS', 'All run_tests.py instance finished successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Some run_tests.py instance have failed.',
do_newline=True)
if os.path.splitext(f)[1] == '.template':
out_dir = args.base + root[len('templates'):]
out = out_dir + '/' + os.path.splitext(f)[0]
if not os.path.exists(out_dir):
os.makedirs(out_dir)
cmd = base_cmd[:]
cmd.append('-P')
cmd.append(preprocessed_build)
cmd.append('-o')
if test is None:
cmd.append(out)
else:
tf = tempfile.mkstemp()
test[out] = tf[1]
os.close(tf[0])
cmd.append(test[out])
cmd.append(args.base + '/' + root + '/' + f)
jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None))
jobset.run(pre_jobs, maxjobs=args.jobs)
jobset.run(jobs, maxjobs=args.jobs)
if test is not None:
for s, g in test.iteritems():
if os.path.isfile(g):
assert 0 == os.system('diff %s %s' % (s, g)), s
os.unlink(g)
else:
assert 0 == os.system('diff -r %s %s' % (s, g)), s
shutil.rmtree(g, ignore_errors=True)
# get list of worker addresses for each language.
worker_addresses = dict([(str(language), []) for language in languages])
for job in qpsworker_jobs:
worker_addresses[str(job.language)].append(job.host_and_port)
try:
scenarios = create_scenarios(languages,
workers_by_lang=worker_addresses,
remote_host=args.remote_driver_host,
regex=args.regex,
bq_result_table=args.bq_result_table)
if not scenarios:
raise Exception('No scenarios to run')
jobset.message('START', 'Running scenarios.', do_newline=True)
num_failures, _ = jobset.run(scenarios, newline_on_success=True, maxjobs=1)
if num_failures == 0:
jobset.message('SUCCESS',
'All scenarios finished successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Some of the scenarios failed.',
do_newline=True)
sys.exit(1)
except:
traceback.print_exc()
raise
finally:
finish_qps_workers(qpsworker_jobs)
if not scenarios:
raise Exception('No scenarios to run')
total_scenario_failures = 0
qps_workers_killed = 0
merged_resultset = {}
for scenario in scenarios:
if args.dry_run:
print(scenario.name)
else:
try:
for worker in scenario.workers:
worker.start()
scenario_failures, resultset = jobset.run([scenario.jobspec,
create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host)],
newline_on_success=True, maxjobs=1)
total_scenario_failures += scenario_failures
merged_resultset = dict(itertools.chain(merged_resultset.iteritems(),
resultset.iteritems()))
finally:
# Consider qps workers that need to be killed as failures
qps_workers_killed += finish_qps_workers(scenario.workers)
report_utils.render_junit_xml_report(merged_resultset, 'report.xml',
suite_name='benchmarks')
if total_scenario_failures > 0 or qps_workers_killed > 0:
print ("%s scenarios failed and %s qps worker jobs killed" % (total_scenario_failures, qps_workers_killed))
sys.exit(1)
for scenario in scenarios:
if args.dry_run:
print(scenario.name)
else:
scenario_failures = 0
try:
for worker in scenario.workers:
worker.start()
jobs = [scenario.jobspec]
if scenario.workers:
jobs.append(
create_quit_jobspec(scenario.workers,
remote_host=args.remote_driver_host))
scenario_failures, resultset = jobset.run(jobs,
newline_on_success=True,
maxjobs=1)
total_scenario_failures += scenario_failures
merged_resultset = dict(
itertools.chain(merged_resultset.iteritems(),
resultset.iteritems()))
finally:
# Consider qps workers that need to be killed as failures
qps_workers_killed += finish_qps_workers(scenario.workers)
if perf_cmd and scenario_failures == 0 and not args.skip_generate_flamegraphs:
workers_and_base_names = {}
for worker in scenario.workers:
if not worker.perf_file_base_name:
raise Exception(
'using perf buf perf report filename is unspecified')
for f in files:
if os.path.splitext(f)[1] == '.template':
out_dir = '.' + root[len('templates'):]
out = out_dir + '/' + os.path.splitext(f)[0]
if not os.path.exists(out_dir):
os.makedirs(out_dir)
cmd = ['python', 'tools/buildgen/mako_renderer.py']
for plugin in plugins:
cmd.append('-p')
cmd.append(plugin)
for js in json:
cmd.append('-d')
cmd.append(js)
cmd.append('-o')
if test is None:
cmd.append(out)
else:
tf = tempfile.mkstemp()
test[out] = tf[1]
os.close(tf[0])
cmd.append(test[out])
cmd.append(root + '/' + f)
jobs.append(jobset.JobSpec(cmd, shortname=out))
jobset.run(jobs)
if test is not None:
for s, g in test.iteritems():
assert (0 == os.system('diff %s %s' % (s, g)))
os.unlink(g)
[s for s in servers]))
if args.http2_interop:
languages_to_build.add(http2Interop)
build_jobs = []
for l in languages_to_build:
job = build_interop_image_jobspec(l)
docker_images[str(l)] = job.tag
build_jobs.append(job)
if build_jobs:
jobset.message('START',
'Building interop docker images.',
do_newline=True)
num_failures, _ = jobset.run(build_jobs,
newline_on_success=True,
maxjobs=args.jobs)
if num_failures == 0:
jobset.message('SUCCESS',
'All docker images built successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Failed to build interop docker images.',
do_newline=True)
for image in docker_images.itervalues():
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
# Start interop servers.
server_jobs = {}
qps_workers_killed = 0
merged_resultset = {}
perf_report_failures = 0
for scenario in scenarios:
if args.dry_run:
print(scenario.name)
else:
scenario_failures = 0
try:
for worker in scenario.workers:
worker.start()
jobs = [scenario.jobspec]
if scenario.workers:
jobs.append(create_quit_jobspec(scenario.workers, remote_host=args.remote_driver_host))
scenario_failures, resultset = jobset.run(jobs, newline_on_success=True, maxjobs=1)
total_scenario_failures += scenario_failures
merged_resultset = dict(itertools.chain(merged_resultset.iteritems(),
resultset.iteritems()))
finally:
# Consider qps workers that need to be killed as failures
qps_workers_killed += finish_qps_workers(scenario.workers)
if perf_cmd and scenario_failures == 0 and not args.skip_generate_flamegraphs:
workers_and_base_names = {}
for worker in scenario.workers:
if not worker.perf_file_base_name:
raise Exception('using perf buf perf report filename is unspecified')
workers_and_base_names[worker.host_and_port] = worker.perf_file_base_name
perf_report_failures += run_collect_perf_profile_jobs(workers_and_base_names, scenario.name)
jobs = []
for root, dirs, files in os.walk('templates'):
for f in files:
if os.path.splitext(f)[1] == '.template':
out = '.' + root[len('templates'):] + '/' + os.path.splitext(f)[0]
cmd = ['tools/buildgen/mako_renderer.py']
for plugin in plugins:
cmd.append('-p')
cmd.append(plugin)
for js in json:
cmd.append('-d')
cmd.append(js)
cmd.append('-o')
if test is None:
cmd.append(out)
else:
tf = tempfile.mkstemp()
test[out] = tf[1]
os.close(tf[0])
cmd.append(test[out])
cmd.append(root + '/' + f)
jobs.append(jobset.JobSpec(cmd, shortname=out))
jobset.run(jobs)
if test is not None:
for s, g in test.iteritems():
assert(0 == os.system('diff %s %s' % (s, g)))
os.unlink(g)
