def _test_release(lang, runtime, release, image, xml_report_tree, skip_tests):
total_num_failures = 0
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = _generate_test_case_jobspecs(lang, runtime, release,
suite_name)
if not job_spec_list:
jobset.message('FAILED', 'No test cases were found.', do_newline=True)
total_num_failures += 1
else:
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs,
skip_jobs=skip_tests)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table)
if skip_tests:
jobset.message('FAILED', 'Tests were skipped', do_newline=True)
total_num_failures += 1
if num_failures:
total_num_failures += num_failures
report_utils.append_junit_xml_results(xml_report_tree, resultset,
'grpc_interop_matrix',
suite_name, str(uuid.uuid4()))
return total_num_failures
def run_tests_for_lang(lang, runtime, images):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
for image_tuple in images:
release, image = image_tuple
jobset.message('START', 'Testing %s' % image, do_newline=True)
# Download the docker image before running each test case.
subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
_docker_images_cleanup.append(image)
job_spec_list = find_test_cases(lang,release)
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image':image},
maxjobs=args.jobs)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(
_xml_report_tree,
resultset,
'grpc_interop_matrix',
'%s__%s %s'%(lang,runtime,release),
str(uuid.uuid4()))
def _run_tests_for_lang(lang, runtime, images, xml_report_tree):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
skip_tests = False
if not _pull_images_for_lang(lang, images):
jobset.message(
'FAILED',
'Image download failed. Skipping tests for language "%s"' % lang,
do_newline=True)
skip_tests = True
total_num_failures = 0
for release, image in images:
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = _generate_test_case_jobspecs(lang, runtime, release,
suite_name)
if not job_spec_list:
jobset.message('FAILED',
'No test cases were found.',
do_newline=True)
total_num_failures += 1
continue
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs,
skip_jobs=skip_tests)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table)
if skip_tests:
jobset.message('FAILED', 'Tests were skipped', do_newline=True)
total_num_failures += 1
elif num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(xml_report_tree, resultset,
'grpc_interop_matrix',
suite_name, str(uuid.uuid4()))
# cleanup all downloaded docker images
for _, image in images:
if not args.keep:
_cleanup_docker_image(image)
return total_num_failures
def _run_tests_for_lang(lang, runtime, images, xml_report_tree):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
skip_tests = False
if not _pull_images_for_lang(lang, images):
jobset.message(
'FAILED',
'Image download failed. Skipping tests for language "%s"' % lang,
do_newline=True)
skip_tests = True
total_num_failures = 0
for release, image in images:
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = _generate_test_case_jobspecs(lang, runtime, release,
suite_name)
if not job_spec_list:
jobset.message(
'FAILED', 'No test cases were found.', do_newline=True)
total_num_failures += 1
continue
num_failures, resultset = jobset.run(
job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs,
skip_jobs=skip_tests)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table)
if skip_tests:
jobset.message('FAILED', 'Tests were skipped', do_newline=True)
total_num_failures += 1
elif num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(xml_report_tree, resultset,
'grpc_interop_matrix', suite_name,
str(uuid.uuid4()))
# cleanup all downloaded docker images
for _, image in images:
if not args.keep:
_cleanup_docker_image(image)
return total_num_failures
def run_tests_for_lang(lang, runtime, images):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
# Fine to ignore return value as failure to download will result in test failure
# later anyway.
pull_images_for_lang(lang, images)
total_num_failures = 0
for release, image in images:
jobset.message('START', 'Testing %s' % image, do_newline=True)
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = find_test_cases(lang, runtime, release, suite_name)
if not job_spec_list:
jobset.message('FAILED',
'No test cases were found.',
do_newline=True)
return 1
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table, args)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(_xml_report_tree, resultset,
'grpc_interop_matrix',
suite_name, str(uuid.uuid4()))
if not args.keep:
cleanup(image)
return total_num_failures
def run_tests_for_lang(lang, runtime, images):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
total_num_failures = 0
for image_tuple in images:
release, image = image_tuple
jobset.message('START', 'Testing %s' % image, do_newline=True)
# Download the docker image before running each test case.
subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = find_test_cases(lang, runtime, release, suite_name)
if not job_spec_list:
jobset.message(
'FAILED', 'No test cases were found.', do_newline=True)
return 1
num_failures, resultset = jobset.run(
job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table, args)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(_xml_report_tree, resultset,
'grpc_interop_matrix', suite_name,
str(uuid.uuid4()))
if not args.keep:
cleanup(image)
return total_num_failures
def run_tests_for_lang(lang, runtime, images):
"""Find and run all test cases for a language.
images is a list of (<release-tag>, <image-full-path>) tuple.
"""
total_num_failures = 0
for image_tuple in images:
release, image = image_tuple
jobset.message('START', 'Testing %s' % image, do_newline=True)
# Download the docker image before running each test case.
subprocess.check_call(['gcloud', 'docker', '--', 'pull', image])
suite_name = '%s__%s_%s' % (lang, runtime, release)
job_spec_list = find_test_cases(lang, runtime, release, suite_name)
if not job_spec_list:
jobset.message('FAILED',
'No test cases were found.',
do_newline=True)
return 1
num_failures, resultset = jobset.run(job_spec_list,
newline_on_success=True,
add_env={'docker_image': image},
maxjobs=args.jobs)
if args.bq_result_table and resultset:
upload_test_results.upload_interop_results_to_bq(
resultset, args.bq_result_table)
if num_failures:
jobset.message('FAILED', 'Some tests failed', do_newline=True)
total_num_failures += num_failures
else:
jobset.message('SUCCESS', 'All tests passed', do_newline=True)
report_utils.append_junit_xml_results(_xml_report_tree, resultset,
'grpc_interop_matrix',
suite_name, str(uuid.uuid4()))
if not args.keep:
cleanup(image)
return total_num_failures
