def _test_release(lang, runtime, release, image, xml_report_tree, skip_tests): total_num_failures = 0 suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = _generate_test_case_jobspecs(lang, runtime, release, suite_name) if not job_spec_list: jobset.message('FAILED', 'No test cases were found.', do_newline=True) total_num_failures += 1 else: num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs, skip_jobs=skip_tests) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table) if skip_tests: jobset.message('FAILED', 'Tests were skipped', do_newline=True) total_num_failures += 1 if num_failures: total_num_failures += num_failures report_utils.append_junit_xml_results(xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) return total_num_failures
def run_tests_for_lang(lang, runtime, images): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ for image_tuple in images: release, image = image_tuple jobset.message('START', 'Testing %s' % image, do_newline=True) # Download the docker image before running each test case. subprocess.check_call(['gcloud', 'docker', '--', 'pull', image]) _docker_images_cleanup.append(image) job_spec_list = find_test_cases(lang,release) num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image':image}, maxjobs=args.jobs) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results( _xml_report_tree, resultset, 'grpc_interop_matrix', '%s__%s %s'%(lang,runtime,release), str(uuid.uuid4()))
def _run_tests_for_lang(lang, runtime, images, xml_report_tree): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ skip_tests = False if not _pull_images_for_lang(lang, images): jobset.message( 'FAILED', 'Image download failed. Skipping tests for language "%s"' % lang, do_newline=True) skip_tests = True total_num_failures = 0 for release, image in images: suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = _generate_test_case_jobspecs(lang, runtime, release, suite_name) if not job_spec_list: jobset.message('FAILED', 'No test cases were found.', do_newline=True) total_num_failures += 1 continue num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs, skip_jobs=skip_tests) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table) if skip_tests: jobset.message('FAILED', 'Tests were skipped', do_newline=True) total_num_failures += 1 elif num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results(xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) # cleanup all downloaded docker images for _, image in images: if not args.keep: _cleanup_docker_image(image) return total_num_failures
def _run_tests_for_lang(lang, runtime, images, xml_report_tree): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ skip_tests = False if not _pull_images_for_lang(lang, images): jobset.message( 'FAILED', 'Image download failed. Skipping tests for language "%s"' % lang, do_newline=True) skip_tests = True total_num_failures = 0 for release, image in images: suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = _generate_test_case_jobspecs(lang, runtime, release, suite_name) if not job_spec_list: jobset.message( 'FAILED', 'No test cases were found.', do_newline=True) total_num_failures += 1 continue num_failures, resultset = jobset.run( job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs, skip_jobs=skip_tests) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table) if skip_tests: jobset.message('FAILED', 'Tests were skipped', do_newline=True) total_num_failures += 1 elif num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results(xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) # cleanup all downloaded docker images for _, image in images: if not args.keep: _cleanup_docker_image(image) return total_num_failures
def run_tests_for_lang(lang, runtime, images): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ # Fine to ignore return value as failure to download will result in test failure # later anyway. pull_images_for_lang(lang, images) total_num_failures = 0 for release, image in images: jobset.message('START', 'Testing %s' % image, do_newline=True) suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = find_test_cases(lang, runtime, release, suite_name) if not job_spec_list: jobset.message('FAILED', 'No test cases were found.', do_newline=True) return 1 num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table, args) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results(_xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) if not args.keep: cleanup(image) return total_num_failures
def run_tests_for_lang(lang, runtime, images): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ total_num_failures = 0 for image_tuple in images: release, image = image_tuple jobset.message('START', 'Testing %s' % image, do_newline=True) # Download the docker image before running each test case. subprocess.check_call(['gcloud', 'docker', '--', 'pull', image]) suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = find_test_cases(lang, runtime, release, suite_name) if not job_spec_list: jobset.message( 'FAILED', 'No test cases were found.', do_newline=True) return 1 num_failures, resultset = jobset.run( job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table, args) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results(_xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) if not args.keep: cleanup(image) return total_num_failures
def run_tests_for_lang(lang, runtime, images): """Find and run all test cases for a language. images is a list of (<release-tag>, <image-full-path>) tuple. """ total_num_failures = 0 for image_tuple in images: release, image = image_tuple jobset.message('START', 'Testing %s' % image, do_newline=True) # Download the docker image before running each test case. subprocess.check_call(['gcloud', 'docker', '--', 'pull', image]) suite_name = '%s__%s_%s' % (lang, runtime, release) job_spec_list = find_test_cases(lang, runtime, release, suite_name) if not job_spec_list: jobset.message('FAILED', 'No test cases were found.', do_newline=True) return 1 num_failures, resultset = jobset.run(job_spec_list, newline_on_success=True, add_env={'docker_image': image}, maxjobs=args.jobs) if args.bq_result_table and resultset: upload_test_results.upload_interop_results_to_bq( resultset, args.bq_result_table) if num_failures: jobset.message('FAILED', 'Some tests failed', do_newline=True) total_num_failures += num_failures else: jobset.message('SUCCESS', 'All tests passed', do_newline=True) report_utils.append_junit_xml_results(_xml_report_tree, resultset, 'grpc_interop_matrix', suite_name, str(uuid.uuid4())) if not args.keep: cleanup(image) return total_num_failures