def dns_server_in_docker_jobspec(grpclb_ips, fallback_ips, shortname,
cause_no_error_no_data_for_balancer_a_record):
container_name = dockerjob.random_name(shortname)
run_dns_server_cmdline = [
'python',
'test/cpp/naming/utils/run_dns_server_for_lb_interop_tests.py',
'--grpclb_ips=%s' % ','.join(grpclb_ips),
'--fallback_ips=%s' % ','.join(fallback_ips),
]
if cause_no_error_no_data_for_balancer_a_record:
run_dns_server_cmdline.append(
'--cause_no_error_no_data_for_balancer_a_record')
docker_cmdline = docker_run_cmdline(
run_dns_server_cmdline,
cwd='/var/local/git/grpc',
image=docker_images.get(_FAKE_SERVERS_SAFENAME),
docker_args=['--name=%s' % container_name])
jobset.message(
'IDLE',
'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline),
do_newline=True)
server_job = jobset.JobSpec(
cmdline=docker_cmdline, shortname=shortname, timeout_seconds=30 * 60)
server_job.container_name = container_name
return server_job
def dns_server_in_docker_jobspec(grpclb_ips, fallback_ips, shortname,
cause_no_error_no_data_for_balancer_a_record):
container_name = dockerjob.random_name(shortname)
run_dns_server_cmdline = [
'python',
'test/cpp/naming/utils/run_dns_server_for_lb_interop_tests.py',
'--grpclb_ips=%s' % ','.join(grpclb_ips),
'--fallback_ips=%s' % ','.join(fallback_ips),
]
if cause_no_error_no_data_for_balancer_a_record:
run_dns_server_cmdline.append(
'--cause_no_error_no_data_for_balancer_a_record')
docker_cmdline = docker_run_cmdline(
run_dns_server_cmdline,
cwd='/var/local/git/grpc',
image=docker_images.get(_FAKE_SERVERS_SAFENAME),
docker_args=['--name=%s' % container_name])
jobset.message('IDLE',
'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline),
do_newline=True)
server_job = jobset.JobSpec(cmdline=docker_cmdline,
shortname=shortname,
timeout_seconds=30 * 60)
server_job.container_name = container_name
return server_job
def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name,
flame_graph_reports):
perf_report_jobs = []
global profile_output_files
for host_and_port in hosts_and_base_names:
perf_base_name = hosts_and_base_names[host_and_port]
output_filename = '%s-%s' % (scenario_name, perf_base_name)
# from the base filename, create .svg output filename
host = host_and_port.split(':')[0]
profile_output_files.append('%s.svg' % output_filename)
perf_report_jobs.append(
perf_report_processor_job(host, perf_base_name, output_filename,
flame_graph_reports))
jobset.message('START',
'Collecting perf reports from qps workers',
do_newline=True)
failures, _ = jobset.run(perf_report_jobs,
newline_on_success=True,
maxjobs=1,
clear_alarms=False)
jobset.message('END',
'Collecting perf reports from qps workers',
do_newline=True)
return failures
def run_collect_perf_profile_jobs(hosts_and_base_names, scenario_name):
perf_report_jobs = []
global profile_output_files
for host_and_port in hosts_and_base_names:
perf_base_name = hosts_and_base_names[host_and_port]
output_filename = '%s-%s' % (scenario_name, perf_base_name)
# from the base filename, create .svg output filename
host = host_and_port.split(':')[0]
profile_output_files.append('%s.svg' % output_filename)
perf_report_jobs.append(perf_report_processor_job(host, perf_base_name, output_filename))
jobset.message('START', 'Collecting perf reports from qps workers', do_newline=True)
failures, _ = jobset.run(perf_report_jobs, newline_on_success=True, maxjobs=1)
jobset.message('END', 'Collecting perf reports from qps workers', do_newline=True)
return failures
def lb_client_interop_jobspec(language,
dns_server_ip,
docker_image,
transport_security='tls'):
"""Runs a gRPC client under test in a docker container"""
interop_only_options = [
'--server_host=%s' % _SERVICE_NAME,
'--server_port=%d' % _FALLBACK_SERVER_PORT
] + transport_security_to_args(transport_security)
# Don't set the server host override in any client;
# Go and Java default to no override.
# We're using a DNS server so there's no need.
if language.safename == 'c++':
interop_only_options += ['--server_host_override=""']
# Don't set --use_test_ca; we're configuring
# clients to use test CA's via alternate means.
interop_only_options += ['--use_test_ca=false']
client_args = language.client_cmd(interop_only_options)
container_name = dockerjob.random_name(
'lb_interop_client_%s' % language.safename)
docker_cmdline = docker_run_cmdline(
client_args,
environ=language.global_env(),
image=docker_image,
cwd=language.client_cwd,
docker_args=[
'--dns=%s' % dns_server_ip,
'--net=host',
'--name=%s' % container_name,
'-v',
'{grpc_grpc_root_dir}:/external_mount:ro'.format(
grpc_grpc_root_dir=ROOT),
])
jobset.message(
'IDLE',
'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline),
do_newline=True)
test_job = jobset.JobSpec(
cmdline=docker_cmdline,
shortname=('lb_interop_client:%s' % language),
timeout_seconds=_TEST_TIMEOUT,
kill_handler=_job_kill_handler)
test_job.container_name = container_name
return test_job
def lb_client_interop_jobspec(language,
dns_server_ip,
docker_image,
transport_security='tls'):
"""Runs a gRPC client under test in a docker container"""
interop_only_options = [
'--server_host=%s' % _SERVICE_NAME,
'--server_port=%d' % _FALLBACK_SERVER_PORT
] + transport_security_to_args(transport_security)
# Don't set the server host override in any client;
# Go and Java default to no override.
# We're using a DNS server so there's no need.
if language.safename == 'c++':
interop_only_options += ['--server_host_override=""']
# Don't set --use_test_ca; we're configuring
# clients to use test CA's via alternate means.
interop_only_options += ['--use_test_ca=false']
client_args = language.client_cmd(interop_only_options)
container_name = dockerjob.random_name('lb_interop_client_%s' %
language.safename)
docker_cmdline = docker_run_cmdline(
client_args,
environ=language.global_env(),
image=docker_image,
cwd=language.client_cwd,
docker_args=[
'--dns=%s' % dns_server_ip,
'--net=host',
'--name=%s' % container_name,
'-v',
'{grpc_grpc_root_dir}:/external_mount:ro'.format(
grpc_grpc_root_dir=ROOT),
])
jobset.message('IDLE',
'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline),
do_newline=True)
test_job = jobset.JobSpec(cmdline=docker_cmdline,
shortname=('lb_interop_client:%s' % language),
timeout_seconds=_TEST_TIMEOUT,
kill_handler=_job_kill_handler)
test_job.container_name = container_name
return test_job
def grpc_server_in_docker_jobspec(server_cmdline, shortname):
container_name = dockerjob.random_name(shortname)
environ = {
'GRPC_GO_LOG_VERBOSITY_LEVEL': '3',
'GRPC_GO_LOG_SEVERITY_LEVEL': 'INFO ',
}
docker_cmdline = docker_run_cmdline(
server_cmdline,
cwd='/go',
image=docker_images.get(_FAKE_SERVERS_SAFENAME),
environ=environ,
docker_args=['--name=%s' % container_name])
jobset.message(
'IDLE',
'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline),
do_newline=True)
server_job = jobset.JobSpec(
cmdline=docker_cmdline, shortname=shortname, timeout_seconds=30 * 60)
server_job.container_name = container_name
return server_job
def grpc_server_in_docker_jobspec(server_cmdline, shortname):
container_name = dockerjob.random_name(shortname)
environ = {
'GRPC_GO_LOG_VERBOSITY_LEVEL': '3',
'GRPC_GO_LOG_SEVERITY_LEVEL': 'INFO ',
}
docker_cmdline = docker_run_cmdline(
server_cmdline,
cwd='/go',
image=docker_images.get(_FAKE_SERVERS_SAFENAME),
environ=environ,
docker_args=['--name=%s' % container_name])
jobset.message('IDLE',
'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline),
do_newline=True)
server_job = jobset.JobSpec(cmdline=docker_cmdline,
shortname=shortname,
timeout_seconds=30 * 60)
server_job.container_name = container_name
return server_job
def prepare_remote_hosts(hosts, prepare_local=False):
"""Prepares remote hosts (and maybe prepare localhost as well)."""
prepare_timeout = 5*60
prepare_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
prepare_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_prepare.sh'],
shortname='remote_host_prepare.%s' % host,
environ = {'USER_AT_HOST': user_at_host},
timeout_seconds=prepare_timeout))
if prepare_local:
# Prepare localhost as well
prepare_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/kill_workers.sh'],
shortname='local_prepare',
timeout_seconds=prepare_timeout))
jobset.message('START', 'Preparing hosts.', do_newline=True)
num_failures, _ = jobset.run(
prepare_jobs, newline_on_success=True, maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS',
'Prepare step completed successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Failed to prepare remote hosts.',
do_newline=True)
sys.exit(1)
def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), build_local=False):
"""Builds performance worker on remote hosts (and maybe also locally)."""
build_timeout = 15*60
build_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_build.sh'] + languages,
shortname='remote_host_build.%s' % host,
environ = {'USER_AT_HOST': user_at_host, 'CONFIG': 'opt'},
timeout_seconds=build_timeout))
if build_local:
# Build locally as well
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/build_performance.sh'] + languages,
shortname='local_build',
environ = {'CONFIG': 'opt'},
timeout_seconds=build_timeout))
jobset.message('START', 'Building.', do_newline=True)
num_failures, _ = jobset.run(
build_jobs, newline_on_success=True, maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS',
'Built successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Build failed.',
do_newline=True)
sys.exit(1)
def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), build_local=False):
"""Builds performance worker on remote hosts (and maybe also locally)."""
build_timeout = 15*60
build_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_build.sh'] + languages,
shortname='remote_host_build.%s' % host,
environ = {'USER_AT_HOST': user_at_host, 'CONFIG': 'opt'},
timeout_seconds=build_timeout))
if build_local:
# Build locally as well
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/build_performance.sh'] + languages,
shortname='local_build',
environ = {'CONFIG': 'opt'},
timeout_seconds=build_timeout))
jobset.message('START', 'Building.', do_newline=True)
num_failures, _ = jobset.run(
build_jobs, newline_on_success=True, maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS',
'Built successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Build failed.',
do_newline=True)
sys.exit(1)
def archive_repo(languages):
"""Archives local version of repo including submodules."""
# Directory contains symlinks that can't be correctly untarred on Windows
# so we just skip them as a workaround.
# See https://github.com/grpc/grpc/issues/16334
bad_symlinks_dir = '../grpc/third_party/libcxx/test/std/experimental/filesystem/Inputs/static_test_env'
cmdline = [
'tar', '--exclude', bad_symlinks_dir, '-cf', '../grpc.tar', '../grpc/'
]
if 'java' in languages:
cmdline.append('../grpc-java')
if 'go' in languages:
cmdline.append('../grpc-go')
if 'node' in languages or 'node_purejs' in languages:
cmdline.append('../grpc-node')
archive_job = jobset.JobSpec(
cmdline=cmdline, shortname='archive_repo', timeout_seconds=3 * 60)
jobset.message('START', 'Archiving local repository.', do_newline=True)
num_failures, _ = jobset.run(
[archive_job], newline_on_success=True, maxjobs=1)
if num_failures == 0:
jobset.message(
'SUCCESS',
'Archive with local repository created successfully.',
do_newline=True)
else:
jobset.message(
'FAILED', 'Failed to archive local repository.', do_newline=True)
sys.exit(1)
def archive_repo(languages):
"""Archives local version of repo including submodules."""
cmdline = ['tar', '-cf', '../grpc.tar', '../grpc/']
if 'java' in languages:
cmdline.append('../grpc-java')
if 'go' in languages:
cmdline.append('../grpc-go')
archive_job = jobset.JobSpec(cmdline=cmdline,
shortname='archive_repo',
timeout_seconds=3 * 60)
jobset.message('START', 'Archiving local repository.', do_newline=True)
num_failures, _ = jobset.run([archive_job],
newline_on_success=True,
maxjobs=1,
clear_alarms=False)
if num_failures == 0:
jobset.message('SUCCESS',
'Archive with local repository created successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Failed to archive local repository.',
do_newline=True)
sys.exit(1)
def archive_repo(languages):
"""Archives local version of repo including submodules."""
# Directory contains symlinks that can't be correctly untarred on Windows
# so we just skip them as a workaround.
# See https://github.com/grpc/grpc/issues/16334
bad_symlinks_dir = '../grpc/third_party/libcxx/test/std/experimental/filesystem/Inputs/static_test_env'
cmdline = [
'tar', '--exclude', bad_symlinks_dir, '-cf', '../grpc.tar', '../grpc/'
]
if 'java' in languages:
cmdline.append('../grpc-java')
if 'go' in languages:
cmdline.append('../grpc-go')
archive_job = jobset.JobSpec(cmdline=cmdline,
shortname='archive_repo',
timeout_seconds=3 * 60)
jobset.message('START', 'Archiving local repository.', do_newline=True)
num_failures, _ = jobset.run([archive_job],
newline_on_success=True,
maxjobs=1)
if num_failures == 0:
jobset.message('SUCCESS',
'Archive with local repository created successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Failed to archive local repository.',
do_newline=True)
sys.exit(1)
def prepare_remote_hosts(hosts, prepare_local=False):
"""Prepares remote hosts (and maybe prepare localhost as well)."""
prepare_timeout = 5 * 60
prepare_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
prepare_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_prepare.sh'],
shortname='remote_host_prepare.%s' % host,
environ={'USER_AT_HOST': user_at_host},
timeout_seconds=prepare_timeout))
if prepare_local:
# Prepare localhost as well
prepare_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/kill_workers.sh'],
shortname='local_prepare',
timeout_seconds=prepare_timeout))
jobset.message('START', 'Preparing hosts.', do_newline=True)
num_failures, _ = jobset.run(prepare_jobs,
newline_on_success=True,
maxjobs=10,
clear_alarms=False)
if num_failures == 0:
jobset.message('SUCCESS',
'Prepare step completed successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Failed to prepare remote hosts.',
do_newline=True)
sys.exit(1)
def build_on_remote_hosts(hosts,
languages=list(scenario_config.LANGUAGES.keys()),
build_local=False):
"""Builds performance worker on remote hosts (and maybe also locally)."""
build_timeout = 45 * 60
# Kokoro VMs (which are local only) do not have caching, so they need more time to build
local_build_timeout = 60 * 60
build_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_build.sh'] +
languages,
shortname='remote_host_build.%s' % host,
environ={
'USER_AT_HOST': user_at_host,
'CONFIG': 'opt'
},
timeout_seconds=build_timeout))
if build_local:
# start port server locally
build_jobs.append(
jobset.JobSpec(
cmdline=['python', 'tools/run_tests/start_port_server.py'],
shortname='local_start_port_server',
timeout_seconds=2 * 60))
# Build locally as well
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/build_performance.sh'] +
languages,
shortname='local_build',
environ={'CONFIG': 'opt'},
timeout_seconds=local_build_timeout))
jobset.message('START', 'Building.', do_newline=True)
num_failures, _ = jobset.run(build_jobs,
newline_on_success=True,
maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS', 'Built successfully.', do_newline=True)
else:
jobset.message('FAILED', 'Build failed.', do_newline=True)
sys.exit(1)
def archive_repo(languages):
"""Archives local version of repo including submodules."""
cmdline = ['tar', '-cf', '../grpc.tar', '../grpc/']
if 'java' in languages:
cmdline.append('../grpc-java')
if 'go' in languages:
cmdline.append('../grpc-go')
archive_job = jobset.JobSpec(
cmdline=cmdline, shortname='archive_repo', timeout_seconds=3 * 60)
jobset.message('START', 'Archiving local repository.', do_newline=True)
num_failures, _ = jobset.run(
[archive_job], newline_on_success=True, maxjobs=1)
if num_failures == 0:
jobset.message(
'SUCCESS',
'Archive with local repository created successfully.',
do_newline=True)
else:
jobset.message(
'FAILED', 'Failed to archive local repository.', do_newline=True)
sys.exit(1)
extra_args.append('%s' % args.runs_per_test)
extra_args.append('--quiet_success')
if args.max_time > 0:
extra_args.extend(('--max_time', '%d' % args.max_time))
all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \
_create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs)
jobs = []
for job in all_jobs:
if not args.filter or all(filter in job.labels for filter in args.filter):
if not any(exclude_label in job.labels for exclude_label in args.exclude):
jobs.append(job)
if not jobs:
jobset.message('FAILED', 'No test suites match given criteria.',
do_newline=True)
sys.exit(1)
print('IMPORTANT: The changes you are testing need to be locally committed')
print('because only the committed changes in the current branch will be')
print('copied to the docker environment or into subworkspaces.')
skipped_jobs = []
if args.filter_pr_tests:
print('Looking for irrelevant tests to skip...')
relevant_jobs = filter_tests(jobs, args.base_branch)
if len(relevant_jobs) == len(jobs):
print('No tests will be skipped.')
else:
print('These tests will be skipped:')
targets += _BUILD_MAP[label]
# Among targets selected by -b, filter out those that don't match the filter
targets = [t for t in targets if all(f in t.labels for f in args.filter)]
targets = sorted(set(targets), key=lambda target: target.name)
# Execute pre-build phase
prebuild_jobs = []
for target in targets:
prebuild_jobs += target.pre_build_jobspecs()
if prebuild_jobs:
num_failures, _ = jobset.run(prebuild_jobs,
newline_on_success=True,
maxjobs=args.jobs)
if num_failures != 0:
jobset.message('FAILED', 'Pre-build phase failed.', do_newline=True)
sys.exit(1)
build_jobs = []
for target in targets:
build_jobs.append(target.build_jobspec())
if not build_jobs:
print('Nothing to build.')
sys.exit(1)
jobset.message('START', 'Building targets.', do_newline=True)
num_failures, resultset = jobset.run(build_jobs,
newline_on_success=True,
maxjobs=args.jobs)
report_utils.render_junit_xml_report(resultset,
args.xml_report,
build_jobs.append(job)
docker_images[str(l.safename)] = job.tag
# First check if a pre-built image was supplied.
if args.servers_image_tag:
docker_images[_FAKE_SERVERS_SAFENAME] = args.servers_image_tag
else:
# Build the test servers in docker and save the fully
# built image.
job = build_interop_image_jobspec(_FAKE_SERVERS_SAFENAME,
basename_prefix='lb_interop')
build_jobs.append(job)
docker_images[_FAKE_SERVERS_SAFENAME] = job.tag
if build_jobs:
jobset.message('START', 'Building interop docker images.', do_newline=True)
print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
num_failures, _ = jobset.run(build_jobs,
newline_on_success=True,
maxjobs=args.jobs)
if num_failures == 0:
jobset.message('SUCCESS',
'All docker images built successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Failed to build interop docker images.',
do_newline=True)
sys.exit(1)
build_jobs.append(job)
docker_images[str(l.safename)] = job.tag
# First check if a pre-built image was supplied.
if args.servers_image_tag:
docker_images[_FAKE_SERVERS_SAFENAME] = args.servers_image_tag
else:
# Build the test servers in docker and save the fully
# built image.
job = build_interop_image_jobspec(
_FAKE_SERVERS_SAFENAME, basename_prefix='lb_interop')
build_jobs.append(job)
docker_images[_FAKE_SERVERS_SAFENAME] = job.tag
if build_jobs:
jobset.message('START', 'Building interop docker images.', do_newline=True)
print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
num_failures, _ = jobset.run(
build_jobs, newline_on_success=True, maxjobs=args.jobs)
if num_failures == 0:
jobset.message(
'SUCCESS', 'All docker images built successfully.', do_newline=True)
else:
jobset.message(
'FAILED', 'Failed to build interop docker images.', do_newline=True)
sys.exit(1)
def wait_until_dns_server_is_up(dns_server_ip):
"""Probes the DNS server until it's running and safe for tests."""
for i in range(0, 30):
def run_one_scenario(scenario_config):
jobset.message('START', 'Run scenario: %s' % scenario_config['name'])
server_jobs = {}
server_addresses = {}
suppress_server_logs = True
try:
backend_addrs = []
fallback_ips = []
grpclb_ips = []
shortname_prefix = scenario_config['name']
# Start backends
for i in xrange(len(scenario_config['backend_configs'])):
backend_config = scenario_config['backend_configs'][i]
backend_shortname = shortname(shortname_prefix, 'backend_server', i)
backend_spec = backend_server_jobspec(
backend_config['transport_sec'], backend_shortname)
backend_job = dockerjob.DockerJob(backend_spec)
server_jobs[backend_shortname] = backend_job
backend_addrs.append('%s:%d' % (backend_job.ip_address(),
_BACKEND_SERVER_PORT))
# Start fallbacks
for i in xrange(len(scenario_config['fallback_configs'])):
fallback_config = scenario_config['fallback_configs'][i]
fallback_shortname = shortname(shortname_prefix, 'fallback_server',
i)
fallback_spec = fallback_server_jobspec(
fallback_config['transport_sec'], fallback_shortname)
fallback_job = dockerjob.DockerJob(fallback_spec)
server_jobs[fallback_shortname] = fallback_job
fallback_ips.append(fallback_job.ip_address())
# Start balancers
for i in xrange(len(scenario_config['balancer_configs'])):
balancer_config = scenario_config['balancer_configs'][i]
grpclb_shortname = shortname(shortname_prefix, 'grpclb_server', i)
grpclb_spec = grpclb_jobspec(balancer_config['transport_sec'],
balancer_config['short_stream'],
backend_addrs, grpclb_shortname)
grpclb_job = dockerjob.DockerJob(grpclb_spec)
server_jobs[grpclb_shortname] = grpclb_job
grpclb_ips.append(grpclb_job.ip_address())
# Start DNS server
dns_server_shortname = shortname(shortname_prefix, 'dns_server', 0)
dns_server_spec = dns_server_in_docker_jobspec(
grpclb_ips, fallback_ips, dns_server_shortname,
scenario_config['cause_no_error_no_data_for_balancer_a_record'])
dns_server_job = dockerjob.DockerJob(dns_server_spec)
server_jobs[dns_server_shortname] = dns_server_job
# Get the IP address of the docker container running the DNS server.
# The DNS server is running on port 53 of that IP address. Note we will
# point the DNS resolvers of grpc clients under test to our controlled
# DNS server by effectively modifying the /etc/resolve.conf "nameserver"
# lists of their docker containers.
dns_server_ip = dns_server_job.ip_address()
wait_until_dns_server_is_up(dns_server_ip)
# Run clients
jobs = []
for lang_name in languages:
# Skip languages that are known to not currently
# work for this test.
if not args.no_skips and lang_name in scenario_config.get(
'skip_langs', []):
jobset.message('IDLE',
'Skipping scenario: %s for language: %s\n' %
(scenario_config['name'], lang_name))
continue
lang = _LANGUAGES[lang_name]
test_job = lb_client_interop_jobspec(
lang,
dns_server_ip,
docker_image=docker_images.get(lang.safename),
transport_security=scenario_config['transport_sec'])
jobs.append(test_job)
jobset.message('IDLE', 'Jobs to run: \n%s\n' % '\n'.join(
str(job) for job in jobs))
num_failures, resultset = jobset.run(
jobs, newline_on_success=True, maxjobs=args.jobs)
report_utils.render_junit_xml_report(resultset, 'sponge_log.xml')
if num_failures:
suppress_server_logs = False
jobset.message(
'FAILED',
'Scenario: %s. Some tests failed' % scenario_config['name'],
do_newline=True)
else:
jobset.message(
'SUCCESS',
'Scenario: %s. All tests passed' % scenario_config['name'],
do_newline=True)
return num_failures
finally:
# Check if servers are still running.
for server, job in server_jobs.items():
if not job.is_running():
print('Server "%s" has exited prematurely.' % server)
suppress_failure = suppress_server_logs and not args.verbose
dockerjob.finish_jobs(
[j for j in six.itervalues(server_jobs)],
suppress_failure=suppress_failure)
def run_one_scenario(scenario_config):
jobset.message('START', 'Run scenario: %s' % scenario_config['name'])
server_jobs = {}
server_addresses = {}
suppress_server_logs = True
try:
backend_addrs = []
fallback_ips = []
grpclb_ips = []
shortname_prefix = scenario_config['name']
# Start backends
for i in xrange(len(scenario_config['backend_configs'])):
backend_config = scenario_config['backend_configs'][i]
backend_shortname = shortname(shortname_prefix, 'backend_server',
i)
backend_spec = backend_server_jobspec(
backend_config['transport_sec'], backend_shortname)
backend_job = dockerjob.DockerJob(backend_spec)
server_jobs[backend_shortname] = backend_job
backend_addrs.append(
'%s:%d' % (backend_job.ip_address(), _BACKEND_SERVER_PORT))
# Start fallbacks
for i in xrange(len(scenario_config['fallback_configs'])):
fallback_config = scenario_config['fallback_configs'][i]
fallback_shortname = shortname(shortname_prefix, 'fallback_server',
i)
fallback_spec = fallback_server_jobspec(
fallback_config['transport_sec'], fallback_shortname)
fallback_job = dockerjob.DockerJob(fallback_spec)
server_jobs[fallback_shortname] = fallback_job
fallback_ips.append(fallback_job.ip_address())
# Start balancers
for i in xrange(len(scenario_config['balancer_configs'])):
balancer_config = scenario_config['balancer_configs'][i]
grpclb_shortname = shortname(shortname_prefix, 'grpclb_server', i)
grpclb_spec = grpclb_jobspec(balancer_config['transport_sec'],
balancer_config['short_stream'],
backend_addrs, grpclb_shortname)
grpclb_job = dockerjob.DockerJob(grpclb_spec)
server_jobs[grpclb_shortname] = grpclb_job
grpclb_ips.append(grpclb_job.ip_address())
# Start DNS server
dns_server_shortname = shortname(shortname_prefix, 'dns_server', 0)
dns_server_spec = dns_server_in_docker_jobspec(
grpclb_ips, fallback_ips, dns_server_shortname,
scenario_config['cause_no_error_no_data_for_balancer_a_record'])
dns_server_job = dockerjob.DockerJob(dns_server_spec)
server_jobs[dns_server_shortname] = dns_server_job
# Get the IP address of the docker container running the DNS server.
# The DNS server is running on port 53 of that IP address. Note we will
# point the DNS resolvers of grpc clients under test to our controlled
# DNS server by effectively modifying the /etc/resolve.conf "nameserver"
# lists of their docker containers.
dns_server_ip = dns_server_job.ip_address()
wait_until_dns_server_is_up(dns_server_ip)
# Run clients
jobs = []
for lang_name in languages:
# Skip languages that are known to not currently
# work for this test.
if not args.no_skips and lang_name in scenario_config.get(
'skip_langs', []):
jobset.message(
'IDLE', 'Skipping scenario: %s for language: %s\n' %
(scenario_config['name'], lang_name))
continue
lang = _LANGUAGES[lang_name]
test_job = lb_client_interop_jobspec(
lang,
dns_server_ip,
docker_image=docker_images.get(lang.safename),
transport_security=scenario_config['transport_sec'])
jobs.append(test_job)
jobset.message(
'IDLE',
'Jobs to run: \n%s\n' % '\n'.join(str(job) for job in jobs))
num_failures, resultset = jobset.run(jobs,
newline_on_success=True,
maxjobs=args.jobs)
report_utils.render_junit_xml_report(resultset, 'sponge_log.xml')
if num_failures:
suppress_server_logs = False
jobset.message('FAILED',
'Scenario: %s. Some tests failed' %
scenario_config['name'],
do_newline=True)
else:
jobset.message('SUCCESS',
'Scenario: %s. All tests passed' %
scenario_config['name'],
do_newline=True)
return num_failures
finally:
# Check if servers are still running.
for server, job in server_jobs.items():
if not job.is_running():
print('Server "%s" has exited prematurely.' % server)
suppress_failure = suppress_server_logs and not args.verbose
dockerjob.finish_jobs([j for j in six.itervalues(server_jobs)],
suppress_failure=suppress_failure)
# languages for which to build docker images
languages_to_build = set(_LANGUAGES[k]
for k in set([str(l) for l in languages] +
[s for s in servers]))
if args.http2_interop:
languages_to_build.add(http2Interop)
build_jobs = []
for l in languages_to_build:
job = build_interop_image_jobspec(l)
docker_images[str(l)] = job.tag
build_jobs.append(job)
if build_jobs:
jobset.message('START',
'Building interop docker images.',
do_newline=True)
num_failures, _ = jobset.run(build_jobs,
newline_on_success=True,
maxjobs=args.jobs)
if num_failures == 0:
jobset.message('SUCCESS',
'All docker images built successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Failed to build interop docker images.',
do_newline=True)
for image in docker_images.itervalues():
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
extra_args.append('--quiet_success')
all_jobs = _create_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs) + \
_create_portability_test_jobs(extra_args=extra_args, inner_jobs=args.inner_jobs)
jobs = []
for job in all_jobs:
if not args.filter or all(filter in job.labels
for filter in args.filter):
if not any(exclude_label in job.labels
for exclude_label in args.exclude):
jobs.append(job)
if not jobs:
jobset.message('FAILED',
'No test suites match given criteria.',
do_newline=True)
sys.exit(1)
print(
'IMPORTANT: The changes you are testing need to be locally committed')
print('because only the committed changes in the current branch will be')
print('copied to the docker environment or into subworkspaces.')
skipped_jobs = []
if args.filter_pr_tests:
print('Looking for irrelevant tests to skip...')
relevant_jobs = filter_tests(jobs, args.base_branch)
if len(relevant_jobs) == len(jobs):
print('No tests will be skipped.')
for label in args.build:
targets += _BUILD_MAP[label]
# Among targets selected by -b, filter out those that don't match the filter
targets = [t for t in targets if all(f in t.labels for f in args.filter)]
targets = sorted(set(targets))
# Execute pre-build phase
prebuild_jobs = []
for target in targets:
prebuild_jobs += target.pre_build_jobspecs()
if prebuild_jobs:
num_failures, _ = jobset.run(
prebuild_jobs, newline_on_success=True, maxjobs=args.jobs)
if num_failures != 0:
jobset.message('FAILED', 'Pre-build phase failed.', do_newline=True)
sys.exit(1)
build_jobs = []
for target in targets:
build_jobs.append(target.build_jobspec())
if not build_jobs:
print('Nothing to build.')
sys.exit(1)
jobset.message('START', 'Building targets.', do_newline=True)
num_failures, _ = jobset.run(
build_jobs, newline_on_success=True, maxjobs=args.jobs)
if num_failures == 0:
jobset.message('SUCCESS', 'All targets built successfully.',
do_newline=True)
languages_to_build.add(http2Interop)
if args.http2_server_interop:
languages_to_build.add(http2InteropServer)
build_jobs = []
for l in languages_to_build:
if str(l) == 'objc':
# we don't need to build a docker image for objc
continue
job = build_interop_image_jobspec(l)
docker_images[str(l)] = job.tag
build_jobs.append(job)
if build_jobs:
jobset.message('START', 'Building interop docker images.', do_newline=True)
if args.verbose:
print('Jobs to run: \n%s\n' % '\n'.join(str(j) for j in build_jobs))
num_failures, _ = jobset.run(
build_jobs, newline_on_success=True, maxjobs=args.jobs)
if num_failures == 0:
jobset.message('SUCCESS', 'All docker images built successfully.',
do_newline=True)
else:
jobset.message('FAILED', 'Failed to build interop docker images.',
do_newline=True)
for image in six.itervalues(docker_images):
dockerjob.remove_image(image, skip_nonexistent=True)
sys.exit(1)
