def create_docker_jobspec(name,
dockerfile_dir,
shell_command,
environ={},
flake_retries=0,
timeout_retries=0):
"""Creates jobspec for a task running under docker."""
environ = environ.copy()
environ['RUN_COMMAND'] = shell_command
docker_args = []
for k, v in environ.items():
docker_args += ['-e', '%s=%s' % (k, v)]
docker_env = {
'DOCKERFILE_DIR': dockerfile_dir,
'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
'OUTPUT_DIR': 'artifacts'
}
jobspec = jobset.JobSpec(
cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] +
docker_args,
environ=docker_env,
shortname='build_package.%s' % (name),
timeout_seconds=30 * 60,
flake_retries=flake_retries,
timeout_retries=timeout_retries)
return jobspec
def _workspace_jobspec(name,
runtests_args=[],
workspace_name=None,
runtests_envs={},
inner_jobs=_DEFAULT_INNER_JOBS,
timeout_seconds=None):
"""Run a single instance of run_tests.py in a separate workspace"""
if not workspace_name:
workspace_name = 'workspace_%s' % name
if not timeout_seconds:
timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
shortname = 'run_tests_%s' % name
env = {'WORKSPACE_NAME': workspace_name}
env.update(runtests_envs)
# if report base dir is set, we don't need to ".." to come out of the workspace dir
report_dir_prefix = '' if os.getenv('GRPC_TEST_REPORT_BASE_DIR',
None) else '../'
test_job = jobset.JobSpec(cmdline=[
'bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh',
'-t', '-j',
str(inner_jobs), '-x',
'%srun_tests/%s' %
(report_dir_prefix, _report_filename(name)), '--report_suite_name',
'%s' % _safe_report_name(name)
] + runtests_args,
environ=env,
shortname=shortname,
timeout_seconds=timeout_seconds,
logfilename=_matrix_job_logfilename(shortname))
return test_job
def create_scenario_jobspec(scenario_json,
workers,
remote_host=None,
bq_result_table=None,
server_cpu_load=0):
"""Runs one scenario using QPS driver."""
# setting QPS_WORKERS env variable here makes sure it works with SSH too.
cmd = 'QPS_WORKERS="%s" ' % ','.join(workers)
if bq_result_table:
cmd += 'BQ_RESULT_TABLE="%s" ' % bq_result_table
cmd += 'tools/run_tests/performance/run_qps_driver.sh '
cmd += '--scenarios_json=%s ' % pipes.quote(
json.dumps({
'scenarios': [scenario_json]
}))
cmd += '--scenario_result_file=scenario_result.json '
if server_cpu_load != 0:
cmd += '--search_param=offered_load --initial_search_value=1000 --targeted_cpu_load=%d --stride=500 --error_tolerance=0.01' % server_cpu_load
if remote_host:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host)
cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (
user_at_host, pipes.quote(cmd))
return jobset.JobSpec(
cmdline=[cmd],
shortname='%s' % scenario_json['name'],
timeout_seconds=_SCENARIO_TIMEOUT,
shell=True,
verbose_success=True)
def archive_repo(languages):
"""Archives local version of repo including submodules."""
# Directory contains symlinks that can't be correctly untarred on Windows
# so we just skip them as a workaround.
# See https://github.com/grpc/grpc/issues/16334
bad_symlinks_dir = '../grpc/third_party/libcxx/test/std/experimental/filesystem/Inputs/static_test_env'
cmdline = [
'tar', '--exclude', bad_symlinks_dir, '-cf', '../grpc.tar', '../grpc/'
]
if 'java' in languages:
cmdline.append('../grpc-java')
if 'go' in languages:
cmdline.append('../grpc-go')
archive_job = jobset.JobSpec(cmdline=cmdline,
shortname='archive_repo',
timeout_seconds=3 * 60)
jobset.message('START', 'Archiving local repository.', do_newline=True)
num_failures, _ = jobset.run([archive_job],
newline_on_success=True,
maxjobs=1)
if num_failures == 0:
jobset.message('SUCCESS',
'Archive with local repository created successfully.',
do_newline=True)
else:
jobset.message('FAILED',
'Failed to archive local repository.',
do_newline=True)
sys.exit(1)
def create_netperf_jobspec(server_host='localhost',
client_host=None,
bq_result_table=None):
"""Runs netperf benchmark."""
cmd = 'NETPERF_SERVER_HOST="%s" ' % server_host
if bq_result_table:
cmd += 'BQ_RESULT_TABLE="%s" ' % bq_result_table
if client_host:
# If netperf is running remotely, the env variables populated by Jenkins
# won't be available on the client, but we need them for uploading results
# to BigQuery.
jenkins_job_name = os.getenv('JOB_NAME')
if jenkins_job_name:
cmd += 'JOB_NAME="%s" ' % jenkins_job_name
jenkins_build_number = os.getenv('BUILD_NUMBER')
if jenkins_build_number:
cmd += 'BUILD_NUMBER="%s" ' % jenkins_build_number
cmd += 'tools/run_tests/performance/run_netperf.sh'
if client_host:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, client_host)
cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (
user_at_host, pipes.quote(cmd))
return jobset.JobSpec(
cmdline=[cmd],
shortname='netperf',
timeout_seconds=_NETPERF_TIMEOUT,
shell=True,
verbose_success=True)
def create_docker_jobspec(name,
dockerfile_dir,
shell_command,
environ={},
flake_retries=0,
timeout_retries=0,
timeout_seconds=30 * 60,
extra_docker_args=None,
verbose_success=False):
"""Creates jobspec for a task running under docker."""
environ = environ.copy()
environ['RUN_COMMAND'] = shell_command
environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name
docker_args = []
for k, v in environ.items():
docker_args += ['-e', '%s=%s' % (k, v)]
docker_env = {
'DOCKERFILE_DIR': dockerfile_dir,
'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
'OUTPUT_DIR': 'artifacts'
}
if extra_docker_args is not None:
docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args
jobspec = jobset.JobSpec(
cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] +
docker_args,
environ=docker_env,
shortname='build_artifact.%s' % (name),
timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries,
verbose_success=verbose_success)
return jobspec
def archive_repo(languages):
"""Archives local version of repo including submodules."""
cmdline = ['tar', '-cf', '../grpc.tar', '../grpc/']
if 'java' in languages:
cmdline.append('../grpc-java')
if 'go' in languages:
cmdline.append('../grpc-go')
if 'node' in languages or 'node_purejs' in languages:
cmdline.append('../grpc-node')
archive_job = jobset.JobSpec(
cmdline=cmdline, shortname='archive_repo', timeout_seconds=3 * 60)
jobset.message('START', 'Archiving local repository.', do_newline=True)
num_failures, _ = jobset.run(
[archive_job], newline_on_success=True, maxjobs=1)
if num_failures == 0:
jobset.message(
'SUCCESS',
'Archive with local repository created successfully.',
do_newline=True)
else:
jobset.message(
'FAILED', 'Failed to archive local repository.', do_newline=True)
sys.exit(1)
def create_jobspec(name,
cmdline,
environ={},
shell=False,
flake_retries=0,
timeout_retries=0,
timeout_seconds=30 * 60,
use_workspace=False,
cpu_cost=1.0,
verbose_success=False):
"""Creates jobspec."""
environ = environ.copy()
if use_workspace:
environ['WORKSPACE_NAME'] = 'workspace_%s' % name
environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name)
cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh'
] + cmdline
else:
environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name)
jobspec = jobset.JobSpec(cmdline=cmdline,
environ=environ,
shortname='build_artifact.%s' % (name),
timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries,
shell=shell,
cpu_cost=cpu_cost,
verbose_success=verbose_success)
return jobspec
def create_docker_jobspec(name,
dockerfile_dir,
shell_command,
environ={},
flake_retries=0,
timeout_retries=0,
copy_rel_path=None,
timeout_seconds=30 * 60):
"""Creates jobspec for a task running under docker."""
environ = environ.copy()
environ['RUN_COMMAND'] = shell_command
# the entire repo will be cloned if copy_rel_path is not set.
if copy_rel_path:
environ['RELATIVE_COPY_PATH'] = copy_rel_path
docker_args = []
for k, v in environ.items():
docker_args += ['-e', '%s=%s' % (k, v)]
docker_env = {
'DOCKERFILE_DIR': dockerfile_dir,
'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh'
}
jobspec = jobset.JobSpec(
cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] +
docker_args,
environ=docker_env,
shortname='distribtest.%s' % (name),
timeout_seconds=timeout_seconds,
flake_retries=flake_retries,
timeout_retries=timeout_retries)
return jobspec
def _workspace_jobspec(name,
runtests_args=[],
workspace_name=None,
runtests_envs={},
inner_jobs=_DEFAULT_INNER_JOBS,
timeout_seconds=None):
"""Run a single instance of run_tests.py in a separate workspace"""
if not workspace_name:
workspace_name = 'workspace_%s' % name
if not timeout_seconds:
timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
shortname = 'run_tests_%s' % name
env = {'WORKSPACE_NAME': workspace_name}
env.update(runtests_envs)
test_job = jobset.JobSpec(cmdline=[
'bash', 'tools/run_tests/helper_scripts/run_tests_in_workspace.sh',
'-t', '-j',
str(inner_jobs), '-x',
'../run_tests/%s' % _report_filename(name), '--report_suite_name',
'%s' % _safe_report_name(name)
] + runtests_args,
environ=env,
shortname=shortname,
timeout_seconds=timeout_seconds,
logfilename=_matrix_job_logfilename(shortname))
return test_job
def create_qpsworker_job(language,
shortname=None,
port=10000,
remote_host=None,
perf_cmd=None):
cmdline = (language.worker_cmdline() + ['--driver_port=%s' % port])
if remote_host:
host_and_port = '%s:%s' % (remote_host, port)
else:
host_and_port = 'localhost:%s' % port
perf_file_base_name = None
if perf_cmd:
perf_file_base_name = '%s-%s' % (host_and_port, shortname)
# specify -o output file so perf.data gets collected when worker stopped
cmdline = perf_cmd + ['-o', '%s-perf.data' % perf_file_base_name
] + cmdline
if remote_host:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host)
ssh_cmd = ['ssh']
ssh_cmd.extend([
str(user_at_host),
'cd ~/performance_workspace/grpc/ && %s' % ' '.join(cmdline)
])
cmdline = ssh_cmd
jobspec = jobset.JobSpec(
cmdline=cmdline,
shortname=shortname,
timeout_seconds=5 * 60, # workers get restarted after each scenario
verbose_success=True)
return QpsWorkerJob(jobspec, language, host_and_port, perf_file_base_name)
def dns_server_in_docker_jobspec(grpclb_ips, fallback_ips, shortname,
cause_no_error_no_data_for_balancer_a_record):
container_name = dockerjob.random_name(shortname)
run_dns_server_cmdline = [
'python',
'test/cpp/naming/utils/run_dns_server_for_lb_interop_tests.py',
'--grpclb_ips=%s' % ','.join(grpclb_ips),
'--fallback_ips=%s' % ','.join(fallback_ips),
]
if cause_no_error_no_data_for_balancer_a_record:
run_dns_server_cmdline.append(
'--cause_no_error_no_data_for_balancer_a_record')
docker_cmdline = docker_run_cmdline(
run_dns_server_cmdline,
cwd='/var/local/git/grpc',
image=docker_images.get(_FAKE_SERVERS_SAFENAME),
docker_args=['--name=%s' % container_name])
jobset.message('IDLE',
'docker_cmdline:\b|%s|' % ' '.join(docker_cmdline),
do_newline=True)
server_job = jobset.JobSpec(cmdline=docker_cmdline,
shortname=shortname,
timeout_seconds=30 * 60)
server_job.container_name = container_name
return server_job
def server_jobspec(language,
docker_image,
insecure=False,
manual_cmd_log=None):
"""Create jobspec for running a server"""
container_name = dockerjob.random_name('interop_server_%s' %
language.safename)
cmdline = bash_cmdline(
language.server_cmd([
'--port=%s' % _DEFAULT_SERVER_PORT,
'--use_tls=%s' % ('false' if insecure else 'true')
]))
environ = language.global_env()
docker_args = ['--name=%s' % container_name]
if language.safename == 'http2':
# we are running the http2 interop server. Open next N ports beginning
# with the server port. These ports are used for http2 interop test
# (one test case per port).
docker_args += list(
itertools.chain.from_iterable(
('-p', str(_DEFAULT_SERVER_PORT + i))
for i in range(len(_HTTP2_SERVER_TEST_CASES))))
# Enable docker's healthcheck mechanism.
# This runs a Python script inside the container every second. The script
# pings the http2 server to verify it is ready. The 'health-retries' flag
# specifies the number of consecutive failures before docker will report
# the container's status as 'unhealthy'. Prior to the first 'health_retries'
# failures or the first success, the status will be 'starting'. 'docker ps'
# or 'docker inspect' can be used to see the health of the container on the
# command line.
docker_args += [
'--health-cmd=python test/http2_test/http2_server_health_check.py '
'--server_host=%s --server_port=%d' %
('localhost', _DEFAULT_SERVER_PORT),
'--health-interval=1s',
'--health-retries=5',
'--health-timeout=10s',
]
else:
docker_args += ['-p', str(_DEFAULT_SERVER_PORT)]
docker_cmdline = docker_run_cmdline(cmdline,
image=docker_image,
cwd=language.server_cwd,
environ=environ,
docker_args=docker_args)
if manual_cmd_log is not None:
if manual_cmd_log == []:
manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' %
docker_image)
manual_cmd_log.append(manual_cmdline(docker_cmdline, docker_image))
server_job = jobset.JobSpec(cmdline=docker_cmdline,
environ=environ,
shortname='interop_server_%s' % language,
timeout_seconds=30 * 60)
server_job.container_name = container_name
return server_job
def collect_perf(bm_name, args):
"""generate flamegraphs"""
heading('Flamegraphs: %s' % bm_name)
subprocess.check_call(
['make', bm_name,
'CONFIG=mutrace', '-j', '%d' % multiprocessing.cpu_count()])
benchmarks = []
profile_analysis = []
cleanup = []
for line in subprocess.check_output(['bins/mutrace/%s' % bm_name,
'--benchmark_list_tests']).splitlines():
link(line, '%s.svg' % fnize(line))
benchmarks.append(
jobset.JobSpec(['perf', 'record', '-o', '%s-perf.data' % fnize(line),
'-g', '-F', '997',
'bins/mutrace/%s' % bm_name,
'--benchmark_filter=^%s$' % line,
'--benchmark_min_time=10']))
profile_analysis.append(
jobset.JobSpec(['tools/run_tests/performance/process_local_perf_flamegraphs.sh'],
environ = {
'PERF_BASE_NAME': fnize(line),
'OUTPUT_DIR': 'reports',
'OUTPUT_FILENAME': fnize(line),
}))
cleanup.append(jobset.JobSpec(['rm', '%s-perf.data' % fnize(line)]))
cleanup.append(jobset.JobSpec(['rm', '%s-out.perf' % fnize(line)]))
# periodically flush out the list of jobs: temporary space required for this
# processing is large
if len(benchmarks) >= 20:
# run up to half the cpu count: each benchmark can use up to two cores
# (one for the microbenchmark, one for the data flush)
jobset.run(benchmarks, maxjobs=1,
add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
benchmarks = []
profile_analysis = []
cleanup = []
# run the remaining benchmarks that weren't flushed
if len(benchmarks):
jobset.run(benchmarks, maxjobs=1,
add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
def cloud_to_cloud_jobspec(language,
test_case,
server_name,
server_host,
server_port,
docker_image=None,
insecure=False):
"""Creates jobspec for cloud-to-cloud interop test"""
interop_only_options = [
'--server_host_override=foo.test.google.fr',
'--use_tls=%s' % ('false' if insecure else 'true'),
'--use_test_ca=true',
]
common_options = [
'--test_case=%s' % test_case,
'--server_host=%s' % server_host,
]
if test_case in _HTTP2_BADSERVER_TEST_CASES:
# We are running the http2_badserver_interop test. Adjust command line accordingly.
offset = sorted(_HTTP2_BADSERVER_TEST_CASES).index(test_case)
client_options = common_options + [
'--server_port=%s' % (int(server_port) + offset)
]
cmdline = bash_cmdline(
language.client_cmd_http2interop(client_options))
cwd = language.http2_cwd
else:
client_options = interop_only_options + common_options + [
'--server_port=%s' % server_port
]
cmdline = bash_cmdline(language.client_cmd(client_options))
cwd = language.client_cwd
environ = language.global_env()
if docker_image:
container_name = dockerjob.random_name('interop_client_%s' %
language.safename)
cmdline = docker_run_cmdline(
cmdline,
image=docker_image,
environ=environ,
cwd=cwd,
docker_args=['--net=host', '--name', container_name])
cwd = None
test_job = jobset.JobSpec(cmdline=cmdline,
cwd=cwd,
environ=environ,
shortname='cloud_to_cloud:%s:%s_server:%s' %
(language, server_name, test_case),
timeout_seconds=_TEST_TIMEOUT,
flake_retries=5 if args.allow_flakes else 0,
timeout_retries=2 if args.allow_flakes else 0,
kill_handler=_job_kill_handler)
if docker_image:
test_job.container_name = container_name
return test_job
def cloud_to_prod_jobspec(language,
test_case,
server_host_nickname,
server_host,
docker_image=None,
auth=False,
manual_cmd_log=None,
service_account_key_file=None):
"""Creates jobspec for cloud-to-prod interop test"""
container_name = None
cmdargs = [
'--server_host=%s' % server_host,
'--server_host_override=%s' % server_host, '--server_port=443',
'--use_tls=true',
'--test_case=%s' % test_case
]
environ = dict(language.cloud_to_prod_env(), **language.global_env())
if auth:
auth_cmdargs, auth_env = auth_options(language, test_case,
service_account_key_file)
cmdargs += auth_cmdargs
environ.update(auth_env)
cmdline = bash_cmdline(language.client_cmd(cmdargs))
cwd = language.client_cwd
if docker_image:
container_name = dockerjob.random_name('interop_client_%s' %
language.safename)
cmdline = docker_run_cmdline(
cmdline,
image=docker_image,
cwd=cwd,
environ=environ,
docker_args=['--net=host',
'--name=%s' % container_name])
if manual_cmd_log is not None:
if manual_cmd_log == []:
manual_cmd_log.append('echo "Testing ${docker_image:=%s}"' %
docker_image)
manual_cmd_log.append(manual_cmdline(cmdline, docker_image))
cwd = None
environ = None
suite_name = 'cloud_to_prod_auth' if auth else 'cloud_to_prod'
test_job = jobset.JobSpec(
cmdline=cmdline,
cwd=cwd,
environ=environ,
shortname='%s:%s:%s:%s' %
(suite_name, language, server_host_nickname, test_case),
timeout_seconds=_TEST_TIMEOUT,
flake_retries=4 if args.allow_flakes else 0,
timeout_retries=2 if args.allow_flakes else 0,
kill_handler=_job_kill_handler)
if docker_image:
test_job.container_name = container_name
return test_job
def build_on_remote_hosts(hosts,
languages=list(scenario_config.LANGUAGES.keys()),
build_local=False):
"""Builds performance worker on remote hosts (and maybe also locally)."""
build_timeout = 45 * 60
# Kokoro VMs (which are local only) do not have caching, so they need more time to build
local_build_timeout = 60 * 60
build_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_build.sh'] +
languages,
shortname='remote_host_build.%s' % host,
environ={
'USER_AT_HOST': user_at_host,
'CONFIG': 'opt'
},
timeout_seconds=build_timeout))
if build_local:
# start port server locally
build_jobs.append(
jobset.JobSpec(
cmdline=['python', 'tools/run_tests/start_port_server.py'],
shortname='local_start_port_server',
timeout_seconds=2 * 60))
# Build locally as well
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/build_performance.sh'] +
languages,
shortname='local_build',
environ={'CONFIG': 'opt'},
timeout_seconds=local_build_timeout))
jobset.message('START', 'Building.', do_newline=True)
num_failures, _ = jobset.run(build_jobs,
newline_on_success=True,
maxjobs=10)
if num_failures == 0:
jobset.message('SUCCESS', 'Built successfully.', do_newline=True)
else:
jobset.message('FAILED', 'Build failed.', do_newline=True)
sys.exit(1)
def collect_latency(bm_name, args):
"""generate latency profiles"""
benchmarks = []
profile_analysis = []
cleanup = []
heading('Latency Profiles: %s' % bm_name)
subprocess.check_call(
['make', bm_name,
'CONFIG=basicprof', '-j', '%d' % multiprocessing.cpu_count()])
for line in subprocess.check_output(['bins/basicprof/%s' % bm_name,
'--benchmark_list_tests']).splitlines():
link(line, '%s.txt' % fnize(line))
benchmarks.append(
jobset.JobSpec(['bins/basicprof/%s' % bm_name, '--benchmark_filter=^%s$' % line],
environ={'LATENCY_TRACE': '%s.trace' % fnize(line)}))
profile_analysis.append(
jobset.JobSpec([sys.executable,
'tools/profiling/latency_profile/profile_analyzer.py',
'--source', '%s.trace' % fnize(line), '--fmt', 'simple',
'--out', 'reports/%s.txt' % fnize(line)], timeout_seconds=None))
cleanup.append(jobset.JobSpec(['rm', '%s.trace' % fnize(line)]))
# periodically flush out the list of jobs: profile_analysis jobs at least
# consume upwards of five gigabytes of ram in some cases, and so analysing
# hundreds of them at once is impractical -- but we want at least some
# concurrency or the work takes too long
if len(benchmarks) >= min(4, multiprocessing.cpu_count()):
# run up to half the cpu count: each benchmark can use up to two cores
# (one for the microbenchmark, one for the data flush)
jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2),
add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
benchmarks = []
profile_analysis = []
cleanup = []
# run the remaining benchmarks that weren't flushed
if len(benchmarks):
jobset.run(benchmarks, maxjobs=max(1, multiprocessing.cpu_count()/2),
add_env={'GRPC_TEST_PORT_SERVER': 'localhost:%d' % port_server_port})
jobset.run(profile_analysis, maxjobs=multiprocessing.cpu_count())
jobset.run(cleanup, maxjobs=multiprocessing.cpu_count())
def _docker_jobspec(name, runtests_args=[], inner_jobs=_DEFAULT_INNER_JOBS):
"""Run a single instance of run_tests.py in a docker container"""
test_job = jobset.JobSpec(cmdline=[
'python', 'tools/run_tests/run_tests.py', '--use_docker', '-t', '-j',
str(inner_jobs), '-x',
'report_%s_%s' % (name, _REPORT_SUFFIX), '--report_suite_name',
'%s' % name
] + runtests_args,
shortname='run_tests_%s' % name,
timeout_seconds=_RUNTESTS_TIMEOUT)
return test_job
def build_interop_stress_image_jobspec(language, tag=None):
"""Creates jobspec for building stress test docker image for a language"""
if not tag:
tag = 'grpc_interop_stress_%s:%s' % (language.safename, uuid.uuid4())
env = {'INTEROP_IMAGE': tag,
'BASE_NAME': 'grpc_interop_stress_%s' % language.safename}
build_job = jobset.JobSpec(cmdline=['tools/run_tests/dockerize/build_interop_stress_image.sh'],
environ=env,
shortname='build_docker_%s' % (language),
timeout_seconds=30 * 60)
build_job.tag = tag
return build_job
def create_jobspec(name, cmdline, environ=None, shell=False,
flake_retries=0, timeout_retries=0):
"""Creates jobspec."""
jobspec = jobset.JobSpec(
cmdline=cmdline,
environ=environ,
shortname='distribtest.%s' % (name),
timeout_seconds=10*60,
flake_retries=flake_retries,
timeout_retries=timeout_retries,
shell=shell)
return jobspec
def server_jobspec(language,
docker_image,
insecure=False,
manual_cmd_log=None):
"""Create jobspec for running a server"""
container_name = dockerjob.random_name('interop_server_%s' %
language.safename)
cmdline = bash_cmdline(
language.server_cmd([
'--port=%s' % _DEFAULT_SERVER_PORT,
'--use_tls=%s' % ('false' if insecure else 'true')
]))
environ = language.global_env()
if language.safename == 'http2':
# we are running the http2 interop server. Open next N ports beginning
# with the server port. These ports are used for http2 interop test
# (one test case per port). We also attach the docker container running
# the server to local network, so we don't have to mess with port mapping
port_args = [
'-p',
str(_DEFAULT_SERVER_PORT + 0),
'-p',
str(_DEFAULT_SERVER_PORT + 1),
'-p',
str(_DEFAULT_SERVER_PORT + 2),
'-p',
str(_DEFAULT_SERVER_PORT + 3),
'-p',
str(_DEFAULT_SERVER_PORT + 4),
'-p',
str(_DEFAULT_SERVER_PORT + 5),
'-p',
str(_DEFAULT_SERVER_PORT + 6),
'--net=host',
]
else:
port_args = ['-p', str(_DEFAULT_SERVER_PORT)]
docker_cmdline = docker_run_cmdline(cmdline,
image=docker_image,
cwd=language.server_cwd,
environ=environ,
docker_args=port_args +
['--name=%s' % container_name])
if manual_cmd_log is not None:
manual_cmd_log.append(manual_cmdline(docker_cmdline))
server_job = jobset.JobSpec(cmdline=docker_cmdline,
environ=environ,
shortname='interop_server_%s' % language,
timeout_seconds=30 * 60)
server_job.container_name = container_name
return server_job
def create_quit_jobspec(workers, remote_host=None):
"""Runs quit using QPS driver."""
# setting QPS_WORKERS env variable here makes sure it works with SSH too.
cmd = 'QPS_WORKERS="%s" bins/opt/qps_json_driver --quit' % ','.join(w.host_and_port for w in workers)
if remote_host:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host)
cmd = 'ssh %s "cd ~/performance_workspace/grpc/ && "%s' % (user_at_host, pipes.quote(cmd))
return jobset.JobSpec(
cmdline=[cmd],
shortname='qps_json_driver.quit',
timeout_seconds=3*60,
shell=True,
verbose_success=True)
def build_interop_image_jobspec(lang_safename, basename_prefix='grpc_interop'):
"""Creates jobspec for building interop docker image for a language"""
tag = '%s_%s:%s' % (basename_prefix, lang_safename, uuid.uuid4())
env = {
'INTEROP_IMAGE': tag,
'BASE_NAME': '%s_%s' % (basename_prefix, lang_safename),
}
build_job = jobset.JobSpec(
cmdline=['tools/run_tests/dockerize/build_interop_image.sh'],
environ=env,
shortname='build_docker_%s' % lang_safename,
timeout_seconds=30 * 60)
build_job.tag = tag
return build_job
def build_on_remote_hosts(hosts,
languages=scenario_config.LANGUAGES.keys(),
build_local=False):
"""Builds performance worker on remote hosts (and maybe also locally)."""
build_timeout = 15 * 60
build_jobs = []
for host in hosts:
user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host)
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/remote_host_build.sh'] +
languages,
shortname='remote_host_build.%s' % host,
environ={
'USER_AT_HOST': user_at_host,
'CONFIG': 'opt'
},
timeout_seconds=build_timeout))
if build_local:
# Build locally as well
build_jobs.append(
jobset.JobSpec(
cmdline=['tools/run_tests/performance/build_performance.sh'] +
languages,
shortname='local_build',
environ={'CONFIG': 'opt'},
timeout_seconds=build_timeout))
jobset.message('START', 'Building.', do_newline=True)
num_failures, _ = jobset.run(build_jobs,
newline_on_success=True,
maxjobs=10,
clear_alarms=False)
if num_failures == 0:
jobset.message('SUCCESS', 'Built successfully.', do_newline=True)
else:
jobset.message('FAILED', 'Build failed.', do_newline=True)
sys.exit(1)
def _workspace_jobspec(name, runtests_args=[], workspace_name=None, inner_jobs=_DEFAULT_INNER_JOBS):
"""Run a single instance of run_tests.py in a separate workspace"""
if not workspace_name:
workspace_name = 'workspace_%s' % name
env = {'WORKSPACE_NAME': workspace_name}
test_job = jobset.JobSpec(
cmdline=['bash',
'tools/run_tests/helper_scripts/run_tests_in_workspace.sh',
'-t',
'-j', str(inner_jobs),
'-x', '../report_%s.xml' % name,
'--report_suite_name', '%s' % name] + runtests_args,
environ=env,
shortname='run_tests_%s' % name,
timeout_seconds=_RUNTESTS_TIMEOUT)
return test_job
def _docker_jobspec(name, runtests_args=[], runtests_envs={},
inner_jobs=_DEFAULT_INNER_JOBS,
timeout_seconds=None):
"""Run a single instance of run_tests.py in a docker container"""
if not timeout_seconds:
timeout_seconds = _DEFAULT_RUNTESTS_TIMEOUT
test_job = jobset.JobSpec(
cmdline=['python', 'tools/run_tests/run_tests.py',
'--use_docker',
'-t',
'-j', str(inner_jobs),
'-x', _report_filename(name),
'--report_suite_name', '%s' % name] + runtests_args,
environ=runtests_envs,
shortname='run_tests_%s' % name,
timeout_seconds=timeout_seconds)
return test_job
def perf_report_processor_job(worker_host, perf_base_name, output_filename,
flame_graph_reports):
print('Creating perf report collection job for %s' % worker_host)
cmd = ''
if worker_host != 'localhost':
user_at_host = "%s@%s" % (_REMOTE_HOST_USERNAME, worker_host)
cmd = "USER_AT_HOST=%s OUTPUT_FILENAME=%s OUTPUT_DIR=%s PERF_BASE_NAME=%s tools/run_tests/performance/process_remote_perf_flamegraphs.sh" % (
user_at_host, output_filename, flame_graph_reports, perf_base_name)
else:
cmd = "OUTPUT_FILENAME=%s OUTPUT_DIR=%s PERF_BASE_NAME=%s tools/run_tests/performance/process_local_perf_flamegraphs.sh" % (
output_filename, flame_graph_reports, perf_base_name)
return jobset.JobSpec(cmdline=cmd,
timeout_seconds=3 * 60,
shell=True,
verbose_success=True,
shortname='process perf report')
def cloud_to_cloud_jobspec(language,
test_cases,
server_addresses,
test_duration_secs,
num_channels_per_server,
num_stubs_per_channel,
metrics_port,
docker_image=None):
"""Creates jobspec for cloud-to-cloud interop test"""
cmdline = bash_login_cmdline(
language.client_cmd([
'--test_cases=%s' % test_cases,
'--server_addresses=%s' % server_addresses,
'--test_duration_secs=%s' % test_duration_secs,
'--num_stubs_per_channel=%s' % num_stubs_per_channel,
'--num_channels_per_server=%s' % num_channels_per_server,
'--metrics_port=%s' % metrics_port
]))
print(cmdline)
cwd = language.client_cwd
environ = language.global_env()
if docker_image:
container_name = dockerjob.random_name('interop_client_%s' %
language.safename)
cmdline = docker_run_cmdline(
cmdline,
image=docker_image,
environ=environ,
cwd=cwd,
docker_args=['--net=host', '--name', container_name])
cwd = None
test_job = jobset.JobSpec(
cmdline=cmdline,
cwd=cwd,
environ=environ,
shortname='cloud_to_cloud:%s:%s_server:stress_test' %
(language, server_name),
timeout_seconds=test_duration_secs * 2,
flake_retries=0,
timeout_retries=0,
kill_handler=_job_kill_handler)
test_job.container_name = container_name
return test_job
def create_jobspec(name,
cmdline,
environ=None,
cwd=None,
shell=False,
flake_retries=0,
timeout_retries=0,
cpu_cost=1.0):
"""Creates jobspec."""
jobspec = jobset.JobSpec(cmdline=cmdline,
environ=environ,
cwd=cwd,
shortname='build_package.%s' % (name),
timeout_seconds=10 * 60,
flake_retries=flake_retries,
timeout_retries=timeout_retries,
cpu_cost=cpu_cost,
shell=shell)
return jobspec
