def run_tests(config):
""" The main function that launches the stress tests """
# Build docker images and push to GKE registry
if config.global_settings.build_docker_images:
for name, docker_image in config.docker_images_dict.iteritems():
if not (docker_image.build_image() and
docker_image.push_to_gke_registry()):
return False
# Create a unique id for this run (Note: Using timestamp instead of UUID to
# make it easier to deduce the date/time of the run just by looking at the run
# run id. This is useful in debugging when looking at records in Biq query)
run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id)
bq_helper = BigQueryHelper(run_id, '', '',
config.global_settings.gcp_project_id, dataset_id,
config.global_settings.summary_table_id,
config.global_settings.qps_table_id)
bq_helper.initialize()
gke = Gke(config.global_settings.gcp_project_id, run_id, dataset_id,
config.global_settings.summary_table_id,
config.global_settings.qps_table_id,
config.global_settings.kubernetes_proxy_port)
is_success = True
try:
print 'Launching servers..'
for name, server_pod_spec in config.server_pod_specs_dict.iteritems():
if not gke.launch_servers(server_pod_spec):
is_success = False # is_success is checked in the 'finally' block
return False
print('Launched servers. Waiting for %d seconds for the server pods to be '
'fully online') % config.global_settings.pod_warmup_secs
time.sleep(config.global_settings.pod_warmup_secs)
for name, client_pod_spec in config.client_pod_specs_dict.iteritems():
if not gke.launch_clients(client_pod_spec):
is_success = False # is_success is checked in the 'finally' block
return False
print('Launched all clients. Waiting for %d seconds for the client pods to '
'be fully online') % config.global_settings.pod_warmup_secs
time.sleep(config.global_settings.pod_warmup_secs)
start_time = datetime.datetime.now()
end_time = start_time + datetime.timedelta(
seconds=config.global_settings.test_duration_secs)
print 'Running the test until %s' % end_time.isoformat()
while True:
if datetime.datetime.now() > end_time:
print 'Test was run for %d seconds' % config.global_settings.test_duration_secs
break
# Check if either stress server or clients have failed (btw, the bq_helper
# monitors all the rows in the summary table and checks if any of them
# have a failure status)
if bq_helper.check_if_any_tests_failed():
is_success = False
print 'Some tests failed.'
break # Don't 'return' here. We still want to call bq_helper to print qps/summary tables
# Tests running fine. Wait until next poll time to check the status
print 'Sleeping for %d seconds..' % config.global_settings.test_poll_interval_secs
time.sleep(config.global_settings.test_poll_interval_secs)
# Print BiqQuery tables
bq_helper.print_qps_records()
bq_helper.print_summary_records()
finally:
# If there was a test failure, we should not delete the pods since they
# would contain useful debug information (logs, core dumps etc)
if is_success:
for name, server_pod_spec in config.server_pod_specs_dict.iteritems():
gke.delete_servers(server_pod_spec)
for name, client_pod_spec in config.client_pod_specs_dict.iteritems():
gke.delete_clients(client_pod_spec)
return is_success
def run_tests(config):
""" The main function that launches the stress tests """
# Build docker images and push to GKE registry
if config.global_settings.build_docker_images:
for name, docker_image in config.docker_images_dict.iteritems():
if not (docker_image.build_image()
and docker_image.push_to_gke_registry()):
return False
# Create a unique id for this run (Note: Using timestamp instead of UUID to
# make it easier to deduce the date/time of the run just by looking at the run
# run id. This is useful in debugging when looking at records in Biq query)
run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
dataset_id = '%s_%s' % (config.global_settings.dataset_id_prefix, run_id)
print 'Run id:', run_id
print 'Dataset id:', dataset_id
bq_helper = BigQueryHelper(run_id, '', '',
config.global_settings.gcp_project_id,
dataset_id,
config.global_settings.summary_table_id,
config.global_settings.qps_table_id)
bq_helper.initialize()
gke = Gke(config.global_settings.gcp_project_id, run_id, dataset_id,
config.global_settings.summary_table_id,
config.global_settings.qps_table_id,
config.global_settings.kubernetes_proxy_port)
is_success = True
try:
print 'Launching servers..'
for name, server_pod_spec in config.server_pod_specs_dict.iteritems():
if not gke.launch_servers(server_pod_spec):
is_success = False # is_success is checked in the 'finally' block
return False
print(
'Launched servers. Waiting for %d seconds for the server pods to be '
'fully online') % config.global_settings.pod_warmup_secs
time.sleep(config.global_settings.pod_warmup_secs)
for name, client_pod_spec in config.client_pod_specs_dict.iteritems():
if not gke.launch_clients(client_pod_spec):
is_success = False # is_success is checked in the 'finally' block
return False
print(
'Launched all clients. Waiting for %d seconds for the client pods to '
'be fully online') % config.global_settings.pod_warmup_secs
time.sleep(config.global_settings.pod_warmup_secs)
start_time = datetime.datetime.now()
end_time = start_time + datetime.timedelta(
seconds=config.global_settings.test_duration_secs)
print 'Running the test until %s' % end_time.isoformat()
while True:
if datetime.datetime.now() > end_time:
print 'Test was run for %d seconds' % config.global_settings.test_duration_secs
break
# Check if either stress server or clients have failed (btw, the bq_helper
# monitors all the rows in the summary table and checks if any of them
# have a failure status)
if bq_helper.check_if_any_tests_failed():
is_success = False
print 'Some tests failed.'
break # Don't 'return' here. We still want to call bq_helper to print qps/summary tables
# Tests running fine. Wait until next poll time to check the status
print 'Sleeping for %d seconds..' % config.global_settings.test_poll_interval_secs
time.sleep(config.global_settings.test_poll_interval_secs)
# Print BiqQuery tables
bq_helper.print_qps_records()
bq_helper.print_summary_records()
finally:
# If there was a test failure, we should not delete the pods since they
# would contain useful debug information (logs, core dumps etc)
if is_success:
for name, server_pod_spec in config.server_pod_specs_dict.iteritems(
):
gke.delete_servers(server_pod_spec)
for name, client_pod_spec in config.client_pod_specs_dict.iteritems(
):
gke.delete_clients(client_pod_spec)
return is_success
import argparse
import os
import sys
stress_test_utils_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../../gcp/stress_test'))
sys.path.append(stress_test_utils_dir)
from stress_test_utils import BigQueryHelper
argp = argparse.ArgumentParser(
description='Print summary tables',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
argp.add_argument('--gcp_project_id',
required=True,
help='The Google Cloud Platform Project Id')
argp.add_argument('--dataset_id', type=str, required=True)
argp.add_argument('--run_id', type=str, required=True)
argp.add_argument('--summary_table_id', type=str, default='summary')
argp.add_argument('--qps_table_id', type=str, default='qps')
argp.add_argument('--summary_only', action='store_true', default=True)
if __name__ == '__main__':
args = argp.parse_args()
bq_helper = BigQueryHelper(args.run_id, '', '', args.gcp_project_id,
args.dataset_id, args.summary_table_id,
args.qps_table_id)
bq_helper.initialize()
if not args.summary_only:
bq_helper.print_qps_records()
bq_helper.print_summary_records()
import argparse
import os
import sys
stress_test_utils_dir = os.path.abspath(os.path.join(
os.path.dirname(__file__), '../../gcp/stress_test'))
sys.path.append(stress_test_utils_dir)
from stress_test_utils import BigQueryHelper
argp = argparse.ArgumentParser(
description='Print summary tables',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
argp.add_argument('--gcp_project_id',
required=True,
help='The Google Cloud Platform Project Id')
argp.add_argument('--dataset_id', type=str, required=True)
argp.add_argument('--run_id', type=str, required=True)
argp.add_argument('--summary_table_id', type=str, default='summary')
argp.add_argument('--qps_table_id', type=str, default='qps')
argp.add_argument('--summary_only', action='store_true', default=True)
if __name__ == '__main__':
args = argp.parse_args()
bq_helper = BigQueryHelper(args.run_id, '', '', args.gcp_project_id,
args.dataset_id, args.summary_table_id,
args.qps_table_id)
bq_helper.initialize()
if not args.summary_only:
bq_helper.print_qps_records()
bq_helper.print_summary_records()
def run_test_main(test_settings, gke_settings, stress_server_settings,
stress_client_clients):
is_success = True
if test_settings.build_docker_image:
is_success = _build_docker_image(gke_settings.docker_image_name,
gke_settings.tag_name)
if not is_success:
return False
is_success = _push_docker_image_to_gke_registry(gke_settings.tag_name)
if not is_success:
return False
# Create a unique id for this run (Note: Using timestamp instead of UUID to
# make it easier to deduce the date/time of the run just by looking at the run
# run id. This is useful in debugging when looking at records in Biq query)
run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
dataset_id = '%s_%s' % (_DATASET_ID_PREFIX, run_id)
# Big Query settings (common for both Stress Server and Client)
bq_settings = BigQuerySettings(run_id, dataset_id, _SUMMARY_TABLE_ID,
_QPS_TABLE_ID)
bq_helper = BigQueryHelper(run_id, '', '', args.project_id, dataset_id,
_SUMMARY_TABLE_ID, _QPS_TABLE_ID)
bq_helper.initialize()
try:
is_success = _launch_server_and_client(gke_settings, stress_server_settings,
stress_client_settings, bq_settings,
test_settings.kubernetes_proxy_port)
if not is_success:
return False
start_time = datetime.datetime.now()
end_time = start_time + datetime.timedelta(
seconds=test_settings.test_duration_secs)
print 'Running the test until %s' % end_time.isoformat()
while True:
if datetime.datetime.now() > end_time:
print 'Test was run for %d seconds' % test_settings.test_duration_secs
break
# Check if either stress server or clients have failed
if bq_helper.check_if_any_tests_failed():
is_success = False
print 'Some tests failed.'
break
# Things seem to be running fine. Wait until next poll time to check the
# status
print 'Sleeping for %d seconds..' % test_settings.test_poll_interval_secs
time.sleep(test_settings.test_poll_interval_secs)
# Print BiqQuery tables
bq_helper.print_summary_records()
bq_helper.print_qps_records()
finally:
# If is_success is False at this point, it means that the stress tests were
# started successfully but failed while running the tests. In this case we
# do should not delete the pods (since they contain all the failure
# information)
if is_success:
_delete_server_and_client(stress_server_settings, stress_client_settings,
test_settings.kubernetes_proxy_port)
return is_success
def run_test_main(test_settings, gke_settings, stress_server_settings,
stress_client_clients):
is_success = True
if test_settings.build_docker_image:
is_success = _build_docker_image(gke_settings.docker_image_name,
gke_settings.tag_name)
if not is_success:
return False
is_success = _push_docker_image_to_gke_registry(gke_settings.tag_name)
if not is_success:
return False
# Create a unique id for this run (Note: Using timestamp instead of UUID to
# make it easier to deduce the date/time of the run just by looking at the run
# run id. This is useful in debugging when looking at records in Biq query)
run_id = datetime.datetime.now().strftime('%Y_%m_%d_%H_%M_%S')
dataset_id = '%s_%s' % (_DATASET_ID_PREFIX, run_id)
# Big Query settings (common for both Stress Server and Client)
bq_settings = BigQuerySettings(run_id, dataset_id, _SUMMARY_TABLE_ID,
_QPS_TABLE_ID)
bq_helper = BigQueryHelper(run_id, '', '', args.project_id, dataset_id,
_SUMMARY_TABLE_ID, _QPS_TABLE_ID)
bq_helper.initialize()
try:
is_success = _launch_server_and_client(
gke_settings, stress_server_settings, stress_client_settings,
bq_settings, test_settings.kubernetes_proxy_port)
if not is_success:
return False
start_time = datetime.datetime.now()
end_time = start_time + datetime.timedelta(
seconds=test_settings.test_duration_secs)
print 'Running the test until %s' % end_time.isoformat()
while True:
if datetime.datetime.now() > end_time:
print 'Test was run for %d seconds' % test_settings.test_duration_secs
break
# Check if either stress server or clients have failed
if bq_helper.check_if_any_tests_failed():
is_success = False
print 'Some tests failed.'
break
# Things seem to be running fine. Wait until next poll time to check the
# status
print 'Sleeping for %d seconds..' % test_settings.test_poll_interval_secs
time.sleep(test_settings.test_poll_interval_secs)
# Print BiqQuery tables
bq_helper.print_summary_records()
bq_helper.print_qps_records()
finally:
# If is_success is False at this point, it means that the stress tests were
# started successfully but failed while running the tests. In this case we
# do should not delete the pods (since they contain all the failure
# information)
if is_success:
_delete_server_and_client(stress_server_settings,
stress_client_settings,
test_settings.kubernetes_proxy_port)
return is_success
