def parallel_list_test_descriptors(rel_test_path):
"""
This is invoked in parallel to list all individual tests within our C++ test programs. Without
this, listing all gtest tests across 330 test programs might take about 5 minutes on TSAN and 2
minutes in debug.
"""
adjust_pythonpath()
wait_for_path_to_exist(YB_PYTHONPATH_ENTRY)
try:
from yb import yb_dist_tests, command_util
except ImportError as ex:
raise ImportError("%s. %s" % (ex.message, get_sys_path_info_str()))
global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict)
global_conf.set_env(propagated_env_vars)
wait_for_path_to_exist(global_conf.build_root)
list_tests_cmd_line = [
os.path.join(global_conf.build_root, rel_test_path),
'--gtest_list_tests'
]
try:
prog_result = command_util.run_program(list_tests_cmd_line)
except OSError, ex:
logging.error("Failed running the command: %s", list_tests_cmd_line)
raise
def parallel_run_test(test_descriptor_str):
"""
This is invoked in parallel to actually run tests.
"""
adjust_pythonpath()
from yb import yb_dist_tests, command_util
global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict)
global_conf.set_env(propagated_env_vars)
yb_dist_tests.global_conf = global_conf
test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str)
os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index)
os.environ['build_type'] = global_conf.build_type
yb_dist_tests.wait_for_clock_sync()
# We could use "run_program" here, but it collects all the output in memory, which is not
# ideal for a large amount of test log output. The "tee" part also makes the output visible in
# the standard error of the Spark task as well, which is sometimes helpful for debugging.
def run_test():
start_time = time.time()
exit_code = os.system(
"bash -c 'set -o pipefail; \"{}\" {} 2>&1 | tee \"{}\"; {}'".
format(global_conf.get_run_test_script_path(),
test_descriptor.args_for_run_test,
test_descriptor.error_output_path,
'exit ${PIPESTATUS[0]}')) >> 8
# The ">> 8" is to get the exit code returned by os.system() in the high 8 bits of the
# result.
elapsed_time_sec = time.time() - start_time
logging.info("Test {} ran on {}, rc={}".format(test_descriptor,
socket.gethostname(),
exit_code))
return exit_code, elapsed_time_sec
exit_code, elapsed_time_sec = run_test()
error_output_path = test_descriptor.error_output_path
failed_without_output = False
if os.path.isfile(error_output_path) and os.path.getsize(
error_output_path) == 0:
if exit_code == 0:
# Test succeeded, no error output.
os.remove(error_output_path)
else:
# Test failed without any output! Re-run with "set -x" to diagnose.
os.environ['YB_DEBUG_RUN_TEST'] = '1'
exit_code, elapsed_time_sec = run_test()
del os.environ['YB_DEBUG_RUN_TEST']
# Also mark this in test results.
failed_without_output = True
return yb_dist_tests.TestResult(
exit_code=exit_code,
test_descriptor=test_descriptor,
elapsed_time_sec=elapsed_time_sec,
failed_without_output=failed_without_output)
def parallel_list_test_descriptors(rel_test_path):
"""
This is invoked in parallel to list all individual tests within our C++ test programs. Without
this, listing all gtest tests across 330 test programs might take about 5 minutes on TSAN and 2
minutes in debug.
"""
adjust_pythonpath()
try:
from yb import yb_dist_tests, command_util
except ImportError as ex:
raise ImportError("%s. %s" % (ex.message, get_sys_path_info_str()))
global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict)
global_conf.set_env(propagated_env_vars)
prog_result = command_util.run_program([
os.path.join(global_conf.build_root, rel_test_path),
'--gtest_list_tests'
])
# --gtest_list_tests gives us the following output format:
# TestSplitArgs.
# Simple
# SimpleWithSpaces
# SimpleWithQuotes
# BadWithQuotes
# Empty
# Error
# BloomFilterReverseCompatibility
# BloomFilterWrapper
# PrefixExtractorFullFilter
# PrefixExtractorBlockFilter
# PrefixScan
# OptimizeFiltersForHits
# BloomStatsTestWithParam/BloomStatsTestWithParam.
# BloomStatsTest/0 # GetParam() = (true, true)
# BloomStatsTest/1 # GetParam() = (true, false)
# BloomStatsTest/2 # GetParam() = (false, false)
# BloomStatsTestWithIter/0 # GetParam() = (true, true)
# BloomStatsTestWithIter/1 # GetParam() = (true, false)
# BloomStatsTestWithIter/2 # GetParam() = (false, false)
current_test = None
test_descriptors = []
test_descriptor_prefix = rel_test_path + yb_dist_tests.TEST_DESCRIPTOR_SEPARATOR
for line in prog_result.stdout.split("\n"):
if ('Starting tracking the heap' in line
or 'Dumping heap profile to' in line):
continue
line = line.rstrip()
trimmed_line = HASH_COMMENT_RE.sub('', line.strip()).strip()
if line.startswith(' '):
test_descriptors.append(test_descriptor_prefix + current_test +
trimmed_line)
else:
current_test = trimmed_line
return test_descriptors
def parallel_run_test(test_descriptor_str):
"""
This is invoked in parallel to actually run tests.
"""
adjust_pythonpath()
from yb import yb_dist_tests, command_util
global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict)
global_conf.set_env(propagated_env_vars)
yb_dist_tests.global_conf = global_conf
test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str)
os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index)
os.environ['build_type'] = global_conf.build_type
yb_dist_tests.wait_for_clock_sync()
# We could use "run_program" here, but it collects all the output in memory, which is not
# ideal for a large amount of test log output. The "tee" part also makes the output visible in
# the standard error of the Spark task as well, which is sometimes helpful for debugging.
def run_test():
start_time = time.time()
exit_code = os.system(
"bash -c 'set -o pipefail; \"{}\" {} 2>&1 | tee \"{}\"; {}'".format(
global_conf.get_run_test_script_path(),
test_descriptor.args_for_run_test,
test_descriptor.error_output_path,
'exit ${PIPESTATUS[0]}')) >> 8
# The ">> 8" is to get the exit code returned by os.system() in the high 8 bits of the
# result.
elapsed_time_sec = time.time() - start_time
logging.info("Test {} ran on {}, rc={}".format(
test_descriptor, socket.gethostname(), exit_code))
return exit_code, elapsed_time_sec
exit_code, elapsed_time_sec = run_test()
error_output_path = test_descriptor.error_output_path
failed_without_output = False
if os.path.isfile(error_output_path) and os.path.getsize(error_output_path) == 0:
if exit_code == 0:
# Test succeeded, no error output.
os.remove(error_output_path)
else:
# Test failed without any output! Re-run with "set -x" to diagnose.
os.environ['YB_DEBUG_RUN_TEST'] = '1'
exit_code, elapsed_time_sec = run_test()
del os.environ['YB_DEBUG_RUN_TEST']
# Also mark this in test results.
failed_without_output = True
return yb_dist_tests.TestResult(
exit_code=exit_code,
test_descriptor=test_descriptor,
elapsed_time_sec=elapsed_time_sec,
failed_without_output=failed_without_output)
def parallel_list_test_descriptors(rel_test_path):
"""
This is invoked in parallel to list all individual tests within our C++ test programs. Without
this, listing all gtest tests across 330 test programs might take about 5 minutes on TSAN and 2
minutes in debug.
"""
adjust_pythonpath()
from yb import yb_dist_tests, command_util
global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict)
global_conf.set_env(propagated_env_vars)
prog_result = command_util.run_program(
[os.path.join(global_conf.build_root, rel_test_path), '--gtest_list_tests'])
# --gtest_list_tests gives us the following output format:
# TestSplitArgs.
# Simple
# SimpleWithSpaces
# SimpleWithQuotes
# BadWithQuotes
# Empty
# Error
# BloomFilterReverseCompatibility
# BloomFilterWrapper
# PrefixExtractorFullFilter
# PrefixExtractorBlockFilter
# PrefixScan
# OptimizeFiltersForHits
# BloomStatsTestWithParam/BloomStatsTestWithParam.
# BloomStatsTest/0 # GetParam() = (true, true)
# BloomStatsTest/1 # GetParam() = (true, false)
# BloomStatsTest/2 # GetParam() = (false, false)
# BloomStatsTestWithIter/0 # GetParam() = (true, true)
# BloomStatsTestWithIter/1 # GetParam() = (true, false)
# BloomStatsTestWithIter/2 # GetParam() = (false, false)
current_test = None
test_descriptors = []
test_descriptor_prefix = rel_test_path + yb_dist_tests.TEST_DESCRIPTOR_SEPARATOR
for line in prog_result.stdout.split("\n"):
if ('Starting tracking the heap' in line or 'Dumping heap profile to' in line):
continue
line = line.rstrip()
trimmed_line = HASH_COMMENT_RE.sub('', line.strip()).strip()
if line.startswith(' '):
test_descriptors.append(test_descriptor_prefix + current_test + trimmed_line)
else:
current_test = trimmed_line
return test_descriptors
def parallel_run_test(test_descriptor_str):
"""
This is invoked in parallel to actually run tests.
"""
adjust_pythonpath()
from yb import yb_dist_tests, command_util
global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict)
global_conf.set_env(propagated_env_vars)
yb_dist_tests.global_conf = global_conf
test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str)
os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index)
os.environ['build_type'] = global_conf.build_type
yb_dist_tests.wait_for_clock_sync()
start_time = time.time()
# We could use "run_program" here, but it collects all the output in memory, which is not
# ideal for a large amount of test log output. The "tee" part also makes the output visible in
# the standard error of the Spark task as well, which is sometimes helpful for debugging.
exit_code = os.system(
("bash -c 'set -o pipefail; \"{}\" {} 2>&1 | tee \"{}\"'").format(
global_conf.get_run_test_script_path(),
test_descriptor.args_for_run_test,
test_descriptor.error_output_path)) >> 8
# The ">> 8" is to get the exit code returned by os.system() in the high 8 bits of the result.
elapsed_time_sec = time.time() - start_time
logging.info("Test {} ran on {}, rc={}".format(test_descriptor,
socket.gethostname(),
exit_code))
error_output_path = test_descriptor.error_output_path
if os.path.isfile(error_output_path) and os.path.getsize(
error_output_path) == 0:
os.remove(error_output_path)
return yb_dist_tests.TestResult(exit_code=exit_code,
test_descriptor=test_descriptor,
elapsed_time_sec=elapsed_time_sec)
def parallel_run_test(test_descriptor_str):
"""
This is invoked in parallel to actually run tests.
"""
adjust_pythonpath()
wait_for_path_to_exist(YB_PYTHONPATH_ENTRY)
try:
from yb import yb_dist_tests, command_util
except ImportError as ex:
raise ImportError("%s. %s" % (ex.message, get_sys_path_info_str()))
global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict)
global_conf.set_env(propagated_env_vars)
wait_for_path_to_exist(global_conf.build_root)
yb_dist_tests.global_conf = global_conf
test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str)
# This is saved in the test result file by process_test_result.py.
os.environ['YB_TEST_DESCRIPTOR_STR'] = test_descriptor_str
os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index)
os.environ['build_type'] = global_conf.build_type
os.environ['YB_RUNNING_TEST_ON_SPARK'] = '1'
os.environ['BUILD_ROOT'] = global_conf.build_root
test_started_running_flag_file = os.path.join(
tempfile.gettempdir(), 'yb_test_started_running_flag_file_%d_%s' %
(os.getpid(), ''.join('%09d' % random.randrange(0, 1000000000)
for i in xrange(4))))
os.environ[
'YB_TEST_STARTED_RUNNING_FLAG_FILE'] = test_started_running_flag_file
os.environ[
'YB_TEST_EXTRA_ERROR_LOG_PATH'] = test_descriptor.error_output_path
yb_dist_tests.wait_for_clock_sync()
# We could use "run_program" here, but it collects all the output in memory, which is not
# ideal for a large amount of test log output. The "tee" part also makes the output visible in
# the standard error of the Spark task as well, which is sometimes helpful for debugging.
def run_test():
start_time_sec = time.time()
runner_oneline = 'set -o pipefail; "%s" %s 2>&1 | tee "%s"; exit ${PIPESTATUS[0]}' % (
global_conf.get_run_test_script_path(),
test_descriptor.args_for_run_test,
test_descriptor.error_output_path)
process = subprocess.Popen([get_bash_path(), '-c', runner_oneline])
found_flag_file = False
while is_pid_running(process.pid):
elapsed_time_sec = time.time() - start_time_sec
termination_reason = None
if elapsed_time_sec > TEST_TIMEOUT_UPPER_BOUND_SEC:
termination_reason = 'ran longer than %d seconds' % TEST_TIMEOUT_UPPER_BOUND_SEC
failed_to_launch = False
if not found_flag_file:
if os.path.exists(test_started_running_flag_file):
found_flag_file = True
elif elapsed_time_sec > TIME_SEC_TO_START_RUNNING_TEST:
termination_reason = (
'could not start running the test in %d seconds (file %s not created). '
'Ran command: {{ %s }}.') % (
TIME_SEC_TO_START_RUNNING_TEST,
test_started_running_flag_file, runner_oneline)
failed_to_launch = True
if termination_reason:
error_msg = "Test %s is being terminated (ran for %.1f seconds), reason: %s" % (
test_descriptor, elapsed_time_sec, termination_reason)
logging.info(error_msg)
try:
os.kill(process.pid, signal.SIGKILL)
except OSError, os_error:
if os_error.errno == errno.ESRCH:
logging.info(
"Process with pid %d disappeared suddenly, that's OK",
process.pid)
pass
raise os_error
if failed_to_launch:
# This exception should bubble up to Spark and cause it to hopefully re-run the
# test one some other node.
raise RuntimeError(error_msg)
break
time.sleep(0.5)
exit_code = process.wait()
elapsed_time_sec = time.time() - start_time_sec
logging.info("Test {} ran on {} in %.1f seconds, rc={}".format(
test_descriptor, socket.gethostname(), exit_code))
return exit_code, elapsed_time_sec
