def parallel_list_test_descriptors(rel_test_path): """ This is invoked in parallel to list all individual tests within our C++ test programs. Without this, listing all gtest tests across 330 test programs might take about 5 minutes on TSAN and 2 minutes in debug. """ adjust_pythonpath() wait_for_path_to_exist(YB_PYTHONPATH_ENTRY) try: from yb import yb_dist_tests, command_util except ImportError as ex: raise ImportError("%s. %s" % (ex.message, get_sys_path_info_str())) global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict) global_conf.set_env(propagated_env_vars) wait_for_path_to_exist(global_conf.build_root) list_tests_cmd_line = [ os.path.join(global_conf.build_root, rel_test_path), '--gtest_list_tests' ] try: prog_result = command_util.run_program(list_tests_cmd_line) except OSError, ex: logging.error("Failed running the command: %s", list_tests_cmd_line) raise
def parallel_run_test(test_descriptor_str): """ This is invoked in parallel to actually run tests. """ adjust_pythonpath() from yb import yb_dist_tests, command_util global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict) global_conf.set_env(propagated_env_vars) yb_dist_tests.global_conf = global_conf test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str) os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index) os.environ['build_type'] = global_conf.build_type yb_dist_tests.wait_for_clock_sync() # We could use "run_program" here, but it collects all the output in memory, which is not # ideal for a large amount of test log output. The "tee" part also makes the output visible in # the standard error of the Spark task as well, which is sometimes helpful for debugging. def run_test(): start_time = time.time() exit_code = os.system( "bash -c 'set -o pipefail; \"{}\" {} 2>&1 | tee \"{}\"; {}'". format(global_conf.get_run_test_script_path(), test_descriptor.args_for_run_test, test_descriptor.error_output_path, 'exit ${PIPESTATUS[0]}')) >> 8 # The ">> 8" is to get the exit code returned by os.system() in the high 8 bits of the # result. elapsed_time_sec = time.time() - start_time logging.info("Test {} ran on {}, rc={}".format(test_descriptor, socket.gethostname(), exit_code)) return exit_code, elapsed_time_sec exit_code, elapsed_time_sec = run_test() error_output_path = test_descriptor.error_output_path failed_without_output = False if os.path.isfile(error_output_path) and os.path.getsize( error_output_path) == 0: if exit_code == 0: # Test succeeded, no error output. os.remove(error_output_path) else: # Test failed without any output! Re-run with "set -x" to diagnose. os.environ['YB_DEBUG_RUN_TEST'] = '1' exit_code, elapsed_time_sec = run_test() del os.environ['YB_DEBUG_RUN_TEST'] # Also mark this in test results. failed_without_output = True return yb_dist_tests.TestResult( exit_code=exit_code, test_descriptor=test_descriptor, elapsed_time_sec=elapsed_time_sec, failed_without_output=failed_without_output)
def parallel_list_test_descriptors(rel_test_path): """ This is invoked in parallel to list all individual tests within our C++ test programs. Without this, listing all gtest tests across 330 test programs might take about 5 minutes on TSAN and 2 minutes in debug. """ adjust_pythonpath() try: from yb import yb_dist_tests, command_util except ImportError as ex: raise ImportError("%s. %s" % (ex.message, get_sys_path_info_str())) global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict) global_conf.set_env(propagated_env_vars) prog_result = command_util.run_program([ os.path.join(global_conf.build_root, rel_test_path), '--gtest_list_tests' ]) # --gtest_list_tests gives us the following output format: # TestSplitArgs. # Simple # SimpleWithSpaces # SimpleWithQuotes # BadWithQuotes # Empty # Error # BloomFilterReverseCompatibility # BloomFilterWrapper # PrefixExtractorFullFilter # PrefixExtractorBlockFilter # PrefixScan # OptimizeFiltersForHits # BloomStatsTestWithParam/BloomStatsTestWithParam. # BloomStatsTest/0 # GetParam() = (true, true) # BloomStatsTest/1 # GetParam() = (true, false) # BloomStatsTest/2 # GetParam() = (false, false) # BloomStatsTestWithIter/0 # GetParam() = (true, true) # BloomStatsTestWithIter/1 # GetParam() = (true, false) # BloomStatsTestWithIter/2 # GetParam() = (false, false) current_test = None test_descriptors = [] test_descriptor_prefix = rel_test_path + yb_dist_tests.TEST_DESCRIPTOR_SEPARATOR for line in prog_result.stdout.split("\n"): if ('Starting tracking the heap' in line or 'Dumping heap profile to' in line): continue line = line.rstrip() trimmed_line = HASH_COMMENT_RE.sub('', line.strip()).strip() if line.startswith(' '): test_descriptors.append(test_descriptor_prefix + current_test + trimmed_line) else: current_test = trimmed_line return test_descriptors
def parallel_run_test(test_descriptor_str): """ This is invoked in parallel to actually run tests. """ adjust_pythonpath() from yb import yb_dist_tests, command_util global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict) global_conf.set_env(propagated_env_vars) yb_dist_tests.global_conf = global_conf test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str) os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index) os.environ['build_type'] = global_conf.build_type yb_dist_tests.wait_for_clock_sync() # We could use "run_program" here, but it collects all the output in memory, which is not # ideal for a large amount of test log output. The "tee" part also makes the output visible in # the standard error of the Spark task as well, which is sometimes helpful for debugging. def run_test(): start_time = time.time() exit_code = os.system( "bash -c 'set -o pipefail; \"{}\" {} 2>&1 | tee \"{}\"; {}'".format( global_conf.get_run_test_script_path(), test_descriptor.args_for_run_test, test_descriptor.error_output_path, 'exit ${PIPESTATUS[0]}')) >> 8 # The ">> 8" is to get the exit code returned by os.system() in the high 8 bits of the # result. elapsed_time_sec = time.time() - start_time logging.info("Test {} ran on {}, rc={}".format( test_descriptor, socket.gethostname(), exit_code)) return exit_code, elapsed_time_sec exit_code, elapsed_time_sec = run_test() error_output_path = test_descriptor.error_output_path failed_without_output = False if os.path.isfile(error_output_path) and os.path.getsize(error_output_path) == 0: if exit_code == 0: # Test succeeded, no error output. os.remove(error_output_path) else: # Test failed without any output! Re-run with "set -x" to diagnose. os.environ['YB_DEBUG_RUN_TEST'] = '1' exit_code, elapsed_time_sec = run_test() del os.environ['YB_DEBUG_RUN_TEST'] # Also mark this in test results. failed_without_output = True return yb_dist_tests.TestResult( exit_code=exit_code, test_descriptor=test_descriptor, elapsed_time_sec=elapsed_time_sec, failed_without_output=failed_without_output)
def parallel_list_test_descriptors(rel_test_path): """ This is invoked in parallel to list all individual tests within our C++ test programs. Without this, listing all gtest tests across 330 test programs might take about 5 minutes on TSAN and 2 minutes in debug. """ adjust_pythonpath() from yb import yb_dist_tests, command_util global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict) global_conf.set_env(propagated_env_vars) prog_result = command_util.run_program( [os.path.join(global_conf.build_root, rel_test_path), '--gtest_list_tests']) # --gtest_list_tests gives us the following output format: # TestSplitArgs. # Simple # SimpleWithSpaces # SimpleWithQuotes # BadWithQuotes # Empty # Error # BloomFilterReverseCompatibility # BloomFilterWrapper # PrefixExtractorFullFilter # PrefixExtractorBlockFilter # PrefixScan # OptimizeFiltersForHits # BloomStatsTestWithParam/BloomStatsTestWithParam. # BloomStatsTest/0 # GetParam() = (true, true) # BloomStatsTest/1 # GetParam() = (true, false) # BloomStatsTest/2 # GetParam() = (false, false) # BloomStatsTestWithIter/0 # GetParam() = (true, true) # BloomStatsTestWithIter/1 # GetParam() = (true, false) # BloomStatsTestWithIter/2 # GetParam() = (false, false) current_test = None test_descriptors = [] test_descriptor_prefix = rel_test_path + yb_dist_tests.TEST_DESCRIPTOR_SEPARATOR for line in prog_result.stdout.split("\n"): if ('Starting tracking the heap' in line or 'Dumping heap profile to' in line): continue line = line.rstrip() trimmed_line = HASH_COMMENT_RE.sub('', line.strip()).strip() if line.startswith(' '): test_descriptors.append(test_descriptor_prefix + current_test + trimmed_line) else: current_test = trimmed_line return test_descriptors
def parallel_run_test(test_descriptor_str): """ This is invoked in parallel to actually run tests. """ adjust_pythonpath() from yb import yb_dist_tests, command_util global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict) global_conf.set_env(propagated_env_vars) yb_dist_tests.global_conf = global_conf test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str) os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index) os.environ['build_type'] = global_conf.build_type yb_dist_tests.wait_for_clock_sync() start_time = time.time() # We could use "run_program" here, but it collects all the output in memory, which is not # ideal for a large amount of test log output. The "tee" part also makes the output visible in # the standard error of the Spark task as well, which is sometimes helpful for debugging. exit_code = os.system( ("bash -c 'set -o pipefail; \"{}\" {} 2>&1 | tee \"{}\"'").format( global_conf.get_run_test_script_path(), test_descriptor.args_for_run_test, test_descriptor.error_output_path)) >> 8 # The ">> 8" is to get the exit code returned by os.system() in the high 8 bits of the result. elapsed_time_sec = time.time() - start_time logging.info("Test {} ran on {}, rc={}".format(test_descriptor, socket.gethostname(), exit_code)) error_output_path = test_descriptor.error_output_path if os.path.isfile(error_output_path) and os.path.getsize( error_output_path) == 0: os.remove(error_output_path) return yb_dist_tests.TestResult(exit_code=exit_code, test_descriptor=test_descriptor, elapsed_time_sec=elapsed_time_sec)
def parallel_run_test(test_descriptor_str): """ This is invoked in parallel to actually run tests. """ adjust_pythonpath() wait_for_path_to_exist(YB_PYTHONPATH_ENTRY) try: from yb import yb_dist_tests, command_util except ImportError as ex: raise ImportError("%s. %s" % (ex.message, get_sys_path_info_str())) global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict) global_conf.set_env(propagated_env_vars) wait_for_path_to_exist(global_conf.build_root) yb_dist_tests.global_conf = global_conf test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str) # This is saved in the test result file by process_test_result.py. os.environ['YB_TEST_DESCRIPTOR_STR'] = test_descriptor_str os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index) os.environ['build_type'] = global_conf.build_type os.environ['YB_RUNNING_TEST_ON_SPARK'] = '1' os.environ['BUILD_ROOT'] = global_conf.build_root test_started_running_flag_file = os.path.join( tempfile.gettempdir(), 'yb_test_started_running_flag_file_%d_%s' % (os.getpid(), ''.join('%09d' % random.randrange(0, 1000000000) for i in xrange(4)))) os.environ[ 'YB_TEST_STARTED_RUNNING_FLAG_FILE'] = test_started_running_flag_file os.environ[ 'YB_TEST_EXTRA_ERROR_LOG_PATH'] = test_descriptor.error_output_path yb_dist_tests.wait_for_clock_sync() # We could use "run_program" here, but it collects all the output in memory, which is not # ideal for a large amount of test log output. The "tee" part also makes the output visible in # the standard error of the Spark task as well, which is sometimes helpful for debugging. def run_test(): start_time_sec = time.time() runner_oneline = 'set -o pipefail; "%s" %s 2>&1 | tee "%s"; exit ${PIPESTATUS[0]}' % ( global_conf.get_run_test_script_path(), test_descriptor.args_for_run_test, test_descriptor.error_output_path) process = subprocess.Popen([get_bash_path(), '-c', runner_oneline]) found_flag_file = False while is_pid_running(process.pid): elapsed_time_sec = time.time() - start_time_sec termination_reason = None if elapsed_time_sec > TEST_TIMEOUT_UPPER_BOUND_SEC: termination_reason = 'ran longer than %d seconds' % TEST_TIMEOUT_UPPER_BOUND_SEC failed_to_launch = False if not found_flag_file: if os.path.exists(test_started_running_flag_file): found_flag_file = True elif elapsed_time_sec > TIME_SEC_TO_START_RUNNING_TEST: termination_reason = ( 'could not start running the test in %d seconds (file %s not created). ' 'Ran command: {{ %s }}.') % ( TIME_SEC_TO_START_RUNNING_TEST, test_started_running_flag_file, runner_oneline) failed_to_launch = True if termination_reason: error_msg = "Test %s is being terminated (ran for %.1f seconds), reason: %s" % ( test_descriptor, elapsed_time_sec, termination_reason) logging.info(error_msg) try: os.kill(process.pid, signal.SIGKILL) except OSError, os_error: if os_error.errno == errno.ESRCH: logging.info( "Process with pid %d disappeared suddenly, that's OK", process.pid) pass raise os_error if failed_to_launch: # This exception should bubble up to Spark and cause it to hopefully re-run the # test one some other node. raise RuntimeError(error_msg) break time.sleep(0.5) exit_code = process.wait() elapsed_time_sec = time.time() - start_time_sec logging.info("Test {} ran on {} in %.1f seconds, rc={}".format( test_descriptor, socket.gethostname(), exit_code)) return exit_code, elapsed_time_sec