def parallel_run_test(test_descriptor_str):
    """
    This is invoked in parallel to actually run tests.
    """
    adjust_pythonpath()
    from yb import yb_dist_tests, command_util

    global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict)
    global_conf.set_env(propagated_env_vars)
    yb_dist_tests.global_conf = global_conf
    test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str)
    os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index)
    os.environ['build_type'] = global_conf.build_type

    yb_dist_tests.wait_for_clock_sync()

    # We could use "run_program" here, but it collects all the output in memory, which is not
    # ideal for a large amount of test log output. The "tee" part also makes the output visible in
    # the standard error of the Spark task as well, which is sometimes helpful for debugging.
    def run_test():
        start_time = time.time()
        exit_code = os.system(
            "bash -c 'set -o pipefail; \"{}\" {} 2>&1 | tee \"{}\"; {}'".
            format(global_conf.get_run_test_script_path(),
                   test_descriptor.args_for_run_test,
                   test_descriptor.error_output_path,
                   'exit ${PIPESTATUS[0]}')) >> 8
        # The ">> 8" is to get the exit code returned by os.system() in the high 8 bits of the
        # result.
        elapsed_time_sec = time.time() - start_time
        logging.info("Test {} ran on {}, rc={}".format(test_descriptor,
                                                       socket.gethostname(),
                                                       exit_code))
        return exit_code, elapsed_time_sec

    exit_code, elapsed_time_sec = run_test()
    error_output_path = test_descriptor.error_output_path

    failed_without_output = False
    if os.path.isfile(error_output_path) and os.path.getsize(
            error_output_path) == 0:
        if exit_code == 0:
            # Test succeeded, no error output.
            os.remove(error_output_path)
        else:
            # Test failed without any output! Re-run with "set -x" to diagnose.
            os.environ['YB_DEBUG_RUN_TEST'] = '1'
            exit_code, elapsed_time_sec = run_test()
            del os.environ['YB_DEBUG_RUN_TEST']
            # Also mark this in test results.
            failed_without_output = True

    return yb_dist_tests.TestResult(
        exit_code=exit_code,
        test_descriptor=test_descriptor,
        elapsed_time_sec=elapsed_time_sec,
        failed_without_output=failed_without_output)
示例#2
0
def parallel_run_test(test_descriptor_str):
    """
    This is invoked in parallel to actually run tests.
    """
    adjust_pythonpath()
    from yb import yb_dist_tests, command_util

    global_conf = yb_dist_tests.set_global_conf_from_dict(global_conf_dict)
    global_conf.set_env(propagated_env_vars)
    yb_dist_tests.global_conf = global_conf
    test_descriptor = yb_dist_tests.TestDescriptor(test_descriptor_str)
    os.environ['YB_TEST_ATTEMPT_INDEX'] = str(test_descriptor.attempt_index)
    os.environ['build_type'] = global_conf.build_type

    yb_dist_tests.wait_for_clock_sync()
    start_time = time.time()

    # We could use "run_program" here, but it collects all the output in memory, which is not
    # ideal for a large amount of test log output. The "tee" part also makes the output visible in
    # the standard error of the Spark task as well, which is sometimes helpful for debugging.
    exit_code = os.system(
        ("bash -c 'set -o pipefail; \"{}\" {} 2>&1 | tee \"{}\"'").format(
            global_conf.get_run_test_script_path(),
            test_descriptor.args_for_run_test,
            test_descriptor.error_output_path)) >> 8
    # The ">> 8" is to get the exit code returned by os.system() in the high 8 bits of the result.
    elapsed_time_sec = time.time() - start_time

    logging.info("Test {} ran on {}, rc={}".format(test_descriptor,
                                                   socket.gethostname(),
                                                   exit_code))
    error_output_path = test_descriptor.error_output_path
    if os.path.isfile(error_output_path) and os.path.getsize(
            error_output_path) == 0:
        os.remove(error_output_path)

    return yb_dist_tests.TestResult(exit_code=exit_code,
                                    test_descriptor=test_descriptor,
                                    elapsed_time_sec=elapsed_time_sec)
示例#3
0
        if os.path.isfile(error_output_path) and os.path.getsize(
                error_output_path) == 0:
            if exit_code == 0:
                # Test succeeded, no error output.
                os.remove(error_output_path)
            else:
                # Test failed without any output! Re-run with "set -x" to diagnose.
                os.environ['YB_DEBUG_RUN_TEST'] = '1'
                exit_code, elapsed_time_sec = run_test()
                del os.environ['YB_DEBUG_RUN_TEST']
                # Also mark this in test results.
                failed_without_output = True

        return yb_dist_tests.TestResult(
            exit_code=exit_code,
            test_descriptor=test_descriptor,
            elapsed_time_sec=elapsed_time_sec,
            failed_without_output=failed_without_output)
    finally:
        delete_if_exists_log_errors(test_started_running_flag_file)


def parallel_list_test_descriptors(rel_test_path):
    """
    This is invoked in parallel to list all individual tests within our C++ test programs. Without
    this, listing all gtest tests across 330 test programs might take about 5 minutes on TSAN and 2
    minutes in debug.
    """
    adjust_pythonpath()
    wait_for_path_to_exist(YB_PYTHONPATH_ENTRY)
    try: