Beispiel #1
0
def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
    if is_core_present(tempdir):
        print('VPP core detected in %s. Last test running was %s' %
              (tempdir, core_crash_test))
        print(single_line_delim)
        spawn_gdb(vpp_binary, get_core_path(tempdir))
        print(single_line_delim)
Beispiel #2
0
def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
    if is_core_present(tempdir):
        print('VPP core detected in %s. Last test running was %s' %
              (tempdir, core_crash_test))
        print(single_line_delim)
        spawn_gdb(vpp_binary, get_core_path(tempdir))
        print(single_line_delim)
Beispiel #3
0
    def add_error(self, test, err, unittest_fn, error_type):
        if error_type == FAIL:
            self.log_error(test, err, 'addFailure')
            error_type_str = colorize("FAIL", RED)
        elif error_type == ERROR:
            self.log_error(test, err, 'addError')
            error_type_str = colorize("ERROR", RED)
        else:
            raise Exception('Error type %s cannot be used to record an '
                            'error or a failure' % error_type)

        unittest_fn(self, test, err)
        if self.current_test_case_info:
            self.result_string = "%s [ temp dir used by test case: %s ]" % \
                                 (error_type_str,
                                  self.current_test_case_info.tempdir)
            self.symlink_failed()
            self.failed_test_cases_info.add(self.current_test_case_info)
            if is_core_present(self.current_test_case_info.tempdir):
                if not self.current_test_case_info.core_crash_test:
                    if isinstance(test, unittest.suite._ErrorHolder):
                        test_name = str(test)
                    else:
                        test_name = "'{}' ({})".format(
                            get_testcase_doc_name(test), test.id())
                    self.current_test_case_info.core_crash_test = test_name
                self.core_crash_test_cases_info.add(
                    self.current_test_case_info)
        else:
            self.result_string = '%s [no temp dir]' % error_type_str

        self.send_result_through_pipe(test, error_type)
Beispiel #4
0
def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
    if is_core_present(tempdir):
        if debug_core:
            print('VPP core detected in %s. Last test running was %s' %
                  (tempdir, core_crash_test))
            print(single_line_delim)
            spawn_gdb(vpp_binary, get_core_path(tempdir))
            print(single_line_delim)
        elif compress_core:
            print("Compressing core-file in test directory `%s'" % tempdir)
            os.system("gzip %s" % get_core_path(tempdir))
Beispiel #5
0
def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
    if is_core_present(tempdir):
        if debug_core:
            print('VPP core detected in %s. Last test running was %s' %
                  (tempdir, core_crash_test))
            print(single_line_delim)
            spawn_gdb(vpp_binary, get_core_path(tempdir))
            print(single_line_delim)
        elif compress_core:
            print("Compressing core-file in test directory `%s'" % tempdir)
            os.system("gzip %s" % get_core_path(tempdir))
Beispiel #6
0
def run_forked(testcase_suites):
    wrapped_testcase_suites = set()

    # suites are unhashable, need to use list
    results = []
    unread_testcases = set()
    finished_unread_testcases = set()
    manager = StreamQueueManager()
    manager.start()
    for i in range(concurrent_tests):
        if testcase_suites:
            wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
                                                     manager)
            wrapped_testcase_suites.add(wrapped_testcase_suite)
            unread_testcases.add(wrapped_testcase_suite)
        else:
            break

    read_from_testcases = threading.Event()
    read_from_testcases.set()
    stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
                                        args=(unread_testcases,
                                              finished_unread_testcases,
                                              read_from_testcases))
    stdouterr_thread.start()

    failed_wrapped_testcases = set()
    stop_run = False

    try:
        while wrapped_testcase_suites:
            finished_testcase_suites = set()
            for wrapped_testcase_suite in wrapped_testcase_suites:
                while wrapped_testcase_suite.result_parent_end.poll():
                    wrapped_testcase_suite.result.process_result(
                        *wrapped_testcase_suite.result_parent_end.recv())
                    wrapped_testcase_suite.last_heard = time.time()

                while wrapped_testcase_suite.keep_alive_parent_end.poll():
                    wrapped_testcase_suite.last_test, \
                        wrapped_testcase_suite.last_test_vpp_binary, \
                        wrapped_testcase_suite.last_test_temp_dir, \
                        wrapped_testcase_suite.vpp_pid = \
                        wrapped_testcase_suite.keep_alive_parent_end.recv()
                    wrapped_testcase_suite.last_heard = time.time()

                if wrapped_testcase_suite.finished_parent_end.poll():
                    wrapped_testcase_suite.finished_parent_end.recv()
                    wrapped_testcase_suite.last_heard = time.time()
                    stop_run = process_finished_testsuite(
                        wrapped_testcase_suite, finished_testcase_suites,
                        failed_wrapped_testcases, results) or stop_run
                    continue

                fail = False
                if wrapped_testcase_suite.last_heard + test_timeout < \
                        time.time():
                    fail = True
                    wrapped_testcase_suite.logger.critical(
                        "Child test runner process timed out "
                        "(last test running was `%s' in `%s')!" %
                        (wrapped_testcase_suite.last_test,
                         wrapped_testcase_suite.last_test_temp_dir))
                elif not wrapped_testcase_suite.child.is_alive():
                    fail = True
                    wrapped_testcase_suite.logger.critical(
                        "Child test runner process unexpectedly died "
                        "(last test running was `%s' in `%s')!" %
                        (wrapped_testcase_suite.last_test,
                         wrapped_testcase_suite.last_test_temp_dir))
                elif wrapped_testcase_suite.last_test_temp_dir and \
                        wrapped_testcase_suite.last_test_vpp_binary:
                    if is_core_present(
                            wrapped_testcase_suite.last_test_temp_dir):
                        wrapped_testcase_suite.add_testclass_with_core()
                        if wrapped_testcase_suite.core_detected_at is None:
                            wrapped_testcase_suite.core_detected_at = \
                                time.time()
                        elif wrapped_testcase_suite.core_detected_at + \
                                core_timeout < time.time():
                            wrapped_testcase_suite.logger.critical(
                                "Child test runner process unresponsive and "
                                "core-file exists in test temporary directory "
                                "(last test running was `%s' in `%s')!" %
                                (wrapped_testcase_suite.last_test,
                                 wrapped_testcase_suite.last_test_temp_dir))
                            fail = True

                if fail:
                    wrapped_testcase_suite.child.terminate()
                    try:
                        # terminating the child process tends to leave orphan
                        # VPP process around
                        if wrapped_testcase_suite.vpp_pid:
                            os.kill(wrapped_testcase_suite.vpp_pid,
                                    signal.SIGTERM)
                    except OSError:
                        # already dead
                        pass
                    wrapped_testcase_suite.result.crashed = True
                    wrapped_testcase_suite.result.process_result(
                        wrapped_testcase_suite.last_test_id, ERROR)
                    stop_run = process_finished_testsuite(
                        wrapped_testcase_suite, finished_testcase_suites,
                        failed_wrapped_testcases, results) or stop_run

            for finished_testcase in finished_testcase_suites:
                finished_testcase.child.join()
                finished_testcase.close_pipes()
                wrapped_testcase_suites.remove(finished_testcase)
                finished_unread_testcases.add(finished_testcase)
                finished_testcase.stdouterr_queue.put(None)
                if stop_run:
                    while testcase_suites:
                        results.append(TestResult(testcase_suites.pop(0)))
                elif testcase_suites:
                    new_testcase = TestCaseWrapper(testcase_suites.pop(0),
                                                   manager)
                    wrapped_testcase_suites.add(new_testcase)
                    unread_testcases.add(new_testcase)
            time.sleep(0.1)
    except Exception:
        for wrapped_testcase_suite in wrapped_testcase_suites:
            wrapped_testcase_suite.child.terminate()
            wrapped_testcase_suite.stdouterr_queue.put(None)
        raise
    finally:
        read_from_testcases.clear()
        stdouterr_thread.join(test_timeout)
        manager.shutdown()

    handle_cores(failed_wrapped_testcases)
    return results
Beispiel #7
0
def run_forked(testcase_suites):
    wrapped_testcase_suites = set()

    # suites are unhashable, need to use list
    results = []
    unread_testcases = set()
    finished_unread_testcases = set()
    manager = StreamQueueManager()
    manager.start()
    for i in range(concurrent_tests):
        if testcase_suites:
            wrapped_testcase_suite = TestCaseWrapper(testcase_suites.pop(0),
                                                     manager)
            wrapped_testcase_suites.add(wrapped_testcase_suite)
            unread_testcases.add(wrapped_testcase_suite)
        else:
            break

    read_from_testcases = threading.Event()
    read_from_testcases.set()
    stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
                                        args=(unread_testcases,
                                              finished_unread_testcases,
                                              read_from_testcases))
    stdouterr_thread.start()

    failed_wrapped_testcases = set()
    stop_run = False

    try:
        while wrapped_testcase_suites:
            finished_testcase_suites = set()
            for wrapped_testcase_suite in wrapped_testcase_suites:
                while wrapped_testcase_suite.result_parent_end.poll():
                    wrapped_testcase_suite.result.process_result(
                        *wrapped_testcase_suite.result_parent_end.recv())
                    wrapped_testcase_suite.last_heard = time.time()

                while wrapped_testcase_suite.keep_alive_parent_end.poll():
                    wrapped_testcase_suite.last_test, \
                        wrapped_testcase_suite.last_test_vpp_binary, \
                        wrapped_testcase_suite.last_test_temp_dir, \
                        wrapped_testcase_suite.vpp_pid = \
                        wrapped_testcase_suite.keep_alive_parent_end.recv()
                    wrapped_testcase_suite.last_heard = time.time()

                if wrapped_testcase_suite.finished_parent_end.poll():
                    wrapped_testcase_suite.finished_parent_end.recv()
                    wrapped_testcase_suite.last_heard = time.time()
                    stop_run = process_finished_testsuite(
                        wrapped_testcase_suite,
                        finished_testcase_suites,
                        failed_wrapped_testcases,
                        results) or stop_run
                    continue

                fail = False
                if wrapped_testcase_suite.last_heard + test_timeout < \
                        time.time():
                    fail = True
                    wrapped_testcase_suite.logger.critical(
                        "Child test runner process timed out "
                        "(last test running was `%s' in `%s')!" %
                        (wrapped_testcase_suite.last_test,
                         wrapped_testcase_suite.last_test_temp_dir))
                elif not wrapped_testcase_suite.child.is_alive():
                    fail = True
                    wrapped_testcase_suite.logger.critical(
                        "Child test runner process unexpectedly died "
                        "(last test running was `%s' in `%s')!" %
                        (wrapped_testcase_suite.last_test,
                         wrapped_testcase_suite.last_test_temp_dir))
                elif wrapped_testcase_suite.last_test_temp_dir and \
                        wrapped_testcase_suite.last_test_vpp_binary:
                    if is_core_present(
                            wrapped_testcase_suite.last_test_temp_dir):
                        wrapped_testcase_suite.add_testclass_with_core()
                        if wrapped_testcase_suite.core_detected_at is None:
                            wrapped_testcase_suite.core_detected_at = \
                                time.time()
                        elif wrapped_testcase_suite.core_detected_at + \
                                core_timeout < time.time():
                            wrapped_testcase_suite.logger.critical(
                                "Child test runner process unresponsive and "
                                "core-file exists in test temporary directory "
                                "(last test running was `%s' in `%s')!" %
                                (wrapped_testcase_suite.last_test,
                                 wrapped_testcase_suite.last_test_temp_dir))
                            fail = True

                if fail:
                    wrapped_testcase_suite.child.terminate()
                    try:
                        # terminating the child process tends to leave orphan
                        # VPP process around
                        if wrapped_testcase_suite.vpp_pid:
                            os.kill(wrapped_testcase_suite.vpp_pid,
                                    signal.SIGTERM)
                    except OSError:
                        # already dead
                        pass
                    wrapped_testcase_suite.result.crashed = True
                    wrapped_testcase_suite.result.process_result(
                        wrapped_testcase_suite.last_test_id, ERROR)
                    stop_run = process_finished_testsuite(
                        wrapped_testcase_suite,
                        finished_testcase_suites,
                        failed_wrapped_testcases,
                        results) or stop_run

            for finished_testcase in finished_testcase_suites:
                finished_testcase.child.join()
                finished_testcase.close_pipes()
                wrapped_testcase_suites.remove(finished_testcase)
                finished_unread_testcases.add(finished_testcase)
                finished_testcase.stdouterr_queue.put(None)
                if stop_run:
                    while testcase_suites:
                        results.append(TestResult(testcase_suites.pop(0)))
                elif testcase_suites:
                    new_testcase = TestCaseWrapper(testcase_suites.pop(0),
                                                   manager)
                    wrapped_testcase_suites.add(new_testcase)
                    unread_testcases.add(new_testcase)
            time.sleep(0.1)
    except Exception:
        for wrapped_testcase_suite in wrapped_testcase_suites:
            wrapped_testcase_suite.child.terminate()
            wrapped_testcase_suite.stdouterr_queue.put(None)
        raise
    finally:
        read_from_testcases.clear()
        stdouterr_thread.join(test_timeout)
        manager.shutdown()

    handle_cores(failed_wrapped_testcases)
    return results
Beispiel #8
0
def run_forked(testcase_suites):
    wrapped_testcase_suites = set()
    solo_testcase_suites = []
    total_test_runners = 0

    # suites are unhashable, need to use list
    results = []
    unread_testcases = set()
    finished_unread_testcases = set()
    manager = StreamQueueManager()
    manager.start()
    total_test_runners = 0
    while total_test_runners < concurrent_tests:
        if testcase_suites:
            a_suite = testcase_suites.pop(0)
            if a_suite.is_tagged_run_solo:
                solo_testcase_suites.append(a_suite)
                continue
            wrapped_testcase_suite = TestCaseWrapper(a_suite, manager)
            wrapped_testcase_suites.add(wrapped_testcase_suite)
            unread_testcases.add(wrapped_testcase_suite)
            total_test_runners = total_test_runners + 1
        else:
            break

    while total_test_runners < 1 and solo_testcase_suites:
        if solo_testcase_suites:
            a_suite = solo_testcase_suites.pop(0)
            wrapped_testcase_suite = TestCaseWrapper(a_suite, manager)
            wrapped_testcase_suites.add(wrapped_testcase_suite)
            unread_testcases.add(wrapped_testcase_suite)
            total_test_runners = total_test_runners + 1
        else:
            break

    read_from_testcases = threading.Event()
    read_from_testcases.set()
    stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
                                        args=(unread_testcases,
                                              finished_unread_testcases,
                                              read_from_testcases))
    stdouterr_thread.start()

    failed_wrapped_testcases = set()
    stop_run = False

    try:
        while wrapped_testcase_suites:
            finished_testcase_suites = set()
            for wrapped_testcase_suite in wrapped_testcase_suites:
                while wrapped_testcase_suite.result_parent_end.poll():
                    wrapped_testcase_suite.result.process_result(
                        *wrapped_testcase_suite.result_parent_end.recv())
                    wrapped_testcase_suite.last_heard = time.time()

                while wrapped_testcase_suite.keep_alive_parent_end.poll():
                    wrapped_testcase_suite.last_test, \
                        wrapped_testcase_suite.last_test_vpp_binary, \
                        wrapped_testcase_suite.last_test_temp_dir, \
                        wrapped_testcase_suite.vpp_pid = \
                        wrapped_testcase_suite.keep_alive_parent_end.recv()
                    wrapped_testcase_suite.last_heard = time.time()

                if wrapped_testcase_suite.finished_parent_end.poll():
                    wrapped_testcase_suite.finished_parent_end.recv()
                    wrapped_testcase_suite.last_heard = time.time()
                    stop_run = process_finished_testsuite(
                        wrapped_testcase_suite, finished_testcase_suites,
                        failed_wrapped_testcases, results) or stop_run
                    continue

                fail = False
                if wrapped_testcase_suite.last_heard + test_timeout < \
                        time.time():
                    fail = True
                    wrapped_testcase_suite.logger.critical(
                        "Child test runner process timed out "
                        "(last test running was `%s' in `%s')!" %
                        (wrapped_testcase_suite.last_test,
                         wrapped_testcase_suite.last_test_temp_dir))
                elif not wrapped_testcase_suite.child.is_alive():
                    fail = True
                    wrapped_testcase_suite.logger.critical(
                        "Child test runner process unexpectedly died "
                        "(last test running was `%s' in `%s')!" %
                        (wrapped_testcase_suite.last_test,
                         wrapped_testcase_suite.last_test_temp_dir))
                elif wrapped_testcase_suite.last_test_temp_dir and \
                        wrapped_testcase_suite.last_test_vpp_binary:
                    if is_core_present(
                            wrapped_testcase_suite.last_test_temp_dir):
                        wrapped_testcase_suite.add_testclass_with_core()
                        if wrapped_testcase_suite.core_detected_at is None:
                            wrapped_testcase_suite.core_detected_at = \
                                time.time()
                        elif wrapped_testcase_suite.core_detected_at + \
                                core_timeout < time.time():
                            wrapped_testcase_suite.logger.critical(
                                "Child test runner process unresponsive and "
                                "core-file exists in test temporary directory "
                                "(last test running was `%s' in `%s')!" %
                                (wrapped_testcase_suite.last_test,
                                 wrapped_testcase_suite.last_test_temp_dir))
                            fail = True

                if fail:
                    wrapped_testcase_suite.child.terminate()
                    try:
                        # terminating the child process tends to leave orphan
                        # VPP process around
                        if wrapped_testcase_suite.vpp_pid:
                            os.kill(wrapped_testcase_suite.vpp_pid,
                                    signal.SIGTERM)
                    except OSError:
                        # already dead
                        pass
                    wrapped_testcase_suite.result.crashed = True
                    wrapped_testcase_suite.result.process_result(
                        wrapped_testcase_suite.last_test_id, ERROR)
                    stop_run = process_finished_testsuite(
                        wrapped_testcase_suite, finished_testcase_suites,
                        failed_wrapped_testcases, results) or stop_run

            for finished_testcase in finished_testcase_suites:
                # Somewhat surprisingly, the join below may
                # timeout, even if client signaled that
                # it finished - so we note it just in case.
                join_start = time.time()
                finished_testcase.child.join(test_finished_join_timeout)
                join_end = time.time()
                if join_end - join_start >= test_finished_join_timeout:
                    finished_testcase.logger.error(
                        "Timeout joining finished test: %s (pid %d)" %
                        (finished_testcase.last_test,
                         finished_testcase.child.pid))
                finished_testcase.close_pipes()
                wrapped_testcase_suites.remove(finished_testcase)
                finished_unread_testcases.add(finished_testcase)
                finished_testcase.stdouterr_queue.put(None)
                total_test_runners = total_test_runners - 1
                if stop_run:
                    while testcase_suites:
                        results.append(TestResult(testcase_suites.pop(0)))
                elif testcase_suites:
                    a_testcase = testcase_suites.pop(0)
                    while a_testcase and a_testcase.is_tagged_run_solo:
                        solo_testcase_suites.append(a_testcase)
                        if testcase_suites:
                            a_testcase = testcase_suites.pop(0)
                        else:
                            a_testcase = None
                    if a_testcase:
                        new_testcase = TestCaseWrapper(a_testcase, manager)
                        wrapped_testcase_suites.add(new_testcase)
                        total_test_runners = total_test_runners + 1
                        unread_testcases.add(new_testcase)
                if solo_testcase_suites and total_test_runners == 0:
                    a_testcase = solo_testcase_suites.pop(0)
                    new_testcase = TestCaseWrapper(a_testcase, manager)
                    wrapped_testcase_suites.add(new_testcase)
                    total_test_runners = total_test_runners + 1
                    unread_testcases.add(new_testcase)
            time.sleep(0.1)
    except Exception:
        for wrapped_testcase_suite in wrapped_testcase_suites:
            wrapped_testcase_suite.child.terminate()
            wrapped_testcase_suite.stdouterr_queue.put(None)
        raise
    finally:
        read_from_testcases.clear()
        stdouterr_thread.join(test_timeout)
        manager.shutdown()

    handle_cores(failed_wrapped_testcases)
    return results
Beispiel #9
0
def run_forked(testcase_suites):
    wrapped_testcase_suites = set()
    solo_testcase_suites = []

    # suites are unhashable, need to use list
    results = []
    unread_testcases = set()
    finished_unread_testcases = set()
    manager = StreamQueueManager()
    manager.start()
    tests_running = 0
    free_cpus = list(available_cpus)

    def on_suite_start(tc):
        nonlocal tests_running
        nonlocal free_cpus
        tests_running = tests_running + 1

    def on_suite_finish(tc):
        nonlocal tests_running
        nonlocal free_cpus
        tests_running = tests_running - 1
        assert tests_running >= 0
        free_cpus.extend(tc.get_assigned_cpus())

    def run_suite(suite):
        nonlocal manager
        nonlocal wrapped_testcase_suites
        nonlocal unread_testcases
        nonlocal free_cpus
        suite.assign_cpus(free_cpus[: suite.cpus_used])
        free_cpus = free_cpus[suite.cpus_used :]
        wrapper = TestCaseWrapper(suite, manager)
        wrapped_testcase_suites.add(wrapper)
        unread_testcases.add(wrapper)
        on_suite_start(suite)

    def can_run_suite(suite):
        return tests_running < max_concurrent_tests and (
            suite.cpus_used <= len(free_cpus) or suite.cpus_used > max_vpp_cpus
        )

    while free_cpus and testcase_suites:
        a_suite = testcase_suites[0]
        if a_suite.is_tagged_run_solo:
            a_suite = testcase_suites.pop(0)
            solo_testcase_suites.append(a_suite)
            continue
        if can_run_suite(a_suite):
            a_suite = testcase_suites.pop(0)
            run_suite(a_suite)
        else:
            break

    if tests_running == 0 and solo_testcase_suites:
        a_suite = solo_testcase_suites.pop(0)
        run_suite(a_suite)

    read_from_testcases = threading.Event()
    read_from_testcases.set()
    stdouterr_thread = threading.Thread(
        target=stdouterr_reader_wrapper,
        args=(unread_testcases, finished_unread_testcases, read_from_testcases),
    )
    stdouterr_thread.start()

    failed_wrapped_testcases = set()
    stop_run = False

    try:
        while wrapped_testcase_suites:
            finished_testcase_suites = set()
            for wrapped_testcase_suite in wrapped_testcase_suites:
                while wrapped_testcase_suite.result_parent_end.poll():
                    wrapped_testcase_suite.result.process_result(
                        *wrapped_testcase_suite.result_parent_end.recv()
                    )
                    wrapped_testcase_suite.last_heard = time.time()

                while wrapped_testcase_suite.keep_alive_parent_end.poll():
                    (
                        wrapped_testcase_suite.last_test,
                        wrapped_testcase_suite.last_test_vpp_binary,
                        wrapped_testcase_suite.last_test_temp_dir,
                        wrapped_testcase_suite.vpp_pid,
                    ) = wrapped_testcase_suite.keep_alive_parent_end.recv()
                    wrapped_testcase_suite.last_heard = time.time()

                if wrapped_testcase_suite.finished_parent_end.poll():
                    wrapped_testcase_suite.finished_parent_end.recv()
                    wrapped_testcase_suite.last_heard = time.time()
                    stop_run = (
                        process_finished_testsuite(
                            wrapped_testcase_suite,
                            finished_testcase_suites,
                            failed_wrapped_testcases,
                            results,
                        )
                        or stop_run
                    )
                    continue

                fail = False
                if wrapped_testcase_suite.last_heard + config.timeout < time.time():
                    fail = True
                    wrapped_testcase_suite.logger.critical(
                        "Child test runner process timed out "
                        "(last test running was `%s' in `%s')!"
                        % (
                            wrapped_testcase_suite.last_test,
                            wrapped_testcase_suite.last_test_temp_dir,
                        )
                    )
                elif not wrapped_testcase_suite.child.is_alive():
                    fail = True
                    wrapped_testcase_suite.logger.critical(
                        "Child test runner process unexpectedly died "
                        "(last test running was `%s' in `%s')!"
                        % (
                            wrapped_testcase_suite.last_test,
                            wrapped_testcase_suite.last_test_temp_dir,
                        )
                    )
                elif (
                    wrapped_testcase_suite.last_test_temp_dir
                    and wrapped_testcase_suite.last_test_vpp_binary
                ):
                    if is_core_present(wrapped_testcase_suite.last_test_temp_dir):
                        wrapped_testcase_suite.add_testclass_with_core()
                        if wrapped_testcase_suite.core_detected_at is None:
                            wrapped_testcase_suite.core_detected_at = time.time()
                        elif (
                            wrapped_testcase_suite.core_detected_at + core_timeout
                            < time.time()
                        ):
                            wrapped_testcase_suite.logger.critical(
                                "Child test runner process unresponsive and "
                                "core-file exists in test temporary directory "
                                "(last test running was `%s' in `%s')!"
                                % (
                                    wrapped_testcase_suite.last_test,
                                    wrapped_testcase_suite.last_test_temp_dir,
                                )
                            )
                            fail = True

                if fail:
                    wrapped_testcase_suite.child.terminate()
                    try:
                        # terminating the child process tends to leave orphan
                        # VPP process around
                        if wrapped_testcase_suite.vpp_pid:
                            os.kill(wrapped_testcase_suite.vpp_pid, signal.SIGTERM)
                    except OSError:
                        # already dead
                        pass
                    wrapped_testcase_suite.result.crashed = True
                    wrapped_testcase_suite.result.process_result(
                        wrapped_testcase_suite.last_test_id, ERROR
                    )
                    stop_run = (
                        process_finished_testsuite(
                            wrapped_testcase_suite,
                            finished_testcase_suites,
                            failed_wrapped_testcases,
                            results,
                        )
                        or stop_run
                    )

            for finished_testcase in finished_testcase_suites:
                # Somewhat surprisingly, the join below may
                # timeout, even if client signaled that
                # it finished - so we note it just in case.
                join_start = time.time()
                finished_testcase.child.join(test_finished_join_timeout)
                join_end = time.time()
                if join_end - join_start >= test_finished_join_timeout:
                    finished_testcase.logger.error(
                        "Timeout joining finished test: %s (pid %d)"
                        % (finished_testcase.last_test, finished_testcase.child.pid)
                    )
                finished_testcase.close_pipes()
                wrapped_testcase_suites.remove(finished_testcase)
                finished_unread_testcases.add(finished_testcase)
                finished_testcase.stdouterr_queue.put(None)
                on_suite_finish(finished_testcase)
                if stop_run:
                    while testcase_suites:
                        results.append(TestResult(testcase_suites.pop(0)))
                elif testcase_suites:
                    a_suite = testcase_suites.pop(0)
                    while a_suite and a_suite.is_tagged_run_solo:
                        solo_testcase_suites.append(a_suite)
                        if testcase_suites:
                            a_suite = testcase_suites.pop(0)
                        else:
                            a_suite = None
                    if a_suite and can_run_suite(a_suite):
                        run_suite(a_suite)
                if solo_testcase_suites and tests_running == 0:
                    a_suite = solo_testcase_suites.pop(0)
                    run_suite(a_suite)
            time.sleep(0.1)
    except Exception:
        for wrapped_testcase_suite in wrapped_testcase_suites:
            wrapped_testcase_suite.child.terminate()
            wrapped_testcase_suite.stdouterr_queue.put(None)
        raise
    finally:
        read_from_testcases.clear()
        stdouterr_thread.join(config.timeout)
        manager.shutdown()

    handle_cores(failed_wrapped_testcases)
    return results