def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
if is_core_present(tempdir):
print('VPP core detected in %s. Last test running was %s' %
(tempdir, core_crash_test))
print(single_line_delim)
spawn_gdb(vpp_binary, get_core_path(tempdir))
print(single_line_delim)
def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
if is_core_present(tempdir):
print('VPP core detected in %s. Last test running was %s' %
(tempdir, core_crash_test))
print(single_line_delim)
spawn_gdb(vpp_binary, get_core_path(tempdir))
print(single_line_delim)
def on_crash(self, core_path):
if self.testcase.debug_core:
# notify parent process that we're handling a core file
open('%s/_core_handled' % self.testcase.tempdir, 'a').close()
spawn_gdb(self.testcase.vpp_bin, core_path, self.logger)
else:
self.logger.critical("Core file present, debug with: gdb %s %s" %
(self.testcase.vpp_bin, core_path))
def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
if is_core_present(tempdir):
if debug_core:
print('VPP core detected in %s. Last test running was %s' %
(tempdir, core_crash_test))
print(single_line_delim)
spawn_gdb(vpp_binary, get_core_path(tempdir))
print(single_line_delim)
elif compress_core:
print("Compressing core-file in test directory `%s'" % tempdir)
os.system("gzip %s" % get_core_path(tempdir))
def check_and_handle_core(vpp_binary, tempdir, core_crash_test):
if is_core_present(tempdir):
if debug_core:
print('VPP core detected in %s. Last test running was %s' %
(tempdir, core_crash_test))
print(single_line_delim)
spawn_gdb(vpp_binary, get_core_path(tempdir))
print(single_line_delim)
elif compress_core:
print("Compressing core-file in test directory `%s'" % tempdir)
os.system("gzip %s" % get_core_path(tempdir))
def run_forked(suite):
keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
result_parent_end, result_child_end = Pipe(duplex=False)
child = Process(target=test_runner_wrapper,
args=(suite, keep_alive_child_end, result_child_end))
child.start()
last_test_temp_dir = None
last_test_vpp_binary = None
last_test = None
result = None
while result is None:
readable = select.select([
keep_alive_parent_end.fileno(),
result_parent_end.fileno(),
], [], [], test_timeout)[0]
if result_parent_end.fileno() in readable:
result = result_parent_end.recv()
elif keep_alive_parent_end.fileno() in readable:
while keep_alive_parent_end.poll():
last_test, last_test_vpp_binary,\
last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
else:
global_logger.critical("Timeout while waiting for child test "
"runner process (last test running was "
"`%s' in `%s')!" %
(last_test, last_test_temp_dir))
failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
lttd = last_test_temp_dir.split("/")[-1]
link_path = '%s%s-FAILED' % (failed_dir, lttd)
global_logger.error("Creating a link to the failed " +
"test: %s -> %s" % (link_path, lttd))
os.symlink(last_test_temp_dir, link_path)
api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
if os.path.isfile(api_post_mortem_path):
global_logger.error("Copying api_post_mortem.%d to %s" %
(vpp_pid, last_test_temp_dir))
shutil.copy2(api_post_mortem_path, last_test_temp_dir)
if last_test_temp_dir and last_test_vpp_binary:
core_path = "%s/core" % last_test_temp_dir
if os.path.isfile(core_path):
global_logger.error("Core-file exists in test temporary "
"directory: %s!" % core_path)
if d and d.lower() == "core":
spawn_gdb(last_test_vpp_binary, core_path,
global_logger)
child.terminate()
result = -1
keep_alive_parent_end.close()
result_parent_end.close()
return result
def on_crash(self, core_path):
if self.testcase.debug_core:
# notify parent process that we're handling a core file
open('%s/_core_handled' % self.testcase.tempdir, 'a').close()
spawn_gdb(self.testcase.vpp_bin, core_path, self.logger)
else:
self.logger.error("Core file present, debug with: gdb %s %s" %
(self.testcase.vpp_bin, core_path))
self.logger.error("Running `file %s':" % core_path)
try:
info = check_output(["file", core_path])
self.logger.error(info)
except CalledProcessError as e:
self.logger.error("Could not run `file' utility on core-file, "
"rc=%s" % e.returncode)
pass
def on_crash(self, core_path):
if self.testcase.debug_core:
if not spawn_gdb(self.testcase.vpp_bin, core_path):
self.logger.error(
"Debugger '%s' does not exist or is not an executable.." %
gdb_path)
else:
return
self.logger.critical("Core file present, debug with: gdb %s %s" %
(self.testcase.vpp_bin, core_path))
def run_forked(suite):
keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
result_parent_end, result_child_end = Pipe(duplex=False)
child = Process(target=test_runner_wrapper,
args=(suite, keep_alive_child_end, result_child_end))
child.start()
last_test_temp_dir = None
last_test_vpp_binary = None
last_test = None
result = None
while result is None:
readable = select.select([keep_alive_parent_end.fileno(),
result_parent_end.fileno(),
],
[], [], test_timeout)[0]
if result_parent_end.fileno() in readable:
result = result_parent_end.recv()
elif keep_alive_parent_end.fileno() in readable:
while keep_alive_parent_end.poll():
last_test, last_test_vpp_binary, last_test_temp_dir =\
keep_alive_parent_end.recv()
else:
global_logger.critical("Timeout while waiting for child test "
"runner process (last test running was "
"`%s' in `%s')!" %
(last_test, last_test_temp_dir))
if last_test_temp_dir and last_test_vpp_binary:
core_path = "%s/core" % last_test_temp_dir
if os.path.isfile(core_path):
global_logger.error("Core-file exists in test temporary "
"directory: %s!" % core_path)
if d and d.lower() == "core":
spawn_gdb(last_test_vpp_binary, core_path,
global_logger)
child.terminate()
result = -1
keep_alive_parent_end.close()
result_parent_end.close()
return result
def run_forked(suite):
keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False)
result_parent_end, result_child_end = Pipe(duplex=False)
failed_parent_end, failed_child_end = Pipe(duplex=False)
child = Process(target=test_runner_wrapper,
args=(suite, keep_alive_child_end, result_child_end,
failed_child_end))
child.start()
last_test_temp_dir = None
last_test_vpp_binary = None
last_test = None
result = None
failed = set()
last_heard = time.time()
core_detected_at = None
debug_core = os.getenv("DEBUG", "").lower() == "core"
while True:
readable = select.select([
keep_alive_parent_end.fileno(),
result_parent_end.fileno(),
failed_parent_end.fileno(),
], [], [], 1)[0]
if result_parent_end.fileno() in readable:
result = result_parent_end.recv()
break
if keep_alive_parent_end.fileno() in readable:
while keep_alive_parent_end.poll():
last_test, last_test_vpp_binary,\
last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv()
last_heard = time.time()
if failed_parent_end.fileno() in readable:
while failed_parent_end.poll():
failed_test = failed_parent_end.recv()
failed.add(failed_test.__name__)
last_heard = time.time()
fail = False
if last_heard + test_timeout < time.time() and \
not os.path.isfile("%s/_core_handled" % last_test_temp_dir):
fail = True
global_logger.critical("Timeout while waiting for child test "
"runner process (last test running was "
"`%s' in `%s')!" %
(last_test, last_test_temp_dir))
elif not child.is_alive():
fail = True
global_logger.critical("Child process unexpectedly died (last "
"test running was `%s' in `%s')!" %
(last_test, last_test_temp_dir))
elif last_test_temp_dir and last_test_vpp_binary:
core_path = "%s/core" % last_test_temp_dir
if os.path.isfile(core_path):
if core_detected_at is None:
core_detected_at = time.time()
elif core_detected_at + core_timeout < time.time():
if not os.path.isfile(
"%s/_core_handled" % last_test_temp_dir):
global_logger.critical(
"Child unresponsive and core-file exists in test "
"temporary directory!")
fail = True
if fail:
failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
lttd = last_test_temp_dir.split("/")[-1]
link_path = '%s%s-FAILED' % (failed_dir, lttd)
global_logger.error("Creating a link to the failed " +
"test: %s -> %s" % (link_path, lttd))
try:
os.symlink(last_test_temp_dir, link_path)
except:
pass
api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid
if os.path.isfile(api_post_mortem_path):
global_logger.error("Copying api_post_mortem.%d to %s" %
(vpp_pid, last_test_temp_dir))
shutil.copy2(api_post_mortem_path, last_test_temp_dir)
if last_test_temp_dir and last_test_vpp_binary:
core_path = "%s/core" % last_test_temp_dir
if os.path.isfile(core_path):
global_logger.error("Core-file exists in test temporary "
"directory: %s!" % core_path)
if debug_core:
spawn_gdb(last_test_vpp_binary, core_path,
global_logger)
child.terminate()
result = -1
break
keep_alive_parent_end.close()
result_parent_end.close()
failed_parent_end.close()
return result, failed
def run_forked(testcases):
wrapped_testcase_suites = set()
# suites are unhashable, need to use list
results = []
debug_core = os.getenv("DEBUG", "").lower() == "core"
unread_testcases = set()
finished_unread_testcases = set()
manager = StreamQueueManager()
manager.start()
for i in range(concurrent_tests):
if len(testcases) > 0:
wrapped_testcase_suite = TestCaseWrapper(testcases.pop(0), manager)
wrapped_testcase_suites.add(wrapped_testcase_suite)
unread_testcases.add(wrapped_testcase_suite)
# time.sleep(1)
else:
break
read_from_testcases = threading.Event()
read_from_testcases.set()
stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper,
args=(unread_testcases,
finished_unread_testcases,
read_from_testcases))
stdouterr_thread.start()
while len(wrapped_testcase_suites) > 0:
finished_testcase_suites = set()
for wrapped_testcase_suite in wrapped_testcase_suites:
readable = select.select(
[wrapped_testcase_suite.keep_alive_parent_end.fileno(),
wrapped_testcase_suite.result_parent_end.fileno(),
wrapped_testcase_suite.partial_result_parent_end.fileno()],
[], [], 1)[0]
if wrapped_testcase_suite.result_parent_end.fileno() in readable:
results.append(
(wrapped_testcase_suite.testcase_suite,
wrapped_testcase_suite.result_parent_end.recv()))
finished_testcase_suites.add(wrapped_testcase_suite)
continue
if wrapped_testcase_suite.partial_result_parent_end.fileno() \
in readable:
while wrapped_testcase_suite.partial_result_parent_end.poll():
wrapped_testcase_suite.partial_result = \
wrapped_testcase_suite.partial_result_parent_end.recv()
wrapped_testcase_suite.last_heard = time.time()
if wrapped_testcase_suite.keep_alive_parent_end.fileno() \
in readable:
while wrapped_testcase_suite.keep_alive_parent_end.poll():
wrapped_testcase_suite.last_test, \
wrapped_testcase_suite.last_test_vpp_binary, \
wrapped_testcase_suite.last_test_temp_dir, \
wrapped_testcase_suite.vpp_pid = \
wrapped_testcase_suite.keep_alive_parent_end.recv()
wrapped_testcase_suite.last_heard = time.time()
fail = False
if wrapped_testcase_suite.last_heard + test_timeout < time.time() \
and not os.path.isfile(
"%s/_core_handled" %
wrapped_testcase_suite.last_test_temp_dir):
fail = True
wrapped_testcase_suite.logger.critical(
"Timeout while waiting for child test "
"runner process (last test running was "
"`%s' in `%s')!" %
(wrapped_testcase_suite.last_test,
wrapped_testcase_suite.last_test_temp_dir))
elif not wrapped_testcase_suite.child.is_alive():
fail = True
wrapped_testcase_suite.logger.critical(
"Child python process unexpectedly died "
"(last test running was `%s' in `%s')!" %
(wrapped_testcase_suite.last_test,
wrapped_testcase_suite.last_test_temp_dir))
elif wrapped_testcase_suite.last_test_temp_dir and \
wrapped_testcase_suite.last_test_vpp_binary:
core_path = "%s/core" % \
wrapped_testcase_suite.last_test_temp_dir
if os.path.isfile(core_path):
if wrapped_testcase_suite.core_detected_at is None:
wrapped_testcase_suite.core_detected_at = time.time()
elif wrapped_testcase_suite.core_detected_at + \
core_timeout < time.time():
if not os.path.isfile(
"%s/_core_handled" %
wrapped_testcase_suite.
last_test_temp_dir):
wrapped_testcase_suite.logger.critical(
"Child python process unresponsive and core-"
"file exists in test temporary directory!")
fail = True
if fail:
failed_dir = os.getenv('VPP_TEST_FAILED_DIR')
lttd = os.path.basename(
wrapped_testcase_suite.last_test_temp_dir)
link_path = '%s%s-FAILED' % (failed_dir, lttd)
wrapped_testcase_suite.logger.error(
"Creating a link to the failed test: %s -> %s" %
(link_path, lttd))
if not os.path.exists(link_path):
os.symlink(wrapped_testcase_suite.last_test_temp_dir,
link_path)
api_post_mortem_path = "/tmp/api_post_mortem.%d" % \
wrapped_testcase_suite.vpp_pid
if os.path.isfile(api_post_mortem_path):
wrapped_testcase_suite.logger.error(
"Copying api_post_mortem.%d to %s" %
(wrapped_testcase_suite.vpp_pid,
wrapped_testcase_suite.last_test_temp_dir))
shutil.copy2(api_post_mortem_path,
wrapped_testcase_suite.last_test_temp_dir)
if wrapped_testcase_suite.last_test_temp_dir and \
wrapped_testcase_suite.last_test_vpp_binary:
core_path = "%s/core" % \
wrapped_testcase_suite.last_test_temp_dir
if os.path.isfile(core_path):
wrapped_testcase_suite.logger.error(
"Core-file exists in test temporary directory: %s!"
% core_path)
check_core_path(wrapped_testcase_suite.logger,
core_path)
wrapped_testcase_suite.logger.debug(
"Running `file %s':" % core_path)
try:
info = check_output(["file", core_path])
wrapped_testcase_suite.logger.debug(info)
except CalledProcessError as e:
wrapped_testcase_suite.logger.error(
"Could not run `file' utility on core-file, "
"rc=%s" % e.returncode)
pass
if debug_core:
spawn_gdb(
wrapped_testcase_suite.last_test_vpp_binary,
core_path, wrapped_testcase_suite.logger)
wrapped_testcase_suite.child.terminate()
try:
# terminating the child process tends to leave orphan
# VPP process around
os.kill(wrapped_testcase_suite.vpp_pid, signal.SIGTERM)
except OSError:
# already dead
pass
results.append((wrapped_testcase_suite.testcase_suite,
wrapped_testcase_suite.partial_result))
finished_testcase_suites.add(wrapped_testcase_suite)
for finished_testcase in finished_testcase_suites:
finished_testcase.child.join()
finished_testcase.close_pipes()
wrapped_testcase_suites.remove(finished_testcase)
finished_unread_testcases.add(finished_testcase)
finished_testcase.stdouterr_queue.put(None)
if len(testcases) > 0:
new_testcase = TestCaseWrapper(testcases.pop(0), manager)
wrapped_testcase_suites.add(new_testcase)
unread_testcases.add(new_testcase)
read_from_testcases.clear()
stdouterr_thread.join(test_timeout)
manager.shutdown()
return results
def on_crash(self, core_path):
if self.testcase.debug_core:
spawn_gdb(self.testcase.vpp_bin, core_path, self.logger)
else:
self.logger.critical("Core file present, debug with: gdb %s %s" %
(self.testcase.vpp_bin, core_path))
