def check_and_handle_core(vpp_binary, tempdir, core_crash_test): if is_core_present(tempdir): print('VPP core detected in %s. Last test running was %s' % (tempdir, core_crash_test)) print(single_line_delim) spawn_gdb(vpp_binary, get_core_path(tempdir)) print(single_line_delim)
def on_crash(self, core_path): if self.testcase.debug_core: # notify parent process that we're handling a core file open('%s/_core_handled' % self.testcase.tempdir, 'a').close() spawn_gdb(self.testcase.vpp_bin, core_path, self.logger) else: self.logger.critical("Core file present, debug with: gdb %s %s" % (self.testcase.vpp_bin, core_path))
def check_and_handle_core(vpp_binary, tempdir, core_crash_test): if is_core_present(tempdir): if debug_core: print('VPP core detected in %s. Last test running was %s' % (tempdir, core_crash_test)) print(single_line_delim) spawn_gdb(vpp_binary, get_core_path(tempdir)) print(single_line_delim) elif compress_core: print("Compressing core-file in test directory `%s'" % tempdir) os.system("gzip %s" % get_core_path(tempdir))
def run_forked(suite): keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False) result_parent_end, result_child_end = Pipe(duplex=False) child = Process(target=test_runner_wrapper, args=(suite, keep_alive_child_end, result_child_end)) child.start() last_test_temp_dir = None last_test_vpp_binary = None last_test = None result = None while result is None: readable = select.select([ keep_alive_parent_end.fileno(), result_parent_end.fileno(), ], [], [], test_timeout)[0] if result_parent_end.fileno() in readable: result = result_parent_end.recv() elif keep_alive_parent_end.fileno() in readable: while keep_alive_parent_end.poll(): last_test, last_test_vpp_binary,\ last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv() else: global_logger.critical("Timeout while waiting for child test " "runner process (last test running was " "`%s' in `%s')!" % (last_test, last_test_temp_dir)) failed_dir = os.getenv('VPP_TEST_FAILED_DIR') lttd = last_test_temp_dir.split("/")[-1] link_path = '%s%s-FAILED' % (failed_dir, lttd) global_logger.error("Creating a link to the failed " + "test: %s -> %s" % (link_path, lttd)) os.symlink(last_test_temp_dir, link_path) api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid if os.path.isfile(api_post_mortem_path): global_logger.error("Copying api_post_mortem.%d to %s" % (vpp_pid, last_test_temp_dir)) shutil.copy2(api_post_mortem_path, last_test_temp_dir) if last_test_temp_dir and last_test_vpp_binary: core_path = "%s/core" % last_test_temp_dir if os.path.isfile(core_path): global_logger.error("Core-file exists in test temporary " "directory: %s!" % core_path) if d and d.lower() == "core": spawn_gdb(last_test_vpp_binary, core_path, global_logger) child.terminate() result = -1 keep_alive_parent_end.close() result_parent_end.close() return result
def on_crash(self, core_path): if self.testcase.debug_core: # notify parent process that we're handling a core file open('%s/_core_handled' % self.testcase.tempdir, 'a').close() spawn_gdb(self.testcase.vpp_bin, core_path, self.logger) else: self.logger.error("Core file present, debug with: gdb %s %s" % (self.testcase.vpp_bin, core_path)) self.logger.error("Running `file %s':" % core_path) try: info = check_output(["file", core_path]) self.logger.error(info) except CalledProcessError as e: self.logger.error("Could not run `file' utility on core-file, " "rc=%s" % e.returncode) pass
def on_crash(self, core_path): if self.testcase.debug_core: if not spawn_gdb(self.testcase.vpp_bin, core_path): self.logger.error( "Debugger '%s' does not exist or is not an executable.." % gdb_path) else: return self.logger.critical("Core file present, debug with: gdb %s %s" % (self.testcase.vpp_bin, core_path))
def run_forked(suite): keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False) result_parent_end, result_child_end = Pipe(duplex=False) child = Process(target=test_runner_wrapper, args=(suite, keep_alive_child_end, result_child_end)) child.start() last_test_temp_dir = None last_test_vpp_binary = None last_test = None result = None while result is None: readable = select.select([keep_alive_parent_end.fileno(), result_parent_end.fileno(), ], [], [], test_timeout)[0] if result_parent_end.fileno() in readable: result = result_parent_end.recv() elif keep_alive_parent_end.fileno() in readable: while keep_alive_parent_end.poll(): last_test, last_test_vpp_binary, last_test_temp_dir =\ keep_alive_parent_end.recv() else: global_logger.critical("Timeout while waiting for child test " "runner process (last test running was " "`%s' in `%s')!" % (last_test, last_test_temp_dir)) if last_test_temp_dir and last_test_vpp_binary: core_path = "%s/core" % last_test_temp_dir if os.path.isfile(core_path): global_logger.error("Core-file exists in test temporary " "directory: %s!" % core_path) if d and d.lower() == "core": spawn_gdb(last_test_vpp_binary, core_path, global_logger) child.terminate() result = -1 keep_alive_parent_end.close() result_parent_end.close() return result
def run_forked(suite): keep_alive_parent_end, keep_alive_child_end = Pipe(duplex=False) result_parent_end, result_child_end = Pipe(duplex=False) failed_parent_end, failed_child_end = Pipe(duplex=False) child = Process(target=test_runner_wrapper, args=(suite, keep_alive_child_end, result_child_end, failed_child_end)) child.start() last_test_temp_dir = None last_test_vpp_binary = None last_test = None result = None failed = set() last_heard = time.time() core_detected_at = None debug_core = os.getenv("DEBUG", "").lower() == "core" while True: readable = select.select([ keep_alive_parent_end.fileno(), result_parent_end.fileno(), failed_parent_end.fileno(), ], [], [], 1)[0] if result_parent_end.fileno() in readable: result = result_parent_end.recv() break if keep_alive_parent_end.fileno() in readable: while keep_alive_parent_end.poll(): last_test, last_test_vpp_binary,\ last_test_temp_dir, vpp_pid = keep_alive_parent_end.recv() last_heard = time.time() if failed_parent_end.fileno() in readable: while failed_parent_end.poll(): failed_test = failed_parent_end.recv() failed.add(failed_test.__name__) last_heard = time.time() fail = False if last_heard + test_timeout < time.time() and \ not os.path.isfile("%s/_core_handled" % last_test_temp_dir): fail = True global_logger.critical("Timeout while waiting for child test " "runner process (last test running was " "`%s' in `%s')!" % (last_test, last_test_temp_dir)) elif not child.is_alive(): fail = True global_logger.critical("Child process unexpectedly died (last " "test running was `%s' in `%s')!" % (last_test, last_test_temp_dir)) elif last_test_temp_dir and last_test_vpp_binary: core_path = "%s/core" % last_test_temp_dir if os.path.isfile(core_path): if core_detected_at is None: core_detected_at = time.time() elif core_detected_at + core_timeout < time.time(): if not os.path.isfile( "%s/_core_handled" % last_test_temp_dir): global_logger.critical( "Child unresponsive and core-file exists in test " "temporary directory!") fail = True if fail: failed_dir = os.getenv('VPP_TEST_FAILED_DIR') lttd = last_test_temp_dir.split("/")[-1] link_path = '%s%s-FAILED' % (failed_dir, lttd) global_logger.error("Creating a link to the failed " + "test: %s -> %s" % (link_path, lttd)) try: os.symlink(last_test_temp_dir, link_path) except: pass api_post_mortem_path = "/tmp/api_post_mortem.%d" % vpp_pid if os.path.isfile(api_post_mortem_path): global_logger.error("Copying api_post_mortem.%d to %s" % (vpp_pid, last_test_temp_dir)) shutil.copy2(api_post_mortem_path, last_test_temp_dir) if last_test_temp_dir and last_test_vpp_binary: core_path = "%s/core" % last_test_temp_dir if os.path.isfile(core_path): global_logger.error("Core-file exists in test temporary " "directory: %s!" % core_path) if debug_core: spawn_gdb(last_test_vpp_binary, core_path, global_logger) child.terminate() result = -1 break keep_alive_parent_end.close() result_parent_end.close() failed_parent_end.close() return result, failed
def run_forked(testcases): wrapped_testcase_suites = set() # suites are unhashable, need to use list results = [] debug_core = os.getenv("DEBUG", "").lower() == "core" unread_testcases = set() finished_unread_testcases = set() manager = StreamQueueManager() manager.start() for i in range(concurrent_tests): if len(testcases) > 0: wrapped_testcase_suite = TestCaseWrapper(testcases.pop(0), manager) wrapped_testcase_suites.add(wrapped_testcase_suite) unread_testcases.add(wrapped_testcase_suite) # time.sleep(1) else: break read_from_testcases = threading.Event() read_from_testcases.set() stdouterr_thread = threading.Thread(target=stdouterr_reader_wrapper, args=(unread_testcases, finished_unread_testcases, read_from_testcases)) stdouterr_thread.start() while len(wrapped_testcase_suites) > 0: finished_testcase_suites = set() for wrapped_testcase_suite in wrapped_testcase_suites: readable = select.select( [wrapped_testcase_suite.keep_alive_parent_end.fileno(), wrapped_testcase_suite.result_parent_end.fileno(), wrapped_testcase_suite.partial_result_parent_end.fileno()], [], [], 1)[0] if wrapped_testcase_suite.result_parent_end.fileno() in readable: results.append( (wrapped_testcase_suite.testcase_suite, wrapped_testcase_suite.result_parent_end.recv())) finished_testcase_suites.add(wrapped_testcase_suite) continue if wrapped_testcase_suite.partial_result_parent_end.fileno() \ in readable: while wrapped_testcase_suite.partial_result_parent_end.poll(): wrapped_testcase_suite.partial_result = \ wrapped_testcase_suite.partial_result_parent_end.recv() wrapped_testcase_suite.last_heard = time.time() if wrapped_testcase_suite.keep_alive_parent_end.fileno() \ in readable: while wrapped_testcase_suite.keep_alive_parent_end.poll(): wrapped_testcase_suite.last_test, \ wrapped_testcase_suite.last_test_vpp_binary, \ wrapped_testcase_suite.last_test_temp_dir, \ wrapped_testcase_suite.vpp_pid = \ wrapped_testcase_suite.keep_alive_parent_end.recv() wrapped_testcase_suite.last_heard = time.time() fail = False if wrapped_testcase_suite.last_heard + test_timeout < time.time() \ and not os.path.isfile( "%s/_core_handled" % wrapped_testcase_suite.last_test_temp_dir): fail = True wrapped_testcase_suite.logger.critical( "Timeout while waiting for child test " "runner process (last test running was " "`%s' in `%s')!" % (wrapped_testcase_suite.last_test, wrapped_testcase_suite.last_test_temp_dir)) elif not wrapped_testcase_suite.child.is_alive(): fail = True wrapped_testcase_suite.logger.critical( "Child python process unexpectedly died " "(last test running was `%s' in `%s')!" % (wrapped_testcase_suite.last_test, wrapped_testcase_suite.last_test_temp_dir)) elif wrapped_testcase_suite.last_test_temp_dir and \ wrapped_testcase_suite.last_test_vpp_binary: core_path = "%s/core" % \ wrapped_testcase_suite.last_test_temp_dir if os.path.isfile(core_path): if wrapped_testcase_suite.core_detected_at is None: wrapped_testcase_suite.core_detected_at = time.time() elif wrapped_testcase_suite.core_detected_at + \ core_timeout < time.time(): if not os.path.isfile( "%s/_core_handled" % wrapped_testcase_suite. last_test_temp_dir): wrapped_testcase_suite.logger.critical( "Child python process unresponsive and core-" "file exists in test temporary directory!") fail = True if fail: failed_dir = os.getenv('VPP_TEST_FAILED_DIR') lttd = os.path.basename( wrapped_testcase_suite.last_test_temp_dir) link_path = '%s%s-FAILED' % (failed_dir, lttd) wrapped_testcase_suite.logger.error( "Creating a link to the failed test: %s -> %s" % (link_path, lttd)) if not os.path.exists(link_path): os.symlink(wrapped_testcase_suite.last_test_temp_dir, link_path) api_post_mortem_path = "/tmp/api_post_mortem.%d" % \ wrapped_testcase_suite.vpp_pid if os.path.isfile(api_post_mortem_path): wrapped_testcase_suite.logger.error( "Copying api_post_mortem.%d to %s" % (wrapped_testcase_suite.vpp_pid, wrapped_testcase_suite.last_test_temp_dir)) shutil.copy2(api_post_mortem_path, wrapped_testcase_suite.last_test_temp_dir) if wrapped_testcase_suite.last_test_temp_dir and \ wrapped_testcase_suite.last_test_vpp_binary: core_path = "%s/core" % \ wrapped_testcase_suite.last_test_temp_dir if os.path.isfile(core_path): wrapped_testcase_suite.logger.error( "Core-file exists in test temporary directory: %s!" % core_path) check_core_path(wrapped_testcase_suite.logger, core_path) wrapped_testcase_suite.logger.debug( "Running `file %s':" % core_path) try: info = check_output(["file", core_path]) wrapped_testcase_suite.logger.debug(info) except CalledProcessError as e: wrapped_testcase_suite.logger.error( "Could not run `file' utility on core-file, " "rc=%s" % e.returncode) pass if debug_core: spawn_gdb( wrapped_testcase_suite.last_test_vpp_binary, core_path, wrapped_testcase_suite.logger) wrapped_testcase_suite.child.terminate() try: # terminating the child process tends to leave orphan # VPP process around os.kill(wrapped_testcase_suite.vpp_pid, signal.SIGTERM) except OSError: # already dead pass results.append((wrapped_testcase_suite.testcase_suite, wrapped_testcase_suite.partial_result)) finished_testcase_suites.add(wrapped_testcase_suite) for finished_testcase in finished_testcase_suites: finished_testcase.child.join() finished_testcase.close_pipes() wrapped_testcase_suites.remove(finished_testcase) finished_unread_testcases.add(finished_testcase) finished_testcase.stdouterr_queue.put(None) if len(testcases) > 0: new_testcase = TestCaseWrapper(testcases.pop(0), manager) wrapped_testcase_suites.add(new_testcase) unread_testcases.add(new_testcase) read_from_testcases.clear() stdouterr_thread.join(test_timeout) manager.shutdown() return results
def on_crash(self, core_path): if self.testcase.debug_core: spawn_gdb(self.testcase.vpp_bin, core_path, self.logger) else: self.logger.critical("Core file present, debug with: gdb %s %s" % (self.testcase.vpp_bin, core_path))