def main(): board_uart_reader = None if args.serial_port: try: board_uart_reader = SerialPortReader(args.serial_port) except serial.SerialException as e: sys.stderr.write('Could not start reader for serial port {}: {}\n'.format(args.serial_port, e)) sys.exit(1) log_formatter = logging.Formatter('%(asctime)-15s %(name)s: %(levelname)s - %(message)s') ch = logging.StreamHandler() ch.setFormatter(log_formatter) ch.setLevel(logging.WARNING) fh = None log_lev = logging.WARNING if args.log_file: if args.log_file == 'stdout': fh = ch else: fh = logging.FileHandler(args.log_file, 'w') fh.setFormatter(log_formatter) if args.debug == 0: log_lev = logging.CRITICAL elif args.debug == 1: log_lev = logging.ERROR elif args.debug == 2: log_lev = logging.WARNING elif args.debug == 3: log_lev = logging.INFO else: log_lev = logging.DEBUG setup_logger(debug_backend_tests.get_logger(), ch, fh, log_lev) if board_uart_reader: setup_logger(board_uart_reader.get_logger(), ch, fh, log_lev) board_uart_reader.start() board_tcl = BOARD_TCL_CONFIG[args.board_type] debug_backend_tests.test_apps_dir = args.apps_dir if args.idf_ver_min != 'auto': debug_backend_tests.IdfVersion.set_current(debug_backend_tests.IdfVersion.fromstr(args.idf_ver_min)) if args.test_runner == 'x': test_runner = xmlrunner.XMLTestRunner(verbosity=2, output=args.test_outdir) elif args.test_runner == 't': test_runner = unittest.TextTestRunner(verbosity=2) else: print("Wrong test-runner argument") return # start debugger, ideally we should run all tests w/o restarting it dbg_start(args.toolchain, args.oocd, args.oocd_tcl, board_tcl['files'], board_tcl['commands'], args.debug_oocd, board_tcl['target_name'], log_lev, ch, fh) res = None try: # run tests from the same directory this file is loader = unittest.TestLoader() loader.suiteClass = debug_backend_tests.DebuggerTestsBunch if not isinstance(args.pattern, list): tests_patterns = [args.pattern, ] else: tests_patterns = args.pattern suite = None for pattern in tests_patterns: pattern_suite = load_tests_by_pattern(loader, os.path.dirname(__file__), pattern) if suite: suite.addTest(pattern_suite) else: suite = pattern_suite # setup loggers in test modules for m in suite.modules: setup_logger(suite.modules[m].get_logger(), ch, fh, log_lev) suite.load_app_bins = not args.no_load global _oocd_inst, _gdb_inst suite.config_tests(_oocd_inst, _gdb_inst, args.toolchain) # RUN TESTS res = test_runner.run(suite) if not res.wasSuccessful() and args.retry: print("===========================================") print("Re-run failed tests. Give a second chance.") print("===========================================") # restart debugger dbg_stop() dbg_start(args.toolchain, args.oocd, args.oocd_tcl, board_tcl['files'], board_tcl['commands'], args.debug_oocd, board_tcl['target_name'], log_lev, ch, fh) err_suite = debug_backend_tests.DebuggerTestsBunch() for e in res.errors: err_suite.addTest(e[0]) for f in res.failures: err_suite.addTest(f[0]) res = test_runner.run(err_suite) except: traceback.print_exc() finally: # stop debugger dbg_stop() if board_uart_reader: board_uart_reader.stop() # check results if not res or not res.wasSuccessful(): sys.exit(-1)
def main(): board_uart_reader = None if args.serial_port: try: board_uart_reader = SerialPortReader(args.serial_port) except serial.SerialException as e: sys.stderr.write( 'Could not start reader for serial port {}: {}\n'.format( args.serial_port, e)) sys.exit(1) log_formatter = logging.Formatter( '%(asctime)-15s %(name)s: %(levelname)s - %(message)s') ch = logging.StreamHandler() ch.setFormatter(log_formatter) ch.setLevel(logging.WARNING) fh = None log_lev = logging.WARNING if args.log_file: if args.log_file == 'stdout': fh = ch else: fh = logging.FileHandler(args.log_file, 'w') fh.setFormatter(log_formatter) if args.debug == 0: log_lev = logging.CRITICAL elif args.debug == 1: log_lev = logging.ERROR elif args.debug == 2: log_lev = logging.WARNING elif args.debug == 3: log_lev = logging.INFO else: log_lev = logging.DEBUG # config log levels in modules def setup_logger(logger, conh, fileh, lev): logger.setLevel(lev) fileh.setLevel(lev) logger.addHandler(conh) if fileh: logger.addHandler(fileh) logger.propagate = False setup_logger(debug_backend.Oocd.get_logger(), ch, fh, log_lev) setup_logger(debug_backend.Gdb.get_logger(), ch, fh, log_lev) setup_logger(debug_backend_tests.get_logger(), ch, fh, log_lev) if board_uart_reader: setup_logger(board_uart_reader.get_logger(), ch, fh, log_lev) board_uart_reader.start() board_tcl = BOARD_TCL_CONFIG[args.board_type] if args.idf_ver_min != 'auto': debug_backend_tests.IdfVersion.set_current( debug_backend_tests.IdfVersion.fromstr(args.idf_ver_min)) # start debugger, ideally we should run all tests w/o restarting it debug_backend.start(args.toolchain, args.oocd, args.oocd_tcl, board_tcl['files'], board_tcl['commands'], args.debug_oocd, target_name=board_tcl['target_name']) debug_backend_tests.test_apps_dir = args.apps_dir # loads tests using pattern <module>[.<test_case>[.<test_method>]] with wildcards (*) in <module> and <test_case> parts if args.test_runner == 'x': test_runner = xmlrunner.XMLTestRunner(verbosity=2, output=args.test_outdir) elif args.test_runner == 't': test_runner = unittest.TextTestRunner(verbosity=2) else: debug_backend_tests.get_logger().error('Wrong test-runner argument') return def load_tests_by_pattern(search_dir, pattern): if pattern.find('*') == -1: return loader.loadTestsFromName(pattern) parts = pattern.split('.') if len(parts) == 1: # '*' or 'te*' or test_*' etc return loader.discover(search_dir, '%s.py' % parts[0]) if parts[0].find('*') == -1: # if module part w/o wildcard # 'xxx.??*' mod_tests = loader.loadTestsFromName(parts[0]) else: # if module part has wildcard # 'xx*.???' mod_tests = loader.discover(search_dir, '%s.py' % parts[0]) if parts[1].find('*') != -1: case_name_pattern = '^%s$' % parts[1].replace('*', '.*') else: case_name_pattern = '^%s$' % parts[1] suite = debug_backend_tests.DebuggerTestsBunch() # look for concrete matched test cases for test in mod_tests: if not issubclass(type(test), debug_backend_tests.DebuggerTestsBase): continue if re.match(case_name_pattern, test.__class__.__name__): if len(parts) == 2 or parts[2] == '*': suite.addTest(test) elif hasattr(test, parts[2]): test_method = getattr(test, parts[2]) test1 = type(test)(test_method.__name__) suite.addTest( debug_backend_tests.DebuggerTestsBunch([test1])) return suite # run tests from the same directory this file is loader = unittest.TestLoader() loader.suiteClass = debug_backend_tests.DebuggerTestsBunch if not isinstance(args.pattern, list): tests_patterns = [ args.pattern, ] else: tests_patterns = args.pattern suite = None for pattern in tests_patterns: pattern_suite = load_tests_by_pattern(os.path.dirname(__file__), pattern) if suite: suite.addTest(pattern_suite) else: suite = pattern_suite # setup loggers in test modules for m in suite.modules: setup_logger(suite.modules[m].get_logger(), ch, fh, log_lev) suite.load_app_bins = not args.no_load try: res = test_runner.run(suite) if not res.wasSuccessful() and args.retry: print '===========================================' print 'Re-run failed tests. Give a second chance.' print '===========================================' # restart debugger debug_backend.stop() debug_backend.start(args.toolchain, args.oocd, args.oocd_tcl, board_tcl['files'], board_tcl['commands']) err_suite = debug_backend_tests.DebuggerTestsBunch() for e in res.errors: err_suite.addTest(e[0]) for f in res.failures: err_suite.addTest(f[0]) res = test_runner.run(err_suite) except: traceback.print_exc() finally: # stop debugger debug_backend.stop() if board_uart_reader: board_uart_reader.stop() # check results if not res or not res.wasSuccessful(): sys.exit(-1)
def main(): log_formatter = logging.Formatter( '%(asctime)-15s %(name)s: %(levelname)s - %(message)s') ch = logging.StreamHandler() ch.setFormatter(log_formatter) ch.setLevel(logging.WARNING) fh = None if args.log_file: fh = logging.FileHandler(args.log_file, 'w') fh.setFormatter(log_formatter) if args.debug == 0: fh.setLevel(logging.CRITICAL) elif args.debug == 1: fh.setLevel(logging.ERROR) elif args.debug == 2: fh.setLevel(logging.WARNING) elif args.debug == 3: fh.setLevel(logging.INFO) else: fh.setLevel(logging.DEBUG) # config log levels in modules def setup_logger(logger, conh, fileh): logger.setLevel(logging.DEBUG) logger.addHandler(conh) if fileh: logger.addHandler(fileh) setup_logger(debug_backend.Oocd.get_logger(), ch, fh) setup_logger(debug_backend.Gdb.get_logger(), ch, fh) setup_logger(debug_backend_tests.get_logger(), ch, fh) # start debugger, ideally we should run all tests w/o restarting it debug_backend.start(args.toolchain, args.oocd, args.oocd_tcl, BOARD_TCL_FILES[args.board_type]) debug_backend_tests.test_apps_dir = args.apps_dir # run tests from the same directory this file is loader = unittest.TestLoader() loader.suiteClass = debug_backend_tests.DebuggerTestsBunch if isinstance(args.pattern, list): suite = loader.loadTestsFromNames(args.pattern) elif args.pattern != '*': suite = loader.loadTestsFromName(args.pattern) else: suite = loader.discover(os.path.dirname(__file__)) # setup loggers in test modules for m in suite.modules: setup_logger(suite.modules[m].get_logger(), ch, fh) suite.load_app_bins = not args.no_load try: res = unittest.TextTestRunner(verbosity=2).run(suite) if not res.wasSuccessful() and args.retry: print '===========================================' print 'Re-run failed tests. Give a second chance.' print '===========================================' # restart debugger debug_backend.stop() debug_backend.start(args.toolchain, args.oocd, args.oocd_tcl, BOARD_TCL_FILES[args.board_type]) err_suite = debug_backend_tests.DebuggerTestsBunch() for e in res.errors: err_suite.addTest(e[0]) for f in res.failures: err_suite.addTest(f[0]) res = unittest.TextTestRunner(verbosity=2).run(err_suite) finally: # stop debugger debug_backend.stop() # check results if not res.wasSuccessful(): sys.exit(-1)
def main(): parser = argparse.ArgumentParser( description='run_tests.py - Run auto-tests', prog='run_tests') parser.add_argument('--gdb', '-g', help='Path to GDB binary', default=os.environ.get('OOCD_TEST_GDB_BIN_PATH', 'xtensa-esp32-elf-gdb')) parser.add_argument('--oocd', '-o', help='Path to OpenOCD binary', default=os.environ.get( 'OOCD_TEST_BIN_PATH', os.path.join(os.getcwd(), 'src', 'openocd'))) parser.add_argument('--oocd-tcl', '-s', help='Path to OpenOCD TCL scripts', default=os.environ.get( 'OOCD_TEST_TCL_DIR', os.path.join(os.getcwd(), 'tcl'))) parser.add_argument('--board-type', '-b', help='Type of the board to run tests on', choices=['esp-wrover-kit'], default=os.environ.get('OOCD_TEST_BOARD', 'esp-wrover-kit')) parser.add_argument('--apps-dir', '-a', help='Path to test apps', default=os.environ.get( 'OOCD_TEST_APPS_DIR', os.path.join(os.getcwd(), 'testing', 'esp', 'test_apps'))) parser.add_argument( '--pattern', '-p', nargs='*', help= 'Pattern of test cases to run. Format: <module>.<test_case>.<test_method>. Several strings can be specified.', default='*') parser.add_argument('--debug', '-d', help='Debug level (0-4)', type=int, default=2) parser.add_argument('--log-file', '-l', help='Path to log file') args = parser.parse_args() log_formatter = logging.Formatter('%(name)s: %(levelname)s - %(message)s') ch = logging.StreamHandler() ch.setFormatter(log_formatter) ch.setLevel(logging.WARNING) fh = None if args.log_file: fh = logging.FileHandler(args.log_file) fh.setFormatter(log_formatter) if args.debug == 0: fh.setLevel(logging.CRITICAL) elif args.debug == 1: fh.setLevel(logging.ERROR) elif args.debug == 2: fh.setLevel(logging.WARNING) elif args.debug == 3: fh.setLevel(logging.INFO) else: fh.setLevel(logging.DEBUG) # config log levels in modules def setup_logger(logger, conh, fileh): logger.setLevel(logging.DEBUG) logger.addHandler(conh) if fileh: logger.addHandler(fileh) setup_logger(debug_backend.Oocd.get_logger(), ch, fh) setup_logger(debug_backend.Gdb.get_logger(), ch, fh) setup_logger(debug_backend_tests.get_logger(), ch, fh) # start debugger, ideally we should run all tests w/o restarting it debug_backend.start(args.gdb, args.oocd, args.oocd_tcl, BOARD_TCL_FILES[args.board_type]) debug_backend_tests.set_apps_dir(args.apps_dir) # run tests from the same directory this file is loader = unittest.TestLoader() loader.suiteClass = debug_backend_tests.DebuggerTestsBunch if isinstance(args.pattern, list): suite = loader.loadTestsFromNames(args.pattern) elif args.pattern != '*': suite = loader.loadTestsFromName(args.pattern) else: suite = loader.discover(os.path.dirname(__file__)) # setup loggers in test modules for m in suite.modules: setup_logger(suite.modules[m].get_logger(), ch, fh) try: res = unittest.TextTestRunner(verbosity=2).run(suite) finally: # stop debugger debug_backend.stop() # check results if len(res.errors) or len(res.failures): sys.exit(-1)