def generate_xml_results(result_list): board_results = split_results_by_board(result_list) suite_id = 0 total_failures = 0 total_tests = 0 total_time = 0 root = ElementTree.Element('testsuites', name="pyocd") root.text = "\n" for board_name, results in board_results.items(): total = 0 failures = 0 suite_time = 0 suite = ElementTree.SubElement(root, 'testsuite', name=board_name, id=str(suite_id)) suite.text = "\n" suite.tail = "\n" suite_id += 1 for result in results: total += 1 if not result.passed: failures += 1 case = result.get_test_case() suite.append(case) suite_time += result.time suite.set('tests', str(total)) suite.set('failures', str(failures)) suite.set('time', "%.3f" % suite_time) total_tests += total total_failures += failures total_time += suite_time root.set('tests', str(total_tests)) root.set('failures', str(total_failures)) root.set('time', "%.3f" % total_time) xml_results = os.path.join( TEST_OUTPUT_DIR, XML_RESULTS_TEMPLATE.format(get_env_file_name())) ElementTree.ElementTree(root).write(xml_results, encoding="UTF-8", xml_declaration=True)
def main(): parser = argparse.ArgumentParser(description='pyOCD automated testing') parser.add_argument('-d', '--debug', action="store_true", help='Enable debug logging') parser.add_argument('-q', '--quiet', action="store_true", help='Hide test progress for 1 job') parser.add_argument('-j', '--jobs', action="store", default=1, type=int, metavar="JOBS", help='Set number of concurrent board tests (default is 1)') parser.add_argument('-b', '--board', action="append", metavar="ID", help="Limit testing to boards with specified unique IDs. Multiple boards can be listed.") args = parser.parse_args() # Allow CI to override the number of concurrent jobs. if 'CI_JOBS' in os.environ: args.jobs = int(os.environ['CI_JOBS']) # Disable multiple jobs on macOS prior to Python 3.4. By default, multiprocessing uses # fork() on Unix, which doesn't work on the Mac because CoreFoundation requires exec() # to be used in order to init correctly (CoreFoundation is used in hidapi). Only on Python # version 3.4+ is the multiprocessing.set_start_method() API available that lets us # switch to the 'spawn' method, i.e. exec(). if args.jobs > 1 and sys.platform.startswith('darwin') and sys.version_info[0:2] < (3, 4): print("WARNING: Cannot support multiple jobs on macOS prior to Python 3.4. Forcing 1 job.") args.jobs = 1 # Setup logging based on concurrency and quiet option. level = logging.DEBUG if args.debug else logging.INFO if args.jobs == 1 and not args.quiet: log_file = LOG_FILE_TEMPLATE.format(get_env_file_name()) # Create common log file. if os.path.exists(log_file): os.remove(log_file) logToConsole = True commonLogFile = open(log_file, "a") else: logToConsole = False commonLogFile = None board_list = [] result_list = [] # Put together list of boards to test board_list = ConnectHelper.get_all_connected_probes(blocking=False) board_id_list = sorted(b.unique_id for b in board_list) # Filter boards. if args.board: board_id_list = [b for b in board_id_list if any(c for c in args.board if c.lower() in b.lower())] # If only 1 job was requested, don't bother spawning processes. start = time() if args.jobs == 1: for n, board_id in enumerate(board_id_list): result_list += test_board(board_id, n, level, logToConsole, commonLogFile) else: # Create a pool of processes to run tests. try: pool = mp.Pool(args.jobs) # Issue board test job to process pool. async_results = [pool.apply_async(test_board, (board_id, n, level, logToConsole, commonLogFile)) for n, board_id in enumerate(board_id_list)] # Gather results. for r in async_results: result_list += r.get(timeout=JOB_TIMEOUT) finally: pool.close() pool.join() stop = time() test_time = (stop - start) print_summary(test_list, result_list, test_time) summary_file = SUMMARY_FILE_TEMPLATE.format(get_env_file_name()) with open(summary_file, "w") as output_file: print_summary(test_list, result_list, test_time, output_file) generate_xml_results(result_list) exit_val = 0 if Test.all_tests_pass(result_list) else -1 exit(exit_val)
def test_gdb(board_id=None, n=0): temp_test_elf_name = None result = GdbTestResult() with ConnectHelper.session_with_chosen_probe( unique_id=board_id, **get_session_options()) as session: board = session.board memory_map = board.target.get_memory_map() ram_region = memory_map.get_default_region_of_type(MemoryType.RAM) rom_region = memory_map.get_boot_memory() target_type = board.target_type binary_file = get_test_binary_path(board.test_binary) if board_id is None: board_id = board.unique_id target_test_params = get_target_test_params(session) test_port = 3333 + n telnet_port = 4444 + n # Hardware breakpoints are not supported above 0x20000000 on # Cortex-M devices with FPB revision 1. fpb = session.target.selected_core.fpb assert fpb is not None ignore_hw_bkpt_result = int(fpb.revision == 1 and ram_region.start >= 0x20000000) # Program with initial test image FileProgrammer(session).program(binary_file, base_address=rom_region.start) # Generate an elf from the binary test file. temp_test_elf_name = binary_to_elf_file(binary_file, rom_region.start) # Write out the test configuration test_params = { "test_port": test_port, "rom_start": rom_region.start, "rom_length": rom_region.length, "ram_start": ram_region.start, "ram_length": ram_region.length, "invalid_start": 0x3E000000, "invalid_length": 0x1000, "expect_error_on_invalid_access": target_test_params['error_on_invalid_access'], "ignore_hw_bkpt_result": ignore_hw_bkpt_result, "test_elf": temp_test_elf_name, } test_param_filename = os.path.join( TEST_OUTPUT_DIR, "gdb_test_params%s_%d.txt" % (get_env_file_name(), n)) with open(test_param_filename, "w") as f: f.write(json.dumps(test_params)) # Remove result from previous run. test_result_filename = os.path.join( TEST_OUTPUT_DIR, "gdb_test_results%s_%d.txt" % (get_env_file_name(), n)) if os.path.exists(test_result_filename): os.remove(test_result_filename) # Run the test gdb_args = [ PYTHON_GDB, "--nh", "-ex", "set $testn=%d" % n, "--command=%s" % GDB_SCRIPT_PATH ] gdb_output_filename = os.path.join( TEST_OUTPUT_DIR, "gdb_output%s_%s_%d.txt" % (get_env_file_name(), board.target_type, n)) with open(gdb_output_filename, "w") as f: LOG.info('Starting gdb (stdout -> %s): %s', gdb_output_filename, ' '.join(gdb_args)) gdb_program = Popen(gdb_args, stdin=PIPE, stdout=f, stderr=STDOUT) server_args = [ 'gdbserver', '--port=%i' % test_port, "--telnet-port=%i" % telnet_port, "--frequency=%i" % target_test_params['test_clock'], "--uid=%s" % board_id, ] server = PyOCDTool() LOG.info('Starting gdbserver: %s', ' '.join(server_args)) server_thread = threading.Thread(target=server.run, args=[server_args]) server_thread.daemon = True server_thread.start() LOG.info('Waiting for gdb to finish...') did_complete = wait_with_deadline(gdb_program, TEST_TIMEOUT_SECONDS) LOG.info('Waiting for server to finish...') server_thread.join(timeout=TEST_TIMEOUT_SECONDS) if not did_complete: LOG.error("Test timed out!") if server_thread.is_alive(): LOG.error('Server is still running!') try: with open(gdb_output_filename, 'r') as f: LOG.debug('Gdb output:\n%s', f.read()) except IOError: pass # Read back the result result.passed = False if did_complete: try: with open(test_result_filename, "r") as f: test_result = json.loads(f.read()) # Print results if set(TEST_RESULT_KEYS).issubset(test_result): print("----------------Test Results----------------") print("HW breakpoint count: %s" % test_result["breakpoint_count"]) print("Watchpoint count: %s" % test_result["watchpoint_count"]) print("Average instruction step time: %s" % test_result["step_time_si"]) print("Average single step time: %s" % test_result["step_time_s"]) print("Average over step time: %s" % test_result["step_time_n"]) print("Failure count: %i" % test_result["fail_count"]) result.passed = test_result["fail_count"] == 0 except IOError as err: LOG.error("Error reading test results: %s", err, exc_info=True) if result.passed: print("GDB TEST PASSED") else: print("GDB TEST FAILED") # Cleanup try: if temp_test_elf_name and os.path.exists(temp_test_elf_name): os.remove(temp_test_elf_name) os.remove(test_result_filename) os.remove(test_param_filename) except IOError as err: pass return result
def test_gdb(board_id=None, n=0): temp_test_elf_name = None result = GdbTestResult() with ConnectHelper.session_with_chosen_probe(unique_id=board_id, **get_session_options()) as session: board = session.board memory_map = board.target.get_memory_map() ram_region = memory_map.get_default_region_of_type(MemoryType.RAM) rom_region = memory_map.get_boot_memory() target_type = board.target_type binary_file = os.path.join(parentdir, 'binaries', board.test_binary) if board_id is None: board_id = board.unique_id target_test_params = get_target_test_params(session) test_port = 3333 + n telnet_port = 4444 + n # Hardware breakpoints are not supported above 0x20000000 on # CortexM devices ignore_hw_bkpt_result = 1 if ram_region.start >= 0x20000000 else 0 # Program with initial test image FileProgrammer(session).program(binary_file, base_address=rom_region.start) # Generate an elf from the binary test file. temp_test_elf_name = binary_to_elf_file(binary_file, rom_region.start) # Write out the test configuration test_params = { "test_port" : test_port, "rom_start" : rom_region.start, "rom_length" : rom_region.length, "ram_start" : ram_region.start, "ram_length" : ram_region.length, "invalid_start" : 0x3E000000, "invalid_length" : 0x1000, "expect_error_on_invalid_access" : target_test_params['error_on_invalid_access'], "ignore_hw_bkpt_result" : ignore_hw_bkpt_result, "test_elf" : temp_test_elf_name, } test_param_filename = "gdb_test_params%s_%d.txt" % (get_env_file_name(), n) with open(test_param_filename, "w") as f: f.write(json.dumps(test_params)) # Run the test gdb = [PYTHON_GDB, "--nh", "-ex", "set $testn=%d" % n, "--command=gdb_script.py"] output_filename = "gdb_output%s_%s_%d.txt" % (get_env_file_name(), board.target_type, n) with open(output_filename, "w") as f: program = Popen(gdb, stdin=PIPE, stdout=f, stderr=STDOUT) args = ['gdbserver', '--port=%i' % test_port, "--telnet-port=%i" % telnet_port, "--frequency=%i" % target_test_params['test_clock'], "--uid=%s" % board_id, ] server = PyOCDTool() server.run(args) program.wait() # Read back the result test_result_filename = "gdb_test_results%s_%d.txt" % (get_env_file_name(), n) with open(test_result_filename, "r") as f: test_result = json.loads(f.read()) # Print results if set(TEST_RESULT_KEYS).issubset(test_result): print("----------------Test Results----------------") print("HW breakpoint count: %s" % test_result["breakpoint_count"]) print("Watchpoint count: %s" % test_result["watchpoint_count"]) print("Average instruction step time: %s" % test_result["step_time_si"]) print("Average single step time: %s" % test_result["step_time_s"]) print("Average over step time: %s" % test_result["step_time_n"]) print("Failure count: %i" % test_result["fail_count"]) result.passed = test_result["fail_count"] == 0 else: result.passed = False # Cleanup if temp_test_elf_name and os.path.exists(temp_test_elf_name): os.remove(temp_test_elf_name) os.remove(test_result_filename) os.remove(test_param_filename) return result