def generate_xml_results(result_list):
board_results = split_results_by_board(result_list)
suite_id = 0
total_failures = 0
total_tests = 0
total_time = 0
root = ElementTree.Element('testsuites', name="pyocd")
root.text = "\n"
for board_name, results in board_results.items():
total = 0
failures = 0
suite_time = 0
suite = ElementTree.SubElement(root,
'testsuite',
name=board_name,
id=str(suite_id))
suite.text = "\n"
suite.tail = "\n"
suite_id += 1
for result in results:
total += 1
if not result.passed:
failures += 1
case = result.get_test_case()
suite.append(case)
suite_time += result.time
suite.set('tests', str(total))
suite.set('failures', str(failures))
suite.set('time', "%.3f" % suite_time)
total_tests += total
total_failures += failures
total_time += suite_time
root.set('tests', str(total_tests))
root.set('failures', str(total_failures))
root.set('time', "%.3f" % total_time)
xml_results = os.path.join(
TEST_OUTPUT_DIR, XML_RESULTS_TEMPLATE.format(get_env_file_name()))
ElementTree.ElementTree(root).write(xml_results,
encoding="UTF-8",
xml_declaration=True)
def main():
parser = argparse.ArgumentParser(description='pyOCD automated testing')
parser.add_argument('-d', '--debug', action="store_true", help='Enable debug logging')
parser.add_argument('-q', '--quiet', action="store_true", help='Hide test progress for 1 job')
parser.add_argument('-j', '--jobs', action="store", default=1, type=int, metavar="JOBS",
help='Set number of concurrent board tests (default is 1)')
parser.add_argument('-b', '--board', action="append", metavar="ID", help="Limit testing to boards with specified unique IDs. Multiple boards can be listed.")
args = parser.parse_args()
# Allow CI to override the number of concurrent jobs.
if 'CI_JOBS' in os.environ:
args.jobs = int(os.environ['CI_JOBS'])
# Disable multiple jobs on macOS prior to Python 3.4. By default, multiprocessing uses
# fork() on Unix, which doesn't work on the Mac because CoreFoundation requires exec()
# to be used in order to init correctly (CoreFoundation is used in hidapi). Only on Python
# version 3.4+ is the multiprocessing.set_start_method() API available that lets us
# switch to the 'spawn' method, i.e. exec().
if args.jobs > 1 and sys.platform.startswith('darwin') and sys.version_info[0:2] < (3, 4):
print("WARNING: Cannot support multiple jobs on macOS prior to Python 3.4. Forcing 1 job.")
args.jobs = 1
# Setup logging based on concurrency and quiet option.
level = logging.DEBUG if args.debug else logging.INFO
if args.jobs == 1 and not args.quiet:
log_file = LOG_FILE_TEMPLATE.format(get_env_file_name())
# Create common log file.
if os.path.exists(log_file):
os.remove(log_file)
logToConsole = True
commonLogFile = open(log_file, "a")
else:
logToConsole = False
commonLogFile = None
board_list = []
result_list = []
# Put together list of boards to test
board_list = ConnectHelper.get_all_connected_probes(blocking=False)
board_id_list = sorted(b.unique_id for b in board_list)
# Filter boards.
if args.board:
board_id_list = [b for b in board_id_list if any(c for c in args.board if c.lower() in b.lower())]
# If only 1 job was requested, don't bother spawning processes.
start = time()
if args.jobs == 1:
for n, board_id in enumerate(board_id_list):
result_list += test_board(board_id, n, level, logToConsole, commonLogFile)
else:
# Create a pool of processes to run tests.
try:
pool = mp.Pool(args.jobs)
# Issue board test job to process pool.
async_results = [pool.apply_async(test_board, (board_id, n, level, logToConsole, commonLogFile))
for n, board_id in enumerate(board_id_list)]
# Gather results.
for r in async_results:
result_list += r.get(timeout=JOB_TIMEOUT)
finally:
pool.close()
pool.join()
stop = time()
test_time = (stop - start)
print_summary(test_list, result_list, test_time)
summary_file = SUMMARY_FILE_TEMPLATE.format(get_env_file_name())
with open(summary_file, "w") as output_file:
print_summary(test_list, result_list, test_time, output_file)
generate_xml_results(result_list)
exit_val = 0 if Test.all_tests_pass(result_list) else -1
exit(exit_val)
def test_gdb(board_id=None, n=0):
temp_test_elf_name = None
result = GdbTestResult()
with ConnectHelper.session_with_chosen_probe(
unique_id=board_id, **get_session_options()) as session:
board = session.board
memory_map = board.target.get_memory_map()
ram_region = memory_map.get_default_region_of_type(MemoryType.RAM)
rom_region = memory_map.get_boot_memory()
target_type = board.target_type
binary_file = get_test_binary_path(board.test_binary)
if board_id is None:
board_id = board.unique_id
target_test_params = get_target_test_params(session)
test_port = 3333 + n
telnet_port = 4444 + n
# Hardware breakpoints are not supported above 0x20000000 on
# Cortex-M devices with FPB revision 1.
fpb = session.target.selected_core.fpb
assert fpb is not None
ignore_hw_bkpt_result = int(fpb.revision == 1
and ram_region.start >= 0x20000000)
# Program with initial test image
FileProgrammer(session).program(binary_file,
base_address=rom_region.start)
# Generate an elf from the binary test file.
temp_test_elf_name = binary_to_elf_file(binary_file, rom_region.start)
# Write out the test configuration
test_params = {
"test_port":
test_port,
"rom_start":
rom_region.start,
"rom_length":
rom_region.length,
"ram_start":
ram_region.start,
"ram_length":
ram_region.length,
"invalid_start":
0x3E000000,
"invalid_length":
0x1000,
"expect_error_on_invalid_access":
target_test_params['error_on_invalid_access'],
"ignore_hw_bkpt_result":
ignore_hw_bkpt_result,
"test_elf":
temp_test_elf_name,
}
test_param_filename = os.path.join(
TEST_OUTPUT_DIR, "gdb_test_params%s_%d.txt" % (get_env_file_name(), n))
with open(test_param_filename, "w") as f:
f.write(json.dumps(test_params))
# Remove result from previous run.
test_result_filename = os.path.join(
TEST_OUTPUT_DIR,
"gdb_test_results%s_%d.txt" % (get_env_file_name(), n))
if os.path.exists(test_result_filename):
os.remove(test_result_filename)
# Run the test
gdb_args = [
PYTHON_GDB, "--nh", "-ex",
"set $testn=%d" % n,
"--command=%s" % GDB_SCRIPT_PATH
]
gdb_output_filename = os.path.join(
TEST_OUTPUT_DIR,
"gdb_output%s_%s_%d.txt" % (get_env_file_name(), board.target_type, n))
with open(gdb_output_filename, "w") as f:
LOG.info('Starting gdb (stdout -> %s): %s', gdb_output_filename,
' '.join(gdb_args))
gdb_program = Popen(gdb_args, stdin=PIPE, stdout=f, stderr=STDOUT)
server_args = [
'gdbserver',
'--port=%i' % test_port,
"--telnet-port=%i" % telnet_port,
"--frequency=%i" % target_test_params['test_clock'],
"--uid=%s" % board_id,
]
server = PyOCDTool()
LOG.info('Starting gdbserver: %s', ' '.join(server_args))
server_thread = threading.Thread(target=server.run, args=[server_args])
server_thread.daemon = True
server_thread.start()
LOG.info('Waiting for gdb to finish...')
did_complete = wait_with_deadline(gdb_program, TEST_TIMEOUT_SECONDS)
LOG.info('Waiting for server to finish...')
server_thread.join(timeout=TEST_TIMEOUT_SECONDS)
if not did_complete:
LOG.error("Test timed out!")
if server_thread.is_alive():
LOG.error('Server is still running!')
try:
with open(gdb_output_filename, 'r') as f:
LOG.debug('Gdb output:\n%s', f.read())
except IOError:
pass
# Read back the result
result.passed = False
if did_complete:
try:
with open(test_result_filename, "r") as f:
test_result = json.loads(f.read())
# Print results
if set(TEST_RESULT_KEYS).issubset(test_result):
print("----------------Test Results----------------")
print("HW breakpoint count: %s" %
test_result["breakpoint_count"])
print("Watchpoint count: %s" % test_result["watchpoint_count"])
print("Average instruction step time: %s" %
test_result["step_time_si"])
print("Average single step time: %s" %
test_result["step_time_s"])
print("Average over step time: %s" %
test_result["step_time_n"])
print("Failure count: %i" % test_result["fail_count"])
result.passed = test_result["fail_count"] == 0
except IOError as err:
LOG.error("Error reading test results: %s", err, exc_info=True)
if result.passed:
print("GDB TEST PASSED")
else:
print("GDB TEST FAILED")
# Cleanup
try:
if temp_test_elf_name and os.path.exists(temp_test_elf_name):
os.remove(temp_test_elf_name)
os.remove(test_result_filename)
os.remove(test_param_filename)
except IOError as err:
pass
return result
def test_gdb(board_id=None, n=0):
temp_test_elf_name = None
result = GdbTestResult()
with ConnectHelper.session_with_chosen_probe(unique_id=board_id, **get_session_options()) as session:
board = session.board
memory_map = board.target.get_memory_map()
ram_region = memory_map.get_default_region_of_type(MemoryType.RAM)
rom_region = memory_map.get_boot_memory()
target_type = board.target_type
binary_file = os.path.join(parentdir, 'binaries',
board.test_binary)
if board_id is None:
board_id = board.unique_id
target_test_params = get_target_test_params(session)
test_port = 3333 + n
telnet_port = 4444 + n
# Hardware breakpoints are not supported above 0x20000000 on
# CortexM devices
ignore_hw_bkpt_result = 1 if ram_region.start >= 0x20000000 else 0
# Program with initial test image
FileProgrammer(session).program(binary_file, base_address=rom_region.start)
# Generate an elf from the binary test file.
temp_test_elf_name = binary_to_elf_file(binary_file, rom_region.start)
# Write out the test configuration
test_params = {
"test_port" : test_port,
"rom_start" : rom_region.start,
"rom_length" : rom_region.length,
"ram_start" : ram_region.start,
"ram_length" : ram_region.length,
"invalid_start" : 0x3E000000,
"invalid_length" : 0x1000,
"expect_error_on_invalid_access" : target_test_params['error_on_invalid_access'],
"ignore_hw_bkpt_result" : ignore_hw_bkpt_result,
"test_elf" : temp_test_elf_name,
}
test_param_filename = "gdb_test_params%s_%d.txt" % (get_env_file_name(), n)
with open(test_param_filename, "w") as f:
f.write(json.dumps(test_params))
# Run the test
gdb = [PYTHON_GDB, "--nh", "-ex", "set $testn=%d" % n, "--command=gdb_script.py"]
output_filename = "gdb_output%s_%s_%d.txt" % (get_env_file_name(), board.target_type, n)
with open(output_filename, "w") as f:
program = Popen(gdb, stdin=PIPE, stdout=f, stderr=STDOUT)
args = ['gdbserver',
'--port=%i' % test_port,
"--telnet-port=%i" % telnet_port,
"--frequency=%i" % target_test_params['test_clock'],
"--uid=%s" % board_id,
]
server = PyOCDTool()
server.run(args)
program.wait()
# Read back the result
test_result_filename = "gdb_test_results%s_%d.txt" % (get_env_file_name(), n)
with open(test_result_filename, "r") as f:
test_result = json.loads(f.read())
# Print results
if set(TEST_RESULT_KEYS).issubset(test_result):
print("----------------Test Results----------------")
print("HW breakpoint count: %s" % test_result["breakpoint_count"])
print("Watchpoint count: %s" % test_result["watchpoint_count"])
print("Average instruction step time: %s" %
test_result["step_time_si"])
print("Average single step time: %s" % test_result["step_time_s"])
print("Average over step time: %s" % test_result["step_time_n"])
print("Failure count: %i" % test_result["fail_count"])
result.passed = test_result["fail_count"] == 0
else:
result.passed = False
# Cleanup
if temp_test_elf_name and os.path.exists(temp_test_elf_name):
os.remove(temp_test_elf_name)
os.remove(test_result_filename)
os.remove(test_param_filename)
return result