def run(self):
for retry_count in range(self.RETRY_COUNT):
test_info = TestInfo(self.test_name)
if retry_count > 0:
test_info.info('Previous attempts %s' % retry_count)
try:
self._run(test_info)
except IOError:
time.sleep(self.DELAY_BEFORE_RETRY_S)
continue
self.parent_test.attach_subtest(test_info)
break
else:
raise Exception("Flashing failed after %i retries" %
self.RETRY_COUNT)
def __init__(self, tfiles_list_path):
'''
This class holds all of the information from the test suite.
This constructor, analyzes the test suite files (classes) create a TestInfo instance for each test method,
together with its proper annotations (e.g. @Ignore)
:param tfiles_list_path: path to mvn .lst file that contains a list of all test files
example: "commons-math\target\maven-status\maven-compiler-plugin\testCompile\default-testCompile\inputFiles.lst"
'''
print_count = 0
abstract_count = 0
self.tinfos = []
with open(tfiles_list_path, "r") as tfiles_paths:
tfile_paths_list = list(tfiles_paths)
for idx, tfile_path in enumerate(tfile_paths_list):
tfile_path = tfile_path.rstrip('\n')
tfile_name = Path(tfile_path).stem
if tfile_name.endswith('AbstractTest'):
abstract_count += 1
continue
tmethodnameTannotations = JavaTestClassFileParser.parse_and_return_methodnameTannotations(tfile_path)
for (tmethod_name, annotations) in tmethodnameTannotations:
tinfo = TestInfo(tfile_path, tfile_name, tmethod_name, annotations)
if Config.SAMPLE_ONLY_ACTUAL_FAULTY_PACKAGES and tinfo.is_in_faulty_package:
self.tinfos.append(tinfo)
if print_count == 20:
logger.debug(f"processed {idx +1} out of {len(tfile_paths_list)} test files information")
print_count = 0
print_count += 1
logger.debug(f"ignored {abstract_count} abstract test classes")
logger.debug(f"added {len(self.tinfos)} test files information")
def __init__(self, yaml_prj, path='.'):
self.prj = yaml_prj
self._path = path
self.name = yaml_prj.name
if_match = self._if_exp.match(self.name)
if if_match is not None:
self._bl_name = if_match.group(1) + '_bl'
self._is_bl = False
self._board_id = self._name_to_board_id[self.name]
bl_match = self._bl_exp.match(self.name)
if bl_match is not None:
self._is_bl = True
if if_match is None and bl_match is None:
raise Exception("Invalid project name %s" % self.name)
self._built = False
self._boards = None
self._bl = None
self._test_info = TestInfo(self.get_name())
build_path = os.path.normpath(path + os.sep + 'projectfiles' + os.sep +
self._tool + os.sep + self.name +
os.sep + 'build')
self._hex_file = os.path.normpath(build_path + os.sep +
self.name + '.hex')
self._bin_file = os.path.normpath(build_path + os.sep +
self.name + '.bin')
# By default test all configurations and boards
self._only_test_first = False
self._load_if = True
self._load_bl = True
self._test_if_bl = True
self._test_ep = True
def run(self):
for retry_count in range(self.RETRY_COUNT):
test_info = TestInfo(self.test_name)
if retry_count > 0:
test_info.info('Previous attempts %s' % retry_count)
try:
self._run(test_info)
except IOError:
time.sleep(self.DELAY_BEFORE_RETRY_S)
# Update board info since a remount could have occurred
self.board.update_board_info()
continue
self.parent_test.attach_subtest(test_info)
break
else:
raise Exception("Flashing failed after %i retries" %
self.RETRY_COUNT)
def run_tests(self):
"""Run all configurations"""
# Tests can only be run once per TestManager instance
assert self._state is self._STATE.CONFIGURED
self._state = self._STATE.COMPLETE
all_tests_pass = True
for test_configuration in self._test_configuration_list:
board = test_configuration.board
test_info = TestInfo(test_configuration.name)
test_configuration.test_info = test_info
test_info.info("Board: %s" % test_configuration.board)
test_info.info("Application: %s" %
test_configuration.if_firmware)
test_info.info("Bootloader: %s" %
test_configuration.bl_firmware)
test_info.info("Target: %s" % test_configuration.target)
if self._load_if:
if_path = test_configuration.if_firmware.hex_path
board.load_interface(if_path, test_info)
valid_bl = test_configuration.bl_firmware is not None
if self._load_bl and valid_bl:
bl_path = test_configuration.bl_firmware.hex_path
board.load_bootloader(bl_path, test_info)
board.set_check_fs_on_remount(True)
if self._test_daplink:
daplink_test(test_configuration, test_info)
if self._test_ep:
test_endpoints(test_configuration, test_info)
if test_info.get_failed():
all_tests_pass = False
self._all_tests_pass = all_tests_pass
def main():
self_path = os.path.abspath(__file__)
test_dir = os.path.dirname(self_path)
daplink_dir = os.path.dirname(test_dir)
# We make assumptions that break if user copies script file outside the test dir
if os.path.basename(test_dir) != "test":
print("Error - this script must reside in the test directory")
exit(-1)
git_sha, local_changes = get_git_info(daplink_dir)
firmware_list = get_firmware_names(daplink_dir)
firmware_choices = [
firmware for firmware in firmware_list if firmware.endswith('_if')
]
description = 'DAPLink validation and testing tool'
parser = argparse.ArgumentParser(description=description)
parser.add_argument('--targetdir',
help='Directory with pre-built target test images.',
default=None)
parser.add_argument('--user',
type=str,
default=None,
help='MBED username (required for compile-api)')
parser.add_argument('--password',
type=str,
default=None,
help='MBED password (required for compile-api)')
parser.add_argument('--firmwaredir',
help='Directory with firmware images to test',
default=None)
parser.add_argument('--firmware',
help='Firmware to test',
action='append',
choices=firmware_choices,
default=[],
required=False)
parser.add_argument('--logdir',
help='Directory to log test results to',
default=DEFAULT_TEST_DIR)
parser.add_argument('--noloadif',
help='Skip load step for interface.',
default=False,
action='store_true')
parser.add_argument('--notestendpt',
help='Dont test the interface '
'USB endpoints.',
default=False,
action='store_true')
parser.add_argument('--loadbl',
help='Load bootloader before test.',
default=False,
action='store_true')
parser.add_argument('--testdl',
help='Run DAPLink specific tests. '
'The DAPLink test tests bootloader updates so use'
'with caution',
default=False,
action='store_true')
parser.add_argument('--testfirst',
help='If multiple boards of the same '
'type are found only test the first one.',
default=False,
action='store_true')
parser.add_argument('--verbose',
help='Verbose output',
choices=VERB_LEVELS,
default=VERB_NORMAL)
parser.add_argument('--dryrun',
default=False,
action='store_true',
help='Print info on configurations but dont '
'actually run tests.')
parser.add_argument(
'--force',
action='store_true',
default=False,
help=
'Try to run tests even if there are problems. Delete logs from previous run.'
)
args = parser.parse_args()
use_prebuilt = args.targetdir is not None
use_compile_api = args.user is not None and args.password is not None
test_info = TestInfo('DAPLink')
# Validate args
# See if user wants to test endpoints. If yes and he didn't provide
# target test binaries, use the Compile API to build them
all_targets = None
if not args.notestendpt:
if not use_prebuilt and not use_compile_api:
print("Endpoint test requires target test images.")
print(" Directory with pre-built target test images")
print(" must be specified with '--targetdir'")
print("OR")
print(" developer.mbed.org login credentials must be ")
print(" specified with '--user' and '--password' so test ")
print(" images can be built with the RESTful Compile API.")
print("NOTE: you can skip the endpoint tests altogether ")
print("with --notestendpt")
exit(-1)
if args.targetdir is not None:
target_dir = args.targetdir
else:
target_dir = daplink_dir + os.sep + 'tmp'
build_target_bundle(target_dir, args.user, args.password,
test_info)
target_bundle = load_target_bundle(target_dir)
all_targets = target_bundle.get_target_list()
if os.path.exists(args.logdir):
if args.force:
shutil.rmtree(args.logdir)
else:
print('Error - test results directory "%s" already exists' %
args.logdir)
exit(-1)
# Get all relevant info
if args.firmwaredir is None:
firmware_bundle = load_bundle_from_project()
else:
firmware_bundle = load_bundle_from_release(args.firmwaredir)
all_firmware = firmware_bundle.get_firmware_list()
all_boards = get_all_attached_daplink_boards()
for board in all_boards:
if board.get_mode() == board.MODE_BL:
print('Switching to APP mode on board: %s' % board.unique_id)
try:
board.set_mode(board.MODE_IF)
except Exception:
print('Unable to switch mode on board: %s' % board.unique_id)
# Make sure firmware is present
firmware_explicitly_specified = len(args.firmware) != 0
if firmware_explicitly_specified:
all_firmware_names = set(fw.name for fw in all_firmware)
firmware_missing = False
for firmware_name in args.firmware:
if firmware_name not in all_firmware_names:
firmware_missing = True
test_info.failure('Cannot find firmware %s' % firmware_name)
if firmware_missing:
test_info.failure('Firmware missing - aborting test')
exit(-1)
# Create manager and add resources
tester = TestManager()
tester.add_firmware(all_firmware)
tester.add_boards(all_boards)
if all_targets is not None:
tester.add_targets(all_targets)
if firmware_explicitly_specified:
tester.set_firmware_filter(args.firmware)
# Configure test manager
tester.set_test_first_board_only(args.testfirst)
tester.set_load_if(not args.noloadif)
tester.set_test_ep(not args.notestendpt)
tester.set_load_bl(args.loadbl)
tester.set_test_daplink(args.testdl)
# Build test configurations
tester.build_test_configurations(test_info)
test_config_list = tester.get_test_configurations()
if len(test_config_list) == 0:
test_info.failure("Nothing that can be tested")
exit(-1)
else:
test_info.info('Test configurations to be run:')
index = 0
for test_config in test_config_list:
test_info.info(' %i: %s' % (index, test_config))
index += 1
test_info.info('')
untested_list = tester.get_untested_firmware()
if len(untested_list) == 0:
test_info.info("All firmware can be tested")
else:
test_info.info('Fimrware that will not be tested:')
for untested_firmware in untested_list:
test_info.info(' %s' % untested_firmware.name)
test_info.info('')
if firmware_explicitly_specified and len(untested_list) != 0:
test_info.failure("Exiting because not all firmware could be tested")
exit(-1)
# If this is a dryrun don't run tests, just print info
if args.dryrun:
exit(0)
# Run tests
tester.run_tests()
# Print test results
tester.print_results(args.verbose)
tester.write_test_results(args.logdir,
git_sha=git_sha,
local_changes=local_changes)
# Warn about untested boards
print('')
for firmware in tester.get_untested_firmware():
print('Warning - configuration %s is untested' % firmware.name)
if tester.all_tests_pass:
print("All boards passed")
exit(0)
else:
print("Test Failed")
exit(-1)
def run_tests(self):
"""Run all configurations"""
# Tests can only be run once per TestManager instance
assert self._state is self._STATE.CONFIGURED
self._state = self._STATE.COMPLETE
all_tests_pass = True
for test_configuration in self._test_configuration_list:
board = test_configuration.board
test_info = TestInfo(test_configuration.name)
test_configuration.test_info = test_info
test_info.info("Board: %s" % test_configuration.board)
test_info.info("Application: %s" % test_configuration.if_firmware)
test_info.info("Bootloader: %s" % test_configuration.bl_firmware)
test_info.info("Target: %s" % test_configuration.target)
if self._load_if:
if_path = test_configuration.if_firmware.hex_path
board.load_interface(if_path, test_info)
valid_bl = test_configuration.bl_firmware is not None
if self._load_bl and valid_bl:
bl_path = test_configuration.bl_firmware.hex_path
board.load_bootloader(bl_path, test_info)
board.set_check_fs_on_remount(True)
if self._test_daplink:
daplink_test(test_configuration, test_info)
if self._test_ep:
test_endpoints(test_configuration, test_info)
if test_info.get_failed():
all_tests_pass = False
self._all_tests_pass = all_tests_pass
def main():
self_path = os.path.abspath(__file__)
test_dir = os.path.dirname(self_path)
daplink_dir = os.path.dirname(test_dir)
# We make assumptions that break if user copies script file outside the test dir
if os.path.basename(test_dir) != "test":
print("Error - this script must reside in the test directory")
exit(-1)
git_sha, local_changes = get_git_info(daplink_dir)
firmware_list = get_firmware_names(daplink_dir)
firmware_choices = [firmware for firmware in firmware_list if
firmware.endswith('_if')]
description = 'DAPLink validation and testing tool'
parser = argparse.ArgumentParser(description=description)
parser.add_argument('--targetdir',
help='Directory with pre-built target test images.',
default=None)
parser.add_argument('--user', type=str, default=None,
help='MBED username (required for compile-api)')
parser.add_argument('--password', type=str, default=None,
help='MBED password (required for compile-api)')
parser.add_argument('--firmwaredir',
help='Directory with firmware images to test',
default=None)
parser.add_argument('--firmware', help='Firmware to test', action='append',
choices=firmware_choices, default=[], required=False)
parser.add_argument('--logdir', help='Directory to log test results to',
default=DEFAULT_TEST_DIR)
parser.add_argument('--noloadif', help='Skip load step for interface.',
default=False, action='store_true')
parser.add_argument('--notestendpt', help='Dont test the interface '
'USB endpoints.', default=False, action='store_true')
parser.add_argument('--loadbl', help='Load bootloader before test.',
default=False, action='store_true')
parser.add_argument('--testdl', help='Run DAPLink specific tests. '
'The DAPLink test tests bootloader updates so use'
'with caution',
default=False, action='store_true')
parser.add_argument('--testfirst', help='If multiple boards of the same '
'type are found only test the first one.',
default=False, action='store_true')
parser.add_argument('--verbose', help='Verbose output',
choices=VERB_LEVELS, default=VERB_NORMAL)
parser.add_argument('--dryrun', default=False, action='store_true',
help='Print info on configurations but dont '
'actually run tests.')
parser.add_argument('--force', action='store_true', default=False,
help='Try to run tests even if there are problems. Delete logs from previous run.')
args = parser.parse_args()
use_prebuilt = args.targetdir is not None
use_compile_api = args.user is not None and args.password is not None
test_info = TestInfo('DAPLink')
# Validate args
# See if user wants to test endpoints. If yes and he didn't provide
# target test binaries, use the Compile API to build them
all_targets = None
if not args.notestendpt:
if not use_prebuilt and not use_compile_api:
print("Endpoint test requires target test images.")
print(" Directory with pre-built target test images")
print(" must be specified with '--targetdir'")
print("OR")
print(" developer.mbed.org login credentials must be ")
print(" specified with '--user' and '--password' so test ")
print(" images can be built with the RESTful Compile API.")
print("NOTE: you can skip the endpoint tests altogether ")
print("with --notestendpt")
exit(-1)
if args.targetdir is not None:
target_dir = args.targetdir
else:
target_dir = daplink_dir + os.sep + 'tmp'
build_target_bundle(target_dir, args.user, args.password, test_info)
target_bundle = load_target_bundle(target_dir)
all_targets = target_bundle.get_target_list()
if os.path.exists(args.logdir):
if args.force:
shutil.rmtree(args.logdir)
else:
print('Error - test results directory "%s" already exists' %
args.logdir)
exit(-1)
# Get all relevant info
if args.firmwaredir is None:
firmware_bundle = load_bundle_from_project()
else:
firmware_bundle = load_bundle_from_release(args.firmwaredir)
all_firmware = firmware_bundle.get_firmware_list()
all_boards = get_all_attached_daplink_boards()
for board in all_boards:
if board.get_mode() == board.MODE_BL:
print('Switching to APP mode on board: %s' % board.unique_id)
try:
board.set_mode(board.MODE_IF)
except Exception:
print('Unable to switch mode on board: %s' % board.unique_id)
# Make sure firmware is present
firmware_explicitly_specified = len(args.firmware) != 0
if firmware_explicitly_specified:
all_firmware_names = set(fw.name for fw in all_firmware)
firmware_missing = False
for firmware_name in args.firmware:
if firmware_name not in all_firmware_names:
firmware_missing = True
test_info.failure('Cannot find firmware %s' % firmware_name)
if firmware_missing:
test_info.failure('Firmware missing - aborting test')
exit(-1)
# Create manager and add resources
tester = TestManager()
tester.add_firmware(all_firmware)
tester.add_boards(all_boards)
if all_targets is not None:
tester.add_targets(all_targets)
if firmware_explicitly_specified:
tester.set_firmware_filter(args.firmware)
# Configure test manager
tester.set_test_first_board_only(args.testfirst)
tester.set_load_if(not args.noloadif)
tester.set_test_ep(not args.notestendpt)
tester.set_load_bl(args.loadbl)
tester.set_test_daplink(args.testdl)
# Build test configurations
tester.build_test_configurations(test_info)
test_config_list = tester.get_test_configurations()
if len(test_config_list) == 0:
test_info.failure("Nothing that can be tested")
exit(-1)
else:
test_info.info('Test configurations to be run:')
index = 0
for test_config in test_config_list:
test_info.info(' %i: %s' % (index, test_config))
index += 1
test_info.info('')
untested_list = tester.get_untested_firmware()
if len(untested_list) == 0:
test_info.info("All firmware can be tested")
else:
test_info.info('Fimrware that will not be tested:')
for untested_firmware in untested_list:
test_info.info(' %s' % untested_firmware.name)
test_info.info('')
if firmware_explicitly_specified and len(untested_list) != 0:
test_info.failure("Exiting because not all firmware could be tested")
exit(-1)
# If this is a dryrun don't run tests, just print info
if args.dryrun:
exit(0)
# Run tests
tester.run_tests()
# Print test results
tester.print_results(args.verbose)
tester.write_test_results(args.logdir,
git_sha=git_sha,
local_changes=local_changes)
# Warn about untested boards
print('')
for firmware in tester.get_untested_firmware():
print('Warning - configuration %s is untested' % firmware.name)
if tester.all_tests_pass:
print("All boards passed")
exit(0)
else:
print("Test Failed")
exit(-1)
def __str__(self):
from common import exc_info_to_string
from test_info import TestInfo
return exc_info_to_string(self.exc_info, TestInfo(self.test))
class ProjectTester(object):
_if_exp = re.compile("^([a-z0-9]+)_([a-z0-9_]+)_if$")
_bl_exp = re.compile("^([a-z0-9]+)_bl$")
_tool = 'uvision'
_name_to_board_id = {
'k20dx_k22f_if': 0x0231,
'k20dx_k64f_if': 0x0240,
'kl26z_microbit_if': 0x9900,
'kl26z_nrf51822_if': 0x9900,
'lpc11u35_lpc812_if': 0x1050,
'lpc11u35_lpc1114_if': 0x1114,
'lpc11u35_efm32gg_stk_if': 0x2015,
'sam3u2c_nrf51822_if': 0x1100,
}
def __init__(self, yaml_prj, path='.'):
self.prj = yaml_prj
self._path = path
self.name = yaml_prj.name
if_match = self._if_exp.match(self.name)
if if_match is not None:
self._bl_name = if_match.group(1) + '_bl'
self._is_bl = False
self._board_id = self._name_to_board_id[self.name]
bl_match = self._bl_exp.match(self.name)
if bl_match is not None:
self._is_bl = True
if if_match is None and bl_match is None:
raise Exception("Invalid project name %s" % self.name)
self._built = False
self._boards = None
self._bl = None
self._test_info = TestInfo(self.get_name())
build_path = os.path.normpath(path + os.sep + 'projectfiles' + os.sep +
self._tool + os.sep + self.name +
os.sep + 'build')
self._hex_file = os.path.normpath(build_path + os.sep +
self.name + '.hex')
self._bin_file = os.path.normpath(build_path + os.sep +
self.name + '.bin')
# By default test all configurations and boards
self._only_test_first = False
self._load_if = True
self._load_bl = True
self._test_if_bl = True
self._test_ep = True
def is_bl(self):
return self._is_bl
def get_name(self):
return self.name
def get_built(self):
"""
Return true if the project has been built in the current session
"""
return self._built
def get_bl_name(self):
"""
Get the name of the bootloader
This function should only be called if the project
is not a bootloader
"""
assert not self.is_bl()
return self._bl_name
def set_bl(self, bl_prj):
"""
Set the boot loader for this interface.
Note - this function should only be called on
an interface project.
"""
assert isinstance(bl_prj, ProjectTester)
assert not self._is_bl
self._bl = bl_prj
def get_binary(self):
"""
Get the binary file associated with this project
Returns None if the bin file has not been created yet.
"""
return self._bin_file if os.path.isfile(self._bin_file) else None
def get_hex(self):
"""
Get the hex file associated with this project
Returns None if the hex file has not been created yet.
"""
return self._hex_file if os.path.isfile(self._hex_file) else None
def get_board_id(self):
"""
Get the board ID for this project
Board ID is only valid if the target is not a bootloader
"""
return self._board_id
def set_test_boards(self, boards):
assert type(boards) is list
self._boards = boards
def get_test_boards(self):
return self._boards
def get_test_info(self):
return self._test_info
def build(self, clean=True):
self._test_info.info("Building %s" % self.name)
start = time.time()
#TODO - build bootloader if it isn't built yet
#TODO - print info on bootloader
ret = self.prj.generate(self._tool, False)
self._test_info.info('Export return value %s' % ret)
if ret != 0:
raise Exception('Export return value %s' % ret)
ret = self.prj.build(self._tool)
self._test_info.info('Build return value %s' % ret)
if ret != 0:
raise Exception('Build return value %s' % ret)
files = self.prj.get_generated_project_files(self._tool)
export_path = files['path']
base_file = os.path.normpath(export_path + os.sep + 'build' +
os.sep + self.name)
built_hex_file = base_file + '.hex'
built_bin_file = base_file + '.bin'
assert (os.path.abspath(self._hex_file) ==
os.path.abspath(built_hex_file))
assert (os.path.abspath(self._bin_file) ==
os.path.abspath(built_bin_file))
self._hex_file = built_hex_file
self._bin_file = built_bin_file
stop = time.time()
self._test_info.info('Build took %s seconds' % (stop - start))
self._built = True
def test_set_first_board_only(self, first):
assert type(first) is bool
self._only_test_first = first
def test_set_load_if(self, load):
assert type(load) is bool
self._load_if = load
def test_set_load_bl(self, load):
assert type(load) is bool
self._load_bl = load
def test_set_test_if_bl(self, run_test):
assert type(run_test) is bool
self._test_if_bl = run_test
def test_set_test_ep(self, run_test):
assert type(run_test) is bool
self._test_ep = run_test
def test(self):
boards_to_test = self._boards
if self._only_test_first:
boards_to_test = self._boards[0:1]
for board in boards_to_test:
# Load interface & bootloader
if self._load_if:
print("Loading interface")
board.load_interface(self.get_hex(), self._test_info)
#TODO - check CRC
if self._load_bl:
pass
#TODO - load bootloader
#TODO - check CRC
if self._test_if_bl:
# Test bootloader
# Test interface
board.test_fs(self._test_info)
board.test_fs_contents(self._test_info)
# Test endpoint
if self._test_ep:
test_endpoints(board, self._test_info)
return not self._test_info.get_failed()