def main(): self_path = os.path.abspath(__file__) test_dir = os.path.dirname(self_path) daplink_dir = os.path.dirname(test_dir) # We make assumptions that break if user copies script file outside the test dir if os.path.basename(test_dir) != "test": print("Error - this script must reside in the test directory") exit(-1) git_sha, local_changes = get_git_info(daplink_dir) firmware_list = get_firmware_names(daplink_dir) firmware_choices = [firmware for firmware in firmware_list if firmware.endswith('_if')] description = 'DAPLink validation and testing tool' parser = argparse.ArgumentParser(description=description) parser.add_argument('--targetdir', help='Directory with pre-built target test images.', default=None) parser.add_argument('--user', type=str, default=None, help='MBED username (required for compile-api)') parser.add_argument('--password', type=str, default=None, help='MBED password (required for compile-api)') parser.add_argument('--firmwaredir', help='Directory with firmware images to test', default=None) parser.add_argument('--firmware', help='Firmware to test', action='append', choices=firmware_choices, default=[], required=False) parser.add_argument('--logdir', help='Directory to log test results to', default=DEFAULT_TEST_DIR) parser.add_argument('--noloadif', help='Skip load step for interface.', default=False, action='store_true') parser.add_argument('--notestendpt', help='Dont test the interface ' 'USB endpoints.', default=False, action='store_true') parser.add_argument('--loadbl', help='Load bootloader before test.', default=False, action='store_true') parser.add_argument('--testdl', help='Run DAPLink specific tests. ' 'The DAPLink test tests bootloader updates so use' 'with caution', default=False, action='store_true') parser.add_argument('--testfirst', help='If multiple boards of the same ' 'type are found only test the first one.', default=False, action='store_true') parser.add_argument('--verbose', help='Verbose output', choices=VERB_LEVELS, default=VERB_NORMAL) parser.add_argument('--dryrun', default=False, action='store_true', help='Print info on configurations but dont ' 'actually run tests.') parser.add_argument('--force', action='store_true', default=False, help='Try to run tests even if there are problems. Delete logs from previous run.') args = parser.parse_args() use_prebuilt = args.targetdir is not None use_compile_api = args.user is not None and args.password is not None test_info = TestInfo('DAPLink') # Validate args # See if user wants to test endpoints. If yes and he didn't provide # target test binaries, use the Compile API to build them all_targets = None if not args.notestendpt: if not use_prebuilt and not use_compile_api: print("Endpoint test requires target test images.") print(" Directory with pre-built target test images") print(" must be specified with '--targetdir'") print("OR") print(" developer.mbed.org login credentials must be ") print(" specified with '--user' and '--password' so test ") print(" images can be built with the RESTful Compile API.") print("NOTE: you can skip the endpoint tests altogether ") print("with --notestendpt") exit(-1) if args.targetdir is not None: target_dir = args.targetdir else: target_dir = daplink_dir + os.sep + 'tmp' build_target_bundle(target_dir, args.user, args.password, test_info) target_bundle = load_target_bundle(target_dir) all_targets = target_bundle.get_target_list() if os.path.exists(args.logdir): if args.force: shutil.rmtree(args.logdir) else: print('Error - test results directory "%s" already exists' % args.logdir) exit(-1) # Get all relevant info if args.firmwaredir is None: firmware_bundle = load_bundle_from_project() else: firmware_bundle = load_bundle_from_release(args.firmwaredir) all_firmware = firmware_bundle.get_firmware_list() all_boards = get_all_attached_daplink_boards() for board in all_boards: if board.get_mode() == board.MODE_BL: print('Switching to APP mode on board: %s' % board.unique_id) try: board.set_mode(board.MODE_IF) except Exception: print('Unable to switch mode on board: %s' % board.unique_id) # Make sure firmware is present firmware_explicitly_specified = len(args.firmware) != 0 if firmware_explicitly_specified: all_firmware_names = set(fw.name for fw in all_firmware) firmware_missing = False for firmware_name in args.firmware: if firmware_name not in all_firmware_names: firmware_missing = True test_info.failure('Cannot find firmware %s' % firmware_name) if firmware_missing: test_info.failure('Firmware missing - aborting test') exit(-1) # Create manager and add resources tester = TestManager() tester.add_firmware(all_firmware) tester.add_boards(all_boards) if all_targets is not None: tester.add_targets(all_targets) if firmware_explicitly_specified: tester.set_firmware_filter(args.firmware) # Configure test manager tester.set_test_first_board_only(args.testfirst) tester.set_load_if(not args.noloadif) tester.set_test_ep(not args.notestendpt) tester.set_load_bl(args.loadbl) tester.set_test_daplink(args.testdl) # Build test configurations tester.build_test_configurations(test_info) test_config_list = tester.get_test_configurations() if len(test_config_list) == 0: test_info.failure("Nothing that can be tested") exit(-1) else: test_info.info('Test configurations to be run:') index = 0 for test_config in test_config_list: test_info.info(' %i: %s' % (index, test_config)) index += 1 test_info.info('') untested_list = tester.get_untested_firmware() if len(untested_list) == 0: test_info.info("All firmware can be tested") else: test_info.info('Fimrware that will not be tested:') for untested_firmware in untested_list: test_info.info(' %s' % untested_firmware.name) test_info.info('') if firmware_explicitly_specified and len(untested_list) != 0: test_info.failure("Exiting because not all firmware could be tested") exit(-1) # If this is a dryrun don't run tests, just print info if args.dryrun: exit(0) # Run tests tester.run_tests() # Print test results tester.print_results(args.verbose) tester.write_test_results(args.logdir, git_sha=git_sha, local_changes=local_changes) # Warn about untested boards print('') for firmware in tester.get_untested_firmware(): print('Warning - configuration %s is untested' % firmware.name) if tester.all_tests_pass: print("All boards passed") exit(0) else: print("Test Failed") exit(-1)
def main(): self_path = os.path.abspath(__file__) test_dir = os.path.dirname(self_path) daplink_dir = os.path.dirname(test_dir) # We make assumptions that break if user copies script file outside the test dir if os.path.basename(test_dir) != "test": print("Error - this script must reside in the test directory") exit(-1) git_sha, local_changes = get_git_info(daplink_dir) firmware_list = get_firmware_names(daplink_dir) firmware_choices = [ firmware for firmware in firmware_list if firmware.endswith('_if') ] description = 'DAPLink validation and testing tool' parser = argparse.ArgumentParser(description=description) parser.add_argument('--targetdir', help='Directory with pre-built target test images.', default=None) parser.add_argument('--user', type=str, default=None, help='MBED username (required for compile-api)') parser.add_argument('--password', type=str, default=None, help='MBED password (required for compile-api)') parser.add_argument('--firmwaredir', help='Directory with firmware images to test', default=None) parser.add_argument('--firmware', help='Firmware to test', action='append', choices=firmware_choices, default=[], required=False) parser.add_argument('--logdir', help='Directory to log test results to', default=DEFAULT_TEST_DIR) parser.add_argument('--noloadif', help='Skip load step for interface.', default=False, action='store_true') parser.add_argument('--notestendpt', help='Dont test the interface ' 'USB endpoints.', default=False, action='store_true') parser.add_argument('--loadbl', help='Load bootloader before test.', default=False, action='store_true') parser.add_argument('--testdl', help='Run DAPLink specific tests. ' 'The DAPLink test tests bootloader updates so use' 'with caution', default=False, action='store_true') parser.add_argument('--testfirst', help='If multiple boards of the same ' 'type are found only test the first one.', default=False, action='store_true') parser.add_argument('--verbose', help='Verbose output', choices=VERB_LEVELS, default=VERB_NORMAL) parser.add_argument('--dryrun', default=False, action='store_true', help='Print info on configurations but dont ' 'actually run tests.') parser.add_argument( '--force', action='store_true', default=False, help= 'Try to run tests even if there are problems. Delete logs from previous run.' ) args = parser.parse_args() use_prebuilt = args.targetdir is not None use_compile_api = args.user is not None and args.password is not None test_info = TestInfo('DAPLink') # Validate args # See if user wants to test endpoints. If yes and he didn't provide # target test binaries, use the Compile API to build them all_targets = None if not args.notestendpt: if not use_prebuilt and not use_compile_api: print("Endpoint test requires target test images.") print(" Directory with pre-built target test images") print(" must be specified with '--targetdir'") print("OR") print(" developer.mbed.org login credentials must be ") print(" specified with '--user' and '--password' so test ") print(" images can be built with the RESTful Compile API.") print("NOTE: you can skip the endpoint tests altogether ") print("with --notestendpt") exit(-1) if args.targetdir is not None: target_dir = args.targetdir else: target_dir = daplink_dir + os.sep + 'tmp' build_target_bundle(target_dir, args.user, args.password, test_info) target_bundle = load_target_bundle(target_dir) all_targets = target_bundle.get_target_list() if os.path.exists(args.logdir): if args.force: shutil.rmtree(args.logdir) else: print('Error - test results directory "%s" already exists' % args.logdir) exit(-1) # Get all relevant info if args.firmwaredir is None: firmware_bundle = load_bundle_from_project() else: firmware_bundle = load_bundle_from_release(args.firmwaredir) all_firmware = firmware_bundle.get_firmware_list() all_boards = get_all_attached_daplink_boards() for board in all_boards: if board.get_mode() == board.MODE_BL: print('Switching to APP mode on board: %s' % board.unique_id) try: board.set_mode(board.MODE_IF) except Exception: print('Unable to switch mode on board: %s' % board.unique_id) # Make sure firmware is present firmware_explicitly_specified = len(args.firmware) != 0 if firmware_explicitly_specified: all_firmware_names = set(fw.name for fw in all_firmware) firmware_missing = False for firmware_name in args.firmware: if firmware_name not in all_firmware_names: firmware_missing = True test_info.failure('Cannot find firmware %s' % firmware_name) if firmware_missing: test_info.failure('Firmware missing - aborting test') exit(-1) # Create manager and add resources tester = TestManager() tester.add_firmware(all_firmware) tester.add_boards(all_boards) if all_targets is not None: tester.add_targets(all_targets) if firmware_explicitly_specified: tester.set_firmware_filter(args.firmware) # Configure test manager tester.set_test_first_board_only(args.testfirst) tester.set_load_if(not args.noloadif) tester.set_test_ep(not args.notestendpt) tester.set_load_bl(args.loadbl) tester.set_test_daplink(args.testdl) # Build test configurations tester.build_test_configurations(test_info) test_config_list = tester.get_test_configurations() if len(test_config_list) == 0: test_info.failure("Nothing that can be tested") exit(-1) else: test_info.info('Test configurations to be run:') index = 0 for test_config in test_config_list: test_info.info(' %i: %s' % (index, test_config)) index += 1 test_info.info('') untested_list = tester.get_untested_firmware() if len(untested_list) == 0: test_info.info("All firmware can be tested") else: test_info.info('Fimrware that will not be tested:') for untested_firmware in untested_list: test_info.info(' %s' % untested_firmware.name) test_info.info('') if firmware_explicitly_specified and len(untested_list) != 0: test_info.failure("Exiting because not all firmware could be tested") exit(-1) # If this is a dryrun don't run tests, just print info if args.dryrun: exit(0) # Run tests tester.run_tests() # Print test results tester.print_results(args.verbose) tester.write_test_results(args.logdir, git_sha=git_sha, local_changes=local_changes) # Warn about untested boards print('') for firmware in tester.get_untested_firmware(): print('Warning - configuration %s is untested' % firmware.name) if tester.all_tests_pass: print("All boards passed") exit(0) else: print("Test Failed") exit(-1)
def main(): # Save current directory cur_dir = os.getcwd() parent_dir = os.path.dirname(cur_dir) os.chdir(parent_dir) # Wrap every project in a ProjectTester object # Tie every bootloader to one or more interface projects projects = list(Generator('projects.yaml').generate()) yaml_dir = os.getcwd() all_projects = [ProjectTester(project, yaml_dir) for project in projects] if_project_list = [project for project in all_projects if not project.is_bl()] bl_project_list = [project for project in all_projects if project.is_bl()] bl_name_to_proj = {project.name: project for project in bl_project_list} for project in if_project_list: bl_name = project.get_bl_name() if bl_name in bl_name_to_proj: project.set_bl(bl_name_to_proj[bl_name]) #TODO - make sure all bootloaders are tied to an interface, make sure all # objects are accounted for # Create list of projects to show user prj_names = [prj.get_name() for prj in if_project_list] name_to_prj = {prj.get_name(): prj for prj in if_project_list} VERB_MINIMAL = 'Minimal' # Just top level errors VERB_NORMAL = 'Normal' # Top level errors and warnings VERB_VERBOSE = 'Verbose' # All errors and warnings VERB_ALL = 'All' # All errors VERB_LEVELS = [VERB_MINIMAL, VERB_NORMAL, VERB_VERBOSE, VERB_ALL] description = 'DAPLink validation and testing tool' parser = argparse.ArgumentParser(description=description) parser.add_argument('--targetdir', help='Directory with pre-build target test images.', default=None) parser.add_argument('--user', type=str, default=None, help='MBED username (required for compile-api)') parser.add_argument('--password', type=str, default=None, help='MBED password (required for compile-api)') parser.add_argument('--project', help='Project to test', action='append', choices=prj_names, default=[], required=False) parser.add_argument('--nobuild', help='Skip build step. Binaries must have been built already.', default=False, action='store_true') parser.add_argument('--noloadif', help='Skip load step for interface.', default=False, action='store_true') parser.add_argument('--noloadbl', help='Skip load step for bootloader.', default=False, action='store_true') parser.add_argument('--notestendpt', help='Dont test the interface USB endpoints.', default=False, action='store_true') parser.add_argument('--testfirst', help='If multiple boards of the same type are found only test the first one.', default=False, action='store_true') parser.add_argument('--verbose', help='Verbose output', choices=VERB_LEVELS, default=VERB_NORMAL) args = parser.parse_args() use_prebuilt = args.targetdir is not None use_compile_api = args.user is not None and args.password is not None # Validate args if not args.notestendpt: if not use_prebuilt and not use_compile_api: print("Endpoint test requires target test images.") print(" Directory with pre-build target test images") print(" must be specified with '--targetdir'") print("OR") print(" Mbed login credentials '--user' and '--password' must") print(" be specified so test images can be built with") print(" the compile API.") exit(-1) boards_explicitly_specified = len(args.project) != 0 # Put together the list of projects to build if boards_explicitly_specified: projects_to_test = [name_to_prj[name] for name in args.project] else: projects_to_test = if_project_list # Collect attached mbed devices all_boards = get_all_attached_daplink_boards() # Attach firmware build credentials if not args.notestendpt: for board in all_boards: if args.targetdir is None: board.set_build_login(args.user, args.password) else: board.set_build_prebuilt_dir(args.targetdir) # Create table mapping each board id to boards board_id_to_board_list = {} for board in all_boards: board_id = board.get_board_id() if board_id not in board_id_to_board_list: board_id_to_board_list[board_id] = [] board_id_to_board_list[board_id].append(board) # Attach each test board to a project for project in projects_to_test: board_id = project.get_board_id() if board_id in board_id_to_board_list: project.set_test_boards(board_id_to_board_list[board_id]) # Fail if a board for the requested project is not attached if boards_explicitly_specified: for project in projects_to_test: if project.get_test_boards() is None: print("No test board(s) for project %s" % project.get_name()) exit(-1) # Build all projects if not args.nobuild: for project in projects_to_test: project.build() # Test all projects with boards that are attached test_passed = True tested_projects = [] untested_projects = [] for project in projects_to_test: if project.get_test_boards() is not None: project.test_set_first_board_only(args.testfirst) project.test_set_load_if(not args.noloadif) project.test_set_load_bl(not args.noloadbl) project.test_set_test_ep(not args.notestendpt) test_passed &= project.test() tested_projects.append(project) else: # Cannot test board untested_projects.append(project) assert (len(tested_projects) + len(untested_projects) == len(projects_to_test)) if len(tested_projects) == 0: print("Test Failed - no connected boards to test") exit(-1) if boards_explicitly_specified: # Error should have been triggered before this # point if there were boards that couldn't be tested assert len(untested_projects) == 0 # Print info for boards tested for project in tested_projects: print('') if args.verbose == VERB_MINIMAL: project.get_test_info().print_msg(TestInfo.FAILURE, 0) elif args.verbose == VERB_NORMAL: project.get_test_info().print_msg(TestInfo.WARNING, None) elif args.verbose == VERB_VERBOSE: project.get_test_info().print_msg(TestInfo.WARNING, None) elif args.verbose == VERB_ALL: project.get_test_info().print_msg(TestInfo.INFO, None) else: # This should never happen assert False # Warn about untested boards print('') for project in untested_projects: print('Warning - project %s is untested' % project.get_name()) if test_passed: print("All boards passed") exit(0) else: print("Test Failed") exit(-1)