def test_list_binaries_for_builds(self): root_path = os.path.dirname(os.path.realpath(__file__)) spec_path = os.path.join(root_path, "resources", "test_spec.json") spec = tests_spec.TestSpec(spec_path) for verbose in [True, False]: # Capture logging output sys.stdout = stdout_capture = six.StringIO() cmake_handlers.list_binaries_for_builds(spec, verbose_footer=verbose) sys.stdout = sys.__stdout__ output = stdout_capture.getvalue() self.assertTrue("available tests for build 'K64F-ARM'" in output) self.assertTrue("available tests for build 'K64F-GCC'" in output) self.assertTrue("tests-example-1" in output) self.assertTrue("tests-example-2" in output) self.assertTrue("tests-example-7" in output) if verbose == True: self.assertTrue("Example: execute" in output)
def create_filtered_test_list(ctest_test_list, test_by_names, skip_test, test_spec=None): """! Filters test case list (filtered with switch -n) and return filtered list. @ctest_test_list List iof tests, originally from CTestTestFile.cmake in yotta module. Now comes from test specification @test_by_names Command line switch -n <test_by_names> @skip_test Command line switch -i <skip_test> @param test_spec Test specification object loaded with --test-spec switch @return """ def filter_names_by_prefix(test_case_name_list, prefix_name): """! @param test_case_name_list List of all test cases @param prefix_name Prefix of test name we are looking for @result Set with names of test names starting with 'prefix_name' """ result = list() for test_name in test_case_name_list: if test_name.startswith(prefix_name): result.append(test_name) return sorted(result) filtered_ctest_test_list = ctest_test_list test_list = None invalid_test_names = [] if filtered_ctest_test_list is None: return {} if test_by_names: filtered_ctest_test_list = {} # Subset of 'ctest_test_list' test_list = test_by_names.split(',') gt_logger.gt_log("test case filter (specified with -n option)") for test_name in set(test_list): if test_name.endswith('*'): # This 'star-sufix' filter allows users to filter tests with fixed prefixes # Example: -n 'TESTS-mbed_drivers* will filter all test cases with name starting with 'TESTS-mbed_drivers' for test_name_filtered in filter_names_by_prefix( ctest_test_list.keys(), test_name[:-1]): gt_logger.gt_log_tab( "test filtered in '%s'" % gt_logger.gt_bright(test_name_filtered)) filtered_ctest_test_list[ test_name_filtered] = ctest_test_list[ test_name_filtered] elif test_name not in ctest_test_list: invalid_test_names.append(test_name) else: gt_logger.gt_log_tab("test filtered in '%s'" % gt_logger.gt_bright(test_name)) filtered_ctest_test_list[test_name] = ctest_test_list[ test_name] if skip_test: test_list = skip_test.split(',') gt_logger.gt_log("test case filter (specified with -i option)") for test_name in set(test_list): if test_name not in ctest_test_list: invalid_test_names.append(test_name) else: gt_logger.gt_log_tab("test filtered out '%s'" % gt_logger.gt_bright(test_name)) del filtered_ctest_test_list[test_name] if invalid_test_names: opt_to_print = '-n' if test_by_names else 'skip-test' gt_logger.gt_log_warn( "invalid test case names (specified with '%s' option)" % opt_to_print) for test_name in invalid_test_names: if test_spec: test_spec_name = test_spec.test_spec_filename gt_logger.gt_log_warn( "test name '%s' not found in '%s' (specified with --test-spec option)" % (gt_logger.gt_bright(test_name), gt_logger.gt_bright(test_spec_name))) else: gt_logger.gt_log_warn( "test name '%s' not found in CTestTestFile.cmake (specified with '%s' option)" % (gt_logger.gt_bright(test_name), opt_to_print)) gt_logger.gt_log_tab("note: test case names are case sensitive") gt_logger.gt_log_tab("note: see list of available test cases below") # Print available test suite names (binary names user can use with -n if test_spec: list_binaries_for_builds(test_spec) else: list_binaries_for_targets() return filtered_ctest_test_list
def get_test_spec(opts): """! Closure encapsulating how we get test specification and load it from file of from yotta module @return Returns tuple of (test specification, ret code). Test specification == None if test spec load was not successful """ test_spec = None # Check if test_spec.json file exist, if so we will pick it up as default file and load it test_spec_file_name = opts.test_spec test_spec_file_name_list = [] # Note: test_spec.json will have higher priority than module.json file # so if we are inside directory with module.json and test_spec.json we will use test spec file # instead of using yotta's module.json file def get_all_test_specs_from_build_dir(path_to_scan): """! Searches for all test_spec.json files @param path_to_scan Directory path used to recursively search for test_spec.json @result List of locations of test_spec.json """ return [os.path.join(dp, f) for dp, dn, filenames in os.walk(path_to_scan) for f in filenames if f == 'test_spec.json'] def merge_multiple_test_specifications_from_file_list(test_spec_file_name_list): """! For each file in test_spec_file_name_list merge all test specifications into one @param test_spec_file_name_list List of paths to different test specifications @return TestSpec object with all test specification data inside """ def copy_builds_between_test_specs(source, destination): """! Copies build key-value pairs between two test_spec dicts @param source Source dictionary @param destination Dictionary with will be applied with 'builds' key-values @return Dictionary with merged source """ result = destination.copy() if 'builds' in source and 'builds' in destination: for k in source['builds']: result['builds'][k] = source['builds'][k] return result merged_test_spec = {} for test_spec_file in test_spec_file_name_list: gt_logger.gt_log_tab("using '%s'"% test_spec_file) try: with open(test_spec_file, 'r') as f: test_spec_data = json.load(f) merged_test_spec = copy_builds_between_test_specs(merged_test_spec, test_spec_data) except Exception as e: gt_logger.gt_log_err("Unexpected error while processing '%s' test specification file"% test_spec_file) gt_logger.gt_log_tab(str(e)) merged_test_spec = {} test_spec = TestSpec() test_spec.parse(merged_test_spec) return test_spec # Test specification look-up if opts.test_spec: # Loading test specification from command line specified file gt_logger.gt_log("test specification file '%s' (specified with --test-spec option)"% opts.test_spec) elif os.path.exists('test_spec.json'): # Test specification file exists in current directory gt_logger.gt_log("using 'test_spec.json' from current directory!") test_spec_file_name = 'test_spec.json' elif os.path.exists('.build'): # Checking .build directory for test specifications test_spec_file_name_list = get_all_test_specs_from_build_dir('.build') elif os.path.exists(os.path.join('mbed-os', '.build')): # Checking mbed-os/.build directory for test specifications test_spec_file_name_list = get_all_test_specs_from_build_dir(os.path.join(['mbed-os', '.build'])) # Actual load and processing of test specification from sources if test_spec_file_name: # Test specification from command line (--test-spec) or default test_spec.json will be used gt_logger.gt_log("using '%s' from current directory!"% test_spec_file_name) test_spec = TestSpec(test_spec_file_name) if opts.list_binaries: list_binaries_for_builds(test_spec) return None, 0 elif test_spec_file_name_list: # Merge multiple test specs into one and keep calm gt_logger.gt_log("using multiple test specifications from current directory!") test_spec = merge_multiple_test_specifications_from_file_list(test_spec_file_name_list) if opts.list_binaries: list_binaries_for_builds(test_spec) return None, 0 elif os.path.exists('module.json'): # If inside yotta module load module data and generate test spec gt_logger.gt_log("using 'module.json' from current directory!") if opts.list_binaries: # List available test binaries (names, no extension) list_binaries_for_targets() return None, 0 else: test_spec = get_test_spec_from_yt_module(opts) else: gt_logger.gt_log_err("greentea should be run inside a Yotta module or --test-spec switch should be used") return None, -1 return test_spec, 0
def create_filtered_test_list(ctest_test_list, test_by_names, skip_test, test_spec=None): """! Filters test case list (filtered with switch -n) and return filtered list. @ctest_test_list List iof tests, originally from CTestTestFile.cmake in yotta module. Now comes from test specification @test_by_names Command line switch -n <test_by_names> @skip_test Command line switch -i <skip_test> @param test_spec Test specification object loaded with --test-spec switch @return """ def filter_names_by_prefix(test_case_name_list, prefix_name): """! @param test_case_name_list List of all test cases @param prefix_name Prefix of test name we are looking for @result Set with names of test names starting with 'prefix_name' """ result = list() for test_name in test_case_name_list: if test_name.startswith(prefix_name): result.append(test_name) return sorted(result) filtered_ctest_test_list = ctest_test_list test_list = None invalid_test_names = [] if filtered_ctest_test_list is None: return {} if test_by_names: filtered_ctest_test_list = {} # Subset of 'ctest_test_list' test_list = test_by_names.split(",") gt_logger.gt_log("test case filter (specified with -n option)") for test_name in set(test_list): if test_name.endswith("*"): # This 'star-sufix' filter allows users to filter tests with fixed prefixes # Example: -n 'TESTS-mbed_drivers* will filter all test cases with name starting with 'TESTS-mbed_drivers' for test_name_filtered in filter_names_by_prefix(ctest_test_list.keys(), test_name[:-1]): gt_logger.gt_log_tab("test filtered in '%s'" % gt_logger.gt_bright(test_name_filtered)) filtered_ctest_test_list[test_name_filtered] = ctest_test_list[test_name_filtered] elif test_name not in ctest_test_list: invalid_test_names.append(test_name) else: gt_logger.gt_log_tab("test filtered in '%s'" % gt_logger.gt_bright(test_name)) filtered_ctest_test_list[test_name] = ctest_test_list[test_name] if skip_test: test_list = skip_test.split(",") gt_logger.gt_log("test case filter (specified with -i option)") for test_name in set(test_list): if test_name not in ctest_test_list: invalid_test_names.append(test_name) else: gt_logger.gt_log_tab("test filtered out '%s'" % gt_logger.gt_bright(test_name)) del filtered_ctest_test_list[test_name] if invalid_test_names: opt_to_print = "-n" if test_by_names else "skip-test" gt_logger.gt_log_warn("invalid test case names (specified with '%s' option)" % opt_to_print) for test_name in invalid_test_names: if test_spec: test_spec_name = test_spec.test_spec_filename gt_logger.gt_log_warn( "test name '%s' not found in '%s' (specified with --test-spec option)" % (gt_logger.gt_bright(test_name), gt_logger.gt_bright(test_spec_name)) ) else: gt_logger.gt_log_warn( "test name '%s' not found in CTestTestFile.cmake (specified with '%s' option)" % (gt_logger.gt_bright(test_name), opt_to_print) ) gt_logger.gt_log_tab("note: test case names are case sensitive") gt_logger.gt_log_tab("note: see list of available test cases below") # Print available test suite names (binary names user can use with -n if test_spec: list_binaries_for_builds(test_spec) else: list_binaries_for_targets() return filtered_ctest_test_list
def get_test_spec(opts): """! Closure encapsulating how we get test specification and load it from file of from yotta module @return Returns tuple of (test specification, ret code). Test specification == None if test spec load was not successful """ test_spec = None # Check if test_spec.json file exist, if so we will pick it up as default file and load it test_spec_file_name = opts.test_spec test_spec_file_name_list = [] # Note: test_spec.json will have higher priority than module.json file # so if we are inside directory with module.json and test_spec.json we will use test spec file # instead of using yotta's module.json file def get_all_test_specs_from_build_dir(path_to_scan): """! Searches for all test_spec.json files @param path_to_scan Directory path used to recursively search for test_spec.json @result List of locations of test_spec.json """ return [ os.path.join(dp, f) for dp, dn, filenames in os.walk(path_to_scan) for f in filenames if f == 'test_spec.json' ] def merge_multiple_test_specifications_from_file_list( test_spec_file_name_list): """! For each file in test_spec_file_name_list merge all test specifications into one @param test_spec_file_name_list List of paths to different test specifications @return TestSpec object with all test specification data inside """ def copy_builds_between_test_specs(source, destination): """! Copies build key-value pairs between two test_spec dicts @param source Source dictionary @param destination Dictionary with will be applied with 'builds' key-values @return Dictionary with merged source """ result = destination.copy() if 'builds' in source and 'builds' in destination: for k in source['builds']: result['builds'][k] = source['builds'][k] return result merged_test_spec = {} for test_spec_file in test_spec_file_name_list: gt_logger.gt_log_tab("using '%s'" % test_spec_file) try: with open(test_spec_file, 'r') as f: test_spec_data = json.load(f) merged_test_spec = copy_builds_between_test_specs( merged_test_spec, test_spec_data) except Exception as e: gt_logger.gt_log_err( "Unexpected error while processing '%s' test specification file" % test_spec_file) gt_logger.gt_log_tab(str(e)) merged_test_spec = {} test_spec = TestSpec() test_spec.parse(merged_test_spec) return test_spec # Test specification look-up if opts.test_spec: # Loading test specification from command line specified file gt_logger.gt_log( "test specification file '%s' (specified with --test-spec option)" % opts.test_spec) elif os.path.exists('test_spec.json'): # Test specification file exists in current directory gt_logger.gt_log("using 'test_spec.json' from current directory!") test_spec_file_name = 'test_spec.json' elif 'BUILD' in os.listdir(os.getcwd()): # Checking 'BUILD' directory for test specifications # Using `os.listdir()` since it preserves case test_spec_file_name_list = get_all_test_specs_from_build_dir('BUILD') elif os.path.exists('.build'): # Checking .build directory for test specifications test_spec_file_name_list = get_all_test_specs_from_build_dir('.build') elif os.path.exists('mbed-os') and 'BUILD' in os.listdir('mbed-os'): # Checking mbed-os/.build directory for test specifications # Using `os.listdir()` since it preserves case test_spec_file_name_list = get_all_test_specs_from_build_dir( os.path.join(['mbed-os', 'BUILD'])) elif os.path.exists(os.path.join('mbed-os', '.build')): # Checking mbed-os/.build directory for test specifications test_spec_file_name_list = get_all_test_specs_from_build_dir( os.path.join(['mbed-os', '.build'])) # Actual load and processing of test specification from sources if test_spec_file_name: # Test specification from command line (--test-spec) or default test_spec.json will be used gt_logger.gt_log("using '%s' from current directory!" % test_spec_file_name) test_spec = TestSpec(test_spec_file_name) if opts.list_binaries: list_binaries_for_builds(test_spec) return None, 0 elif test_spec_file_name_list: # Merge multiple test specs into one and keep calm gt_logger.gt_log( "using multiple test specifications from current directory!") test_spec = merge_multiple_test_specifications_from_file_list( test_spec_file_name_list) if opts.list_binaries: list_binaries_for_builds(test_spec) return None, 0 elif os.path.exists('module.json'): # If inside yotta module load module data and generate test spec gt_logger.gt_log("using 'module.json' from current directory!") if opts.list_binaries: # List available test binaries (names, no extension) list_binaries_for_targets() return None, 0 else: test_spec = get_test_spec_from_yt_module(opts) else: gt_logger.gt_log_err( "greentea should be run inside a Yotta module or --test-spec switch should be used" ) return None, -1 return test_spec, 0