Esempio n. 1
0
    def test_list_binaries_for_builds(self):
        root_path = os.path.dirname(os.path.realpath(__file__))
        spec_path = os.path.join(root_path, "resources", "test_spec.json")

        spec = tests_spec.TestSpec(spec_path)

        for verbose in [True, False]:
            # Capture logging output
            sys.stdout = stdout_capture = six.StringIO()
            cmake_handlers.list_binaries_for_builds(spec,
                                                    verbose_footer=verbose)
            sys.stdout = sys.__stdout__

            output = stdout_capture.getvalue()
            self.assertTrue("available tests for build 'K64F-ARM'" in output)
            self.assertTrue("available tests for build 'K64F-GCC'" in output)
            self.assertTrue("tests-example-1" in output)
            self.assertTrue("tests-example-2" in output)
            self.assertTrue("tests-example-7" in output)

            if verbose == True:
                self.assertTrue("Example: execute" in output)
Esempio n. 2
0
def create_filtered_test_list(ctest_test_list,
                              test_by_names,
                              skip_test,
                              test_spec=None):
    """! Filters test case list (filtered with switch -n) and return filtered list.
    @ctest_test_list List iof tests, originally from CTestTestFile.cmake in yotta module. Now comes from test specification
    @test_by_names Command line switch -n <test_by_names>
    @skip_test Command line switch -i <skip_test>
    @param test_spec Test specification object loaded with --test-spec switch
    @return
    """
    def filter_names_by_prefix(test_case_name_list, prefix_name):
        """!
        @param test_case_name_list List of all test cases
        @param prefix_name Prefix of test name we are looking for
        @result Set with names of test names starting with 'prefix_name'
        """
        result = list()
        for test_name in test_case_name_list:
            if test_name.startswith(prefix_name):
                result.append(test_name)
        return sorted(result)

    filtered_ctest_test_list = ctest_test_list
    test_list = None
    invalid_test_names = []
    if filtered_ctest_test_list is None:
        return {}

    if test_by_names:
        filtered_ctest_test_list = {}  # Subset of 'ctest_test_list'
        test_list = test_by_names.split(',')
        gt_logger.gt_log("test case filter (specified with -n option)")

        for test_name in set(test_list):
            if test_name.endswith('*'):
                # This 'star-sufix' filter allows users to filter tests with fixed prefixes
                # Example: -n 'TESTS-mbed_drivers* will filter all test cases with name starting with 'TESTS-mbed_drivers'
                for test_name_filtered in filter_names_by_prefix(
                        ctest_test_list.keys(), test_name[:-1]):
                    gt_logger.gt_log_tab(
                        "test filtered in '%s'" %
                        gt_logger.gt_bright(test_name_filtered))
                    filtered_ctest_test_list[
                        test_name_filtered] = ctest_test_list[
                            test_name_filtered]
            elif test_name not in ctest_test_list:
                invalid_test_names.append(test_name)
            else:
                gt_logger.gt_log_tab("test filtered in '%s'" %
                                     gt_logger.gt_bright(test_name))
                filtered_ctest_test_list[test_name] = ctest_test_list[
                    test_name]

    if skip_test:
        test_list = skip_test.split(',')
        gt_logger.gt_log("test case filter (specified with -i option)")

        for test_name in set(test_list):
            if test_name not in ctest_test_list:
                invalid_test_names.append(test_name)
            else:
                gt_logger.gt_log_tab("test filtered out '%s'" %
                                     gt_logger.gt_bright(test_name))
                del filtered_ctest_test_list[test_name]

    if invalid_test_names:
        opt_to_print = '-n' if test_by_names else 'skip-test'
        gt_logger.gt_log_warn(
            "invalid test case names (specified with '%s' option)" %
            opt_to_print)
        for test_name in invalid_test_names:
            if test_spec:
                test_spec_name = test_spec.test_spec_filename
                gt_logger.gt_log_warn(
                    "test name '%s' not found in '%s' (specified with --test-spec option)"
                    % (gt_logger.gt_bright(test_name),
                       gt_logger.gt_bright(test_spec_name)))
            else:
                gt_logger.gt_log_warn(
                    "test name '%s' not found in CTestTestFile.cmake (specified with '%s' option)"
                    % (gt_logger.gt_bright(test_name), opt_to_print))
        gt_logger.gt_log_tab("note: test case names are case sensitive")
        gt_logger.gt_log_tab("note: see list of available test cases below")
        # Print available test suite names (binary names user can use with -n
        if test_spec:
            list_binaries_for_builds(test_spec)
        else:
            list_binaries_for_targets()

    return filtered_ctest_test_list
Esempio n. 3
0
def get_test_spec(opts):
    """! Closure encapsulating how we get test specification and load it from file of from yotta module
    @return Returns tuple of (test specification, ret code). Test specification == None if test spec load was not successful
    """
    test_spec = None

    # Check if test_spec.json file exist, if so we will pick it up as default file and load it
    test_spec_file_name = opts.test_spec
    test_spec_file_name_list = []

    # Note: test_spec.json will have higher priority than module.json file
    #       so if we are inside directory with module.json and test_spec.json we will use test spec file
    #       instead of using yotta's module.json file

    def get_all_test_specs_from_build_dir(path_to_scan):
        """! Searches for all test_spec.json files
        @param path_to_scan Directory path used to recursively search for test_spec.json
        @result List of locations of test_spec.json
        """
        return [os.path.join(dp, f) for dp, dn, filenames in os.walk(path_to_scan) for f in filenames if f == 'test_spec.json']

    def merge_multiple_test_specifications_from_file_list(test_spec_file_name_list):
        """! For each file in test_spec_file_name_list merge all test specifications into one
        @param test_spec_file_name_list List of paths to different test specifications
        @return TestSpec object with all test specification data inside
        """

        def copy_builds_between_test_specs(source, destination):
            """! Copies build key-value pairs between two test_spec dicts
                @param source Source dictionary
                @param destination Dictionary with will be applied with 'builds' key-values
                @return Dictionary with merged source
            """
            result = destination.copy()
            if 'builds' in source and 'builds' in destination:
                for k in source['builds']:
                    result['builds'][k] = source['builds'][k]
            return result

        merged_test_spec = {}
        for test_spec_file in test_spec_file_name_list:
            gt_logger.gt_log_tab("using '%s'"% test_spec_file)
            try:
                with open(test_spec_file, 'r') as f:
                    test_spec_data = json.load(f)
                    merged_test_spec = copy_builds_between_test_specs(merged_test_spec, test_spec_data)
            except Exception as e:
                gt_logger.gt_log_err("Unexpected error while processing '%s' test specification file"% test_spec_file)
                gt_logger.gt_log_tab(str(e))
                merged_test_spec = {}

        test_spec = TestSpec()
        test_spec.parse(merged_test_spec)
        return test_spec

    # Test specification look-up
    if opts.test_spec:
        # Loading test specification from command line specified file
        gt_logger.gt_log("test specification file '%s' (specified with --test-spec option)"% opts.test_spec)
    elif os.path.exists('test_spec.json'):
        # Test specification file exists in current directory
        gt_logger.gt_log("using 'test_spec.json' from current directory!")
        test_spec_file_name = 'test_spec.json'
    elif os.path.exists('.build'):
        # Checking .build directory for test specifications
        test_spec_file_name_list = get_all_test_specs_from_build_dir('.build')
    elif os.path.exists(os.path.join('mbed-os', '.build')):
        # Checking mbed-os/.build directory for test specifications
        test_spec_file_name_list = get_all_test_specs_from_build_dir(os.path.join(['mbed-os', '.build']))

    # Actual load and processing of test specification from sources
    if test_spec_file_name:
        # Test specification from command line (--test-spec) or default test_spec.json will be used
        gt_logger.gt_log("using '%s' from current directory!"% test_spec_file_name)
        test_spec = TestSpec(test_spec_file_name)
        if opts.list_binaries:
            list_binaries_for_builds(test_spec)
            return None, 0
    elif test_spec_file_name_list:
        # Merge multiple test specs into one and keep calm
        gt_logger.gt_log("using multiple test specifications from current directory!")
        test_spec = merge_multiple_test_specifications_from_file_list(test_spec_file_name_list)
        if opts.list_binaries:
            list_binaries_for_builds(test_spec)
            return None, 0
    elif os.path.exists('module.json'):
        # If inside yotta module load module data and generate test spec
        gt_logger.gt_log("using 'module.json' from current directory!")
        if opts.list_binaries:
            # List available test binaries (names, no extension)
            list_binaries_for_targets()
            return None, 0
        else:
            test_spec = get_test_spec_from_yt_module(opts)
    else:
        gt_logger.gt_log_err("greentea should be run inside a Yotta module or --test-spec switch should be used")
        return None, -1
    return test_spec, 0
Esempio n. 4
0
def create_filtered_test_list(ctest_test_list, test_by_names, skip_test, test_spec=None):
    """! Filters test case list (filtered with switch -n) and return filtered list.
    @ctest_test_list List iof tests, originally from CTestTestFile.cmake in yotta module. Now comes from test specification
    @test_by_names Command line switch -n <test_by_names>
    @skip_test Command line switch -i <skip_test>
    @param test_spec Test specification object loaded with --test-spec switch
    @return
    """

    def filter_names_by_prefix(test_case_name_list, prefix_name):
        """!
        @param test_case_name_list List of all test cases
        @param prefix_name Prefix of test name we are looking for
        @result Set with names of test names starting with 'prefix_name'
        """
        result = list()
        for test_name in test_case_name_list:
            if test_name.startswith(prefix_name):
                result.append(test_name)
        return sorted(result)

    filtered_ctest_test_list = ctest_test_list
    test_list = None
    invalid_test_names = []
    if filtered_ctest_test_list is None:
        return {}

    if test_by_names:
        filtered_ctest_test_list = {}  # Subset of 'ctest_test_list'
        test_list = test_by_names.split(",")
        gt_logger.gt_log("test case filter (specified with -n option)")

        for test_name in set(test_list):
            if test_name.endswith("*"):
                # This 'star-sufix' filter allows users to filter tests with fixed prefixes
                # Example: -n 'TESTS-mbed_drivers* will filter all test cases with name starting with 'TESTS-mbed_drivers'
                for test_name_filtered in filter_names_by_prefix(ctest_test_list.keys(), test_name[:-1]):
                    gt_logger.gt_log_tab("test filtered in '%s'" % gt_logger.gt_bright(test_name_filtered))
                    filtered_ctest_test_list[test_name_filtered] = ctest_test_list[test_name_filtered]
            elif test_name not in ctest_test_list:
                invalid_test_names.append(test_name)
            else:
                gt_logger.gt_log_tab("test filtered in '%s'" % gt_logger.gt_bright(test_name))
                filtered_ctest_test_list[test_name] = ctest_test_list[test_name]

    if skip_test:
        test_list = skip_test.split(",")
        gt_logger.gt_log("test case filter (specified with -i option)")

        for test_name in set(test_list):
            if test_name not in ctest_test_list:
                invalid_test_names.append(test_name)
            else:
                gt_logger.gt_log_tab("test filtered out '%s'" % gt_logger.gt_bright(test_name))
                del filtered_ctest_test_list[test_name]

    if invalid_test_names:
        opt_to_print = "-n" if test_by_names else "skip-test"
        gt_logger.gt_log_warn("invalid test case names (specified with '%s' option)" % opt_to_print)
        for test_name in invalid_test_names:
            if test_spec:
                test_spec_name = test_spec.test_spec_filename
                gt_logger.gt_log_warn(
                    "test name '%s' not found in '%s' (specified with --test-spec option)"
                    % (gt_logger.gt_bright(test_name), gt_logger.gt_bright(test_spec_name))
                )
            else:
                gt_logger.gt_log_warn(
                    "test name '%s' not found in CTestTestFile.cmake (specified with '%s' option)"
                    % (gt_logger.gt_bright(test_name), opt_to_print)
                )
        gt_logger.gt_log_tab("note: test case names are case sensitive")
        gt_logger.gt_log_tab("note: see list of available test cases below")
        # Print available test suite names (binary names user can use with -n
        if test_spec:
            list_binaries_for_builds(test_spec)
        else:
            list_binaries_for_targets()

    return filtered_ctest_test_list
Esempio n. 5
0
def get_test_spec(opts):
    """! Closure encapsulating how we get test specification and load it from file of from yotta module
    @return Returns tuple of (test specification, ret code). Test specification == None if test spec load was not successful
    """
    test_spec = None

    # Check if test_spec.json file exist, if so we will pick it up as default file and load it
    test_spec_file_name = opts.test_spec
    test_spec_file_name_list = []

    # Note: test_spec.json will have higher priority than module.json file
    #       so if we are inside directory with module.json and test_spec.json we will use test spec file
    #       instead of using yotta's module.json file

    def get_all_test_specs_from_build_dir(path_to_scan):
        """! Searches for all test_spec.json files
        @param path_to_scan Directory path used to recursively search for test_spec.json
        @result List of locations of test_spec.json
        """
        return [
            os.path.join(dp, f) for dp, dn, filenames in os.walk(path_to_scan)
            for f in filenames if f == 'test_spec.json'
        ]

    def merge_multiple_test_specifications_from_file_list(
            test_spec_file_name_list):
        """! For each file in test_spec_file_name_list merge all test specifications into one
        @param test_spec_file_name_list List of paths to different test specifications
        @return TestSpec object with all test specification data inside
        """
        def copy_builds_between_test_specs(source, destination):
            """! Copies build key-value pairs between two test_spec dicts
                @param source Source dictionary
                @param destination Dictionary with will be applied with 'builds' key-values
                @return Dictionary with merged source
            """
            result = destination.copy()
            if 'builds' in source and 'builds' in destination:
                for k in source['builds']:
                    result['builds'][k] = source['builds'][k]
            return result

        merged_test_spec = {}
        for test_spec_file in test_spec_file_name_list:
            gt_logger.gt_log_tab("using '%s'" % test_spec_file)
            try:
                with open(test_spec_file, 'r') as f:
                    test_spec_data = json.load(f)
                    merged_test_spec = copy_builds_between_test_specs(
                        merged_test_spec, test_spec_data)
            except Exception as e:
                gt_logger.gt_log_err(
                    "Unexpected error while processing '%s' test specification file"
                    % test_spec_file)
                gt_logger.gt_log_tab(str(e))
                merged_test_spec = {}

        test_spec = TestSpec()
        test_spec.parse(merged_test_spec)
        return test_spec

    # Test specification look-up
    if opts.test_spec:
        # Loading test specification from command line specified file
        gt_logger.gt_log(
            "test specification file '%s' (specified with --test-spec option)"
            % opts.test_spec)
    elif os.path.exists('test_spec.json'):
        # Test specification file exists in current directory
        gt_logger.gt_log("using 'test_spec.json' from current directory!")
        test_spec_file_name = 'test_spec.json'
    elif 'BUILD' in os.listdir(os.getcwd()):
        # Checking 'BUILD' directory for test specifications
        # Using `os.listdir()` since it preserves case
        test_spec_file_name_list = get_all_test_specs_from_build_dir('BUILD')
    elif os.path.exists('.build'):
        # Checking .build directory for test specifications
        test_spec_file_name_list = get_all_test_specs_from_build_dir('.build')
    elif os.path.exists('mbed-os') and 'BUILD' in os.listdir('mbed-os'):
        # Checking mbed-os/.build directory for test specifications
        # Using `os.listdir()` since it preserves case
        test_spec_file_name_list = get_all_test_specs_from_build_dir(
            os.path.join(['mbed-os', 'BUILD']))
    elif os.path.exists(os.path.join('mbed-os', '.build')):
        # Checking mbed-os/.build directory for test specifications
        test_spec_file_name_list = get_all_test_specs_from_build_dir(
            os.path.join(['mbed-os', '.build']))

    # Actual load and processing of test specification from sources
    if test_spec_file_name:
        # Test specification from command line (--test-spec) or default test_spec.json will be used
        gt_logger.gt_log("using '%s' from current directory!" %
                         test_spec_file_name)
        test_spec = TestSpec(test_spec_file_name)
        if opts.list_binaries:
            list_binaries_for_builds(test_spec)
            return None, 0
    elif test_spec_file_name_list:
        # Merge multiple test specs into one and keep calm
        gt_logger.gt_log(
            "using multiple test specifications from current directory!")
        test_spec = merge_multiple_test_specifications_from_file_list(
            test_spec_file_name_list)
        if opts.list_binaries:
            list_binaries_for_builds(test_spec)
            return None, 0
    elif os.path.exists('module.json'):
        # If inside yotta module load module data and generate test spec
        gt_logger.gt_log("using 'module.json' from current directory!")
        if opts.list_binaries:
            # List available test binaries (names, no extension)
            list_binaries_for_targets()
            return None, 0
        else:
            test_spec = get_test_spec_from_yt_module(opts)
    else:
        gt_logger.gt_log_err(
            "greentea should be run inside a Yotta module or --test-spec switch should be used"
        )
        return None, -1
    return test_spec, 0