コード例 #1
0
ファイル: test_registry.py プロジェクト: xB-2048/checkov
 def test_load_local_module_absolute_path(self):
     registry = ModuleLoaderRegistry(download_external_modules=True)
     source = "/some/module"
     try:
         registry.load(current_dir=self.current_dir, source=source, source_version="latest")
         self.assertEqual(1, 2, 'Module loading should have thrown an error')
     except FileNotFoundError as e:
         self.assertEqual(str(e), source)
コード例 #2
0
ファイル: test_registry.py プロジェクト: xB-2048/checkov
 def test_load_terraform_registry_check_cache(self):
     registry = ModuleLoaderRegistry(download_external_modules=True)
     source1 = "https://github.com/bridgecrewio/checkov_not_working1.git"
     registry.load(current_dir=self.current_dir, source=source1, source_version="latest")
     self.assertTrue(source1 in registry.failed_urls_cache)
     source2 = "https://github.com/bridgecrewio/checkov_not_working2.git"
     registry.load(current_dir=self.current_dir, source=source2, source_version="latest")
     self.assertTrue(source1 in registry.failed_urls_cache and source2 in registry.failed_urls_cache)
コード例 #3
0
ファイル: test_registry.py プロジェクト: tsmithv11/checkov
def test_load_terraform_registry(
    git_getter,
    source,
    source_version,
    expected_content_path,
    expected_git_url,
    expected_dest_dir,
    expected_module_source,
    expected_inner_module,
):
    # given
    current_dir = Path(__file__).parent / "tmp"
    registry = ModuleLoaderRegistry(download_external_modules=True)

    # when
    content = registry.load(current_dir=str(current_dir),
                            source=source,
                            source_version=source_version)

    # then
    assert content.loaded()
    assert content.path() == str(
        Path(DEFAULT_EXTERNAL_MODULES_DIR) / expected_content_path)

    git_getter.assert_called_once_with(expected_git_url, mock.ANY)

    git_loader = next(loader for loader in registry.loaders
                      if isinstance(loader, GenericGitLoader))
    assert git_loader.dest_dir == str(
        Path(DEFAULT_EXTERNAL_MODULES_DIR) / expected_dest_dir)
    assert git_loader.module_source == expected_module_source
    assert git_loader.inner_module == expected_inner_module
コード例 #4
0
ファイル: test_registry.py プロジェクト: xB-2048/checkov
 def test_load_terraform_registry(self):
     registry = ModuleLoaderRegistry(True, DEFAULT_EXTERNAL_MODULES_DIR)
     source = "terraform-aws-modules/security-group/aws"
     with registry.load(current_dir=self.current_dir, source=source, source_version="~> 3.0") as content:
         assert content.loaded()
         expected_content_path = os.path.join(self.current_dir, DEFAULT_EXTERNAL_MODULES_DIR, source)
         self.assertEqual(expected_content_path, content.path(), f"expected content.path() to be {content.path()}, got {expected_content_path}")
コード例 #5
0
ファイル: test_registry.py プロジェクト: jakuta-tech/checkov
 def test_load_terraform_registry(self):
     registry = ModuleLoaderRegistry(True, DEFAULT_EXTERNAL_MODULES_DIR)
     registry.root_dir = self.current_dir
     source = "terraform-aws-modules/security-group/aws"
     content = registry.load(current_dir=self.current_dir, source=source, source_version="~> 3.0")
     assert content.loaded()
     expected_content_path = os.path.join(
         self.current_dir,
         DEFAULT_EXTERNAL_MODULES_DIR,
         "github.com/terraform-aws-modules/terraform-aws-security-group",
     )
     self.assertRegex(content.path(), f"^{expected_content_path}/v3.*")
コード例 #6
0
ファイル: test_registry.py プロジェクト: jakuta-tech/checkov
def test_load_local_path(git_getter, source, expected_content_path, expected_exception):
    # given
    current_dir = Path(__file__).parent
    registry = ModuleLoaderRegistry()

    # when
    with expected_exception:
        content = registry.load(current_dir=str(current_dir), source=source, source_version="latest")

        # then
        assert content.loaded()
        assert content.path() == str(current_dir / expected_content_path)

        git_getter.assert_not_called()
コード例 #7
0
ファイル: parser.py プロジェクト: njgibbon/checkov
    def _load_modules(self,
                      root_dir: str,
                      module_loader_registry: ModuleLoaderRegistry,
                      dir_filter: Callable[[str], bool],
                      module_load_context: Optional[str],
                      keys_referenced_as_modules: Set[str],
                      ignore_unresolved_params: bool = False) -> bool:
        """
        Load modules which have not already been loaded and can be loaded (don't have unresolved parameters).

        :param ignore_unresolved_params:    If true, not-yet-loaded modules will be loaded even if they are
                                            passed parameters that are not fully resolved.
        :return:                            True if there were modules that were not loaded due to unresolved
                                            parameters.
        """
        all_module_definitions = {}
        all_module_evaluations_context = {}
        skipped_a_module = False
        for file in list(self.out_definitions.keys()):
            # Don't process a file in a directory other than the directory we're processing. For example,
            # if we're down dealing with <top_dir>/<module>/something.tf, we don't want to rescan files
            # up in <top_dir>.
            if os.path.dirname(file) != root_dir:
                continue
            # Don't process a file reference which has already been processed
            if file.endswith("]"):
                continue

            file_data = self.out_definitions.get(file)
            if file_data is None:
                continue
            module_calls = file_data.get("module")
            if not module_calls or not isinstance(module_calls, list):
                continue

            for module_index, module_call in enumerate(module_calls):

                if not isinstance(module_call, dict):
                    continue

                # There should only be one module reference per outer dict, but... safety first
                for module_call_name, module_call_data in module_call.items():
                    if not isinstance(module_call_data, dict):
                        continue

                    module_address = (file, module_index, module_call_name)
                    if module_address in self._loaded_modules:
                        continue

                    # Variables being passed to module, "source" and "version" are reserved
                    specified_vars = {
                        k: v[0]
                        for k, v in module_call_data.items()
                        if k != "source" and k != "version"
                    }

                    if not ignore_unresolved_params:
                        has_unresolved_params = False
                        for k, v in specified_vars.items():
                            if not is_acceptable_module_param(
                                    v) or not is_acceptable_module_param(k):
                                has_unresolved_params = True
                                break
                        if has_unresolved_params:
                            skipped_a_module = True
                            continue
                    self._loaded_modules.add(module_address)

                    source = module_call_data.get("source")
                    if not source or not isinstance(source, list):
                        continue
                    source = source[0]
                    if not isinstance(source, str):
                        logging.debug(
                            f"Skipping loading of {module_call_name} as source is not a string, it is: {source}"
                        )
                        continue

                    # Special handling for local sources to make sure we aren't double-parsing
                    if source.startswith("./") or source.startswith("../"):
                        source = os.path.normpath(
                            os.path.join(
                                os.path.dirname(
                                    _remove_module_dependency_in_path(file)),
                                source))

                    version = module_call_data.get("version", "latest")
                    if version and isinstance(version, list):
                        version = version[0]
                    try:
                        with module_loader_registry.load(
                                root_dir, source, version) as content:
                            if not content.loaded():
                                continue

                            self._internal_dir_load(
                                directory=content.path(),
                                module_loader_registry=module_loader_registry,
                                dir_filter=dir_filter,
                                specified_vars=specified_vars,
                                module_load_context=module_load_context,
                                keys_referenced_as_modules=
                                keys_referenced_as_modules)

                            module_definitions = {
                                path: self.out_definitions[path]
                                for path in list(self.out_definitions.keys())
                                if os.path.dirname(path) == content.path()
                            }

                            if not module_definitions:
                                continue

                            # NOTE: Modules are put into the main TF definitions structure "as normal" with the
                            #       notable exception of the file name. For loaded modules referrer information is
                            #       appended to the file name to create this format:
                            #         <file_name>[<referred_file>#<referrer_index>]
                            #       For example:
                            #         /the/path/module/my_module.tf[/the/path/main.tf#0]
                            #       The referrer and index allow a module allow a module to be loaded multiple
                            #       times with differing data.
                            #
                            #       In addition, the referring block will have a "__resolved__" key added with a
                            #       list pointing to the location of the module data that was resolved. For example:
                            #         "__resolved__": ["/the/path/module/my_module.tf[/the/path/main.tf#0]"]

                            resolved_loc_list = module_call_data.get(
                                RESOLVED_MODULE_ENTRY_NAME)
                            if resolved_loc_list is None:
                                resolved_loc_list = []
                                module_call_data[
                                    RESOLVED_MODULE_ENTRY_NAME] = resolved_loc_list

                            # NOTE: Modules can load other modules, so only append referrer information where it
                            #       has not already been added.
                            keys = list(module_definitions.keys())
                            for key in keys:
                                if key.endswith("]") or file.endswith("]"):
                                    continue
                                keys_referenced_as_modules.add(key)
                                new_key = f"{key}[{file}#{module_index}]"
                                module_definitions[
                                    new_key] = module_definitions[key]
                                del module_definitions[key]
                                del self.out_definitions[key]
                                if new_key not in resolved_loc_list:
                                    resolved_loc_list.append(new_key)
                            resolved_loc_list.sort(
                            )  # For testing, need predictable ordering

                            deep_merge.merge(all_module_definitions,
                                             module_definitions)
                    except Exception as e:
                        logging.warning(
                            "Unable to load module (source=\"%s\" version=\"%s\"): %s",
                            source, version, e)
                        pass

        if all_module_definitions:
            deep_merge.merge(self.out_definitions, all_module_definitions)
            deep_merge.merge(self.out_evaluations_context,
                             all_module_evaluations_context)
        return skipped_a_module
コード例 #8
0
    def _load_modules(self, root_dir: str,
                      module_loader_registry: ModuleLoaderRegistry,
                      dir_filter: Callable[[str], bool],
                      module_load_context: Optional[str]):
        all_module_definitions = {}
        all_module_evaluations_context = {}
        for file in list(self.out_definitions.keys()):
            file_data = self.out_definitions.get(file)
            module_calls = file_data.get("module")
            if not module_calls or not isinstance(module_calls, list):
                continue

            for module_index, module_call in enumerate(module_calls):
                if not isinstance(module_call, dict):
                    continue

                # There should only be one module reference per outer dict, but... safety first
                for module_call_name, module_call_data in module_call.items():
                    if not isinstance(module_call_data, dict):
                        continue

                    source = module_call_data.get("source")
                    if not source or not isinstance(source, list):
                        continue
                    source = source[0]

                    # Special handling for local sources to make sure we aren't double-parsing
                    if source.startswith("./") or source.startswith("../"):
                        source = os.path.normpath(
                            os.path.join(
                                os.path.dirname(
                                    _remove_module_dependency_in_path(file)),
                                source))

                    version = module_call_data.get("version", "latest")
                    if version and isinstance(version, list):
                        version = version[0]
                    try:
                        with module_loader_registry.load(
                                root_dir, source, version) as content:
                            if not content.loaded():
                                continue

                            # Variables being passed to module, "source" and "version" are reserved
                            specified_vars = {
                                k: v[0]
                                for k, v in module_call_data.items()
                                if k != "source" and k != "version"
                            }

                            if not dir_filter(os.path.abspath(content.path())):
                                continue
                            self._internal_dir_load(
                                directory=content.path(),
                                module_loader_registry=module_loader_registry,
                                dir_filter=dir_filter,
                                specified_vars=specified_vars,
                                module_load_context=module_load_context)

                            module_definitions = {
                                path: self.out_definitions[path]
                                for path in list(self.out_definitions.keys())
                                if os.path.dirname(path) == content.path()
                            }

                            if not module_definitions:
                                continue

                            # NOTE: Modules are put into the main TF definitions structure "as normal" with the
                            #       notable exception of the file name. For loaded modules referrer information is
                            #       appended to the file name to create this format:
                            #         <file_name>[<referred_file>#<referrer_index>]
                            #       For example:
                            #         /the/path/module/my_module.tf[/the/path/main.tf#0]
                            #       The referrer and index allow a module allow a module to be loaded multiple
                            #       times with differing data.
                            #
                            #       In addition, the referring block will have a "__resolved__" key added with a
                            #       list pointing to the location of the module data that was resolved. For example:
                            #         "__resolved__": ["/the/path/module/my_module.tf[/the/path/main.tf#0]"]

                            resolved_loc_list = module_call_data.get(
                                RESOLVED_MODULE_ENTRY_NAME)
                            if resolved_loc_list is None:
                                resolved_loc_list = []
                                module_call_data[
                                    RESOLVED_MODULE_ENTRY_NAME] = resolved_loc_list

                            # NOTE: Modules can load other modules, so only append referrer information where it
                            #       has not already been added.
                            keys = list(module_definitions.keys())
                            for key in keys:
                                if key.endswith("]"):
                                    continue
                                new_key = f"{key}[{file}#{module_index}]"
                                module_definitions[new_key] = \
                                    module_definitions[key]
                                del module_definitions[key]
                                del self.out_definitions[key]

                                resolved_loc_list.append(new_key)

                            deep_merge.merge(all_module_definitions,
                                             module_definitions)
                    except Exception as e:
                        logging.warning(
                            "Unable to load module (source=\"%s\" version=\"%s\"): %s",
                            source, version, e)
                        pass

        if all_module_definitions:
            deep_merge.merge(self.out_definitions, all_module_definitions)
            deep_merge.merge(self.out_evaluations_context,
                             all_module_evaluations_context)