def test_load_local_module_absolute_path(self): registry = ModuleLoaderRegistry(download_external_modules=True) source = "/some/module" try: registry.load(current_dir=self.current_dir, source=source, source_version="latest") self.assertEqual(1, 2, 'Module loading should have thrown an error') except FileNotFoundError as e: self.assertEqual(str(e), source)
def test_load_terraform_registry_check_cache(self): registry = ModuleLoaderRegistry(download_external_modules=True) source1 = "https://github.com/bridgecrewio/checkov_not_working1.git" registry.load(current_dir=self.current_dir, source=source1, source_version="latest") self.assertTrue(source1 in registry.failed_urls_cache) source2 = "https://github.com/bridgecrewio/checkov_not_working2.git" registry.load(current_dir=self.current_dir, source=source2, source_version="latest") self.assertTrue(source1 in registry.failed_urls_cache and source2 in registry.failed_urls_cache)
def test_load_terraform_registry( git_getter, source, source_version, expected_content_path, expected_git_url, expected_dest_dir, expected_module_source, expected_inner_module, ): # given current_dir = Path(__file__).parent / "tmp" registry = ModuleLoaderRegistry(download_external_modules=True) # when content = registry.load(current_dir=str(current_dir), source=source, source_version=source_version) # then assert content.loaded() assert content.path() == str( Path(DEFAULT_EXTERNAL_MODULES_DIR) / expected_content_path) git_getter.assert_called_once_with(expected_git_url, mock.ANY) git_loader = next(loader for loader in registry.loaders if isinstance(loader, GenericGitLoader)) assert git_loader.dest_dir == str( Path(DEFAULT_EXTERNAL_MODULES_DIR) / expected_dest_dir) assert git_loader.module_source == expected_module_source assert git_loader.inner_module == expected_inner_module
def test_load_terraform_registry(self): registry = ModuleLoaderRegistry(True, DEFAULT_EXTERNAL_MODULES_DIR) source = "terraform-aws-modules/security-group/aws" with registry.load(current_dir=self.current_dir, source=source, source_version="~> 3.0") as content: assert content.loaded() expected_content_path = os.path.join(self.current_dir, DEFAULT_EXTERNAL_MODULES_DIR, source) self.assertEqual(expected_content_path, content.path(), f"expected content.path() to be {content.path()}, got {expected_content_path}")
def test_load_terraform_registry(self): registry = ModuleLoaderRegistry(True, DEFAULT_EXTERNAL_MODULES_DIR) registry.root_dir = self.current_dir source = "terraform-aws-modules/security-group/aws" content = registry.load(current_dir=self.current_dir, source=source, source_version="~> 3.0") assert content.loaded() expected_content_path = os.path.join( self.current_dir, DEFAULT_EXTERNAL_MODULES_DIR, "github.com/terraform-aws-modules/terraform-aws-security-group", ) self.assertRegex(content.path(), f"^{expected_content_path}/v3.*")
def test_load_local_path(git_getter, source, expected_content_path, expected_exception): # given current_dir = Path(__file__).parent registry = ModuleLoaderRegistry() # when with expected_exception: content = registry.load(current_dir=str(current_dir), source=source, source_version="latest") # then assert content.loaded() assert content.path() == str(current_dir / expected_content_path) git_getter.assert_not_called()
def _load_modules(self, root_dir: str, module_loader_registry: ModuleLoaderRegistry, dir_filter: Callable[[str], bool], module_load_context: Optional[str], keys_referenced_as_modules: Set[str], ignore_unresolved_params: bool = False) -> bool: """ Load modules which have not already been loaded and can be loaded (don't have unresolved parameters). :param ignore_unresolved_params: If true, not-yet-loaded modules will be loaded even if they are passed parameters that are not fully resolved. :return: True if there were modules that were not loaded due to unresolved parameters. """ all_module_definitions = {} all_module_evaluations_context = {} skipped_a_module = False for file in list(self.out_definitions.keys()): # Don't process a file in a directory other than the directory we're processing. For example, # if we're down dealing with <top_dir>/<module>/something.tf, we don't want to rescan files # up in <top_dir>. if os.path.dirname(file) != root_dir: continue # Don't process a file reference which has already been processed if file.endswith("]"): continue file_data = self.out_definitions.get(file) if file_data is None: continue module_calls = file_data.get("module") if not module_calls or not isinstance(module_calls, list): continue for module_index, module_call in enumerate(module_calls): if not isinstance(module_call, dict): continue # There should only be one module reference per outer dict, but... safety first for module_call_name, module_call_data in module_call.items(): if not isinstance(module_call_data, dict): continue module_address = (file, module_index, module_call_name) if module_address in self._loaded_modules: continue # Variables being passed to module, "source" and "version" are reserved specified_vars = { k: v[0] for k, v in module_call_data.items() if k != "source" and k != "version" } if not ignore_unresolved_params: has_unresolved_params = False for k, v in specified_vars.items(): if not is_acceptable_module_param( v) or not is_acceptable_module_param(k): has_unresolved_params = True break if has_unresolved_params: skipped_a_module = True continue self._loaded_modules.add(module_address) source = module_call_data.get("source") if not source or not isinstance(source, list): continue source = source[0] if not isinstance(source, str): logging.debug( f"Skipping loading of {module_call_name} as source is not a string, it is: {source}" ) continue # Special handling for local sources to make sure we aren't double-parsing if source.startswith("./") or source.startswith("../"): source = os.path.normpath( os.path.join( os.path.dirname( _remove_module_dependency_in_path(file)), source)) version = module_call_data.get("version", "latest") if version and isinstance(version, list): version = version[0] try: with module_loader_registry.load( root_dir, source, version) as content: if not content.loaded(): continue self._internal_dir_load( directory=content.path(), module_loader_registry=module_loader_registry, dir_filter=dir_filter, specified_vars=specified_vars, module_load_context=module_load_context, keys_referenced_as_modules= keys_referenced_as_modules) module_definitions = { path: self.out_definitions[path] for path in list(self.out_definitions.keys()) if os.path.dirname(path) == content.path() } if not module_definitions: continue # NOTE: Modules are put into the main TF definitions structure "as normal" with the # notable exception of the file name. For loaded modules referrer information is # appended to the file name to create this format: # <file_name>[<referred_file>#<referrer_index>] # For example: # /the/path/module/my_module.tf[/the/path/main.tf#0] # The referrer and index allow a module allow a module to be loaded multiple # times with differing data. # # In addition, the referring block will have a "__resolved__" key added with a # list pointing to the location of the module data that was resolved. For example: # "__resolved__": ["/the/path/module/my_module.tf[/the/path/main.tf#0]"] resolved_loc_list = module_call_data.get( RESOLVED_MODULE_ENTRY_NAME) if resolved_loc_list is None: resolved_loc_list = [] module_call_data[ RESOLVED_MODULE_ENTRY_NAME] = resolved_loc_list # NOTE: Modules can load other modules, so only append referrer information where it # has not already been added. keys = list(module_definitions.keys()) for key in keys: if key.endswith("]") or file.endswith("]"): continue keys_referenced_as_modules.add(key) new_key = f"{key}[{file}#{module_index}]" module_definitions[ new_key] = module_definitions[key] del module_definitions[key] del self.out_definitions[key] if new_key not in resolved_loc_list: resolved_loc_list.append(new_key) resolved_loc_list.sort( ) # For testing, need predictable ordering deep_merge.merge(all_module_definitions, module_definitions) except Exception as e: logging.warning( "Unable to load module (source=\"%s\" version=\"%s\"): %s", source, version, e) pass if all_module_definitions: deep_merge.merge(self.out_definitions, all_module_definitions) deep_merge.merge(self.out_evaluations_context, all_module_evaluations_context) return skipped_a_module
def _load_modules(self, root_dir: str, module_loader_registry: ModuleLoaderRegistry, dir_filter: Callable[[str], bool], module_load_context: Optional[str]): all_module_definitions = {} all_module_evaluations_context = {} for file in list(self.out_definitions.keys()): file_data = self.out_definitions.get(file) module_calls = file_data.get("module") if not module_calls or not isinstance(module_calls, list): continue for module_index, module_call in enumerate(module_calls): if not isinstance(module_call, dict): continue # There should only be one module reference per outer dict, but... safety first for module_call_name, module_call_data in module_call.items(): if not isinstance(module_call_data, dict): continue source = module_call_data.get("source") if not source or not isinstance(source, list): continue source = source[0] # Special handling for local sources to make sure we aren't double-parsing if source.startswith("./") or source.startswith("../"): source = os.path.normpath( os.path.join( os.path.dirname( _remove_module_dependency_in_path(file)), source)) version = module_call_data.get("version", "latest") if version and isinstance(version, list): version = version[0] try: with module_loader_registry.load( root_dir, source, version) as content: if not content.loaded(): continue # Variables being passed to module, "source" and "version" are reserved specified_vars = { k: v[0] for k, v in module_call_data.items() if k != "source" and k != "version" } if not dir_filter(os.path.abspath(content.path())): continue self._internal_dir_load( directory=content.path(), module_loader_registry=module_loader_registry, dir_filter=dir_filter, specified_vars=specified_vars, module_load_context=module_load_context) module_definitions = { path: self.out_definitions[path] for path in list(self.out_definitions.keys()) if os.path.dirname(path) == content.path() } if not module_definitions: continue # NOTE: Modules are put into the main TF definitions structure "as normal" with the # notable exception of the file name. For loaded modules referrer information is # appended to the file name to create this format: # <file_name>[<referred_file>#<referrer_index>] # For example: # /the/path/module/my_module.tf[/the/path/main.tf#0] # The referrer and index allow a module allow a module to be loaded multiple # times with differing data. # # In addition, the referring block will have a "__resolved__" key added with a # list pointing to the location of the module data that was resolved. For example: # "__resolved__": ["/the/path/module/my_module.tf[/the/path/main.tf#0]"] resolved_loc_list = module_call_data.get( RESOLVED_MODULE_ENTRY_NAME) if resolved_loc_list is None: resolved_loc_list = [] module_call_data[ RESOLVED_MODULE_ENTRY_NAME] = resolved_loc_list # NOTE: Modules can load other modules, so only append referrer information where it # has not already been added. keys = list(module_definitions.keys()) for key in keys: if key.endswith("]"): continue new_key = f"{key}[{file}#{module_index}]" module_definitions[new_key] = \ module_definitions[key] del module_definitions[key] del self.out_definitions[key] resolved_loc_list.append(new_key) deep_merge.merge(all_module_definitions, module_definitions) except Exception as e: logging.warning( "Unable to load module (source=\"%s\" version=\"%s\"): %s", source, version, e) pass if all_module_definitions: deep_merge.merge(self.out_definitions, all_module_definitions) deep_merge.merge(self.out_evaluations_context, all_module_evaluations_context)