def test_load_terraform_registry(
git_getter,
source,
source_version,
expected_content_path,
expected_git_url,
expected_dest_dir,
expected_module_source,
expected_inner_module,
):
# given
current_dir = Path(__file__).parent / "tmp"
registry = ModuleLoaderRegistry(download_external_modules=True)
# when
content = registry.load(current_dir=str(current_dir),
source=source,
source_version=source_version)
# then
assert content.loaded()
assert content.path() == str(
Path(DEFAULT_EXTERNAL_MODULES_DIR) / expected_content_path)
git_getter.assert_called_once_with(expected_git_url, mock.ANY)
git_loader = next(loader for loader in registry.loaders
if isinstance(loader, GenericGitLoader))
assert git_loader.dest_dir == str(
Path(DEFAULT_EXTERNAL_MODULES_DIR) / expected_dest_dir)
assert git_loader.module_source == expected_module_source
assert git_loader.inner_module == expected_inner_module
def test_load_terraform_registry(self):
registry = ModuleLoaderRegistry(True, DEFAULT_EXTERNAL_MODULES_DIR)
source = "terraform-aws-modules/security-group/aws"
with registry.load(current_dir=self.current_dir, source=source, source_version="~> 3.0") as content:
assert content.loaded()
expected_content_path = os.path.join(self.current_dir, DEFAULT_EXTERNAL_MODULES_DIR, source)
self.assertEqual(expected_content_path, content.path(), f"expected content.path() to be {content.path()}, got {expected_content_path}")
def test_load_local_module_absolute_path(self):
registry = ModuleLoaderRegistry(download_external_modules=True)
source = "/some/module"
try:
registry.load(current_dir=self.current_dir, source=source, source_version="latest")
self.assertEqual(1, 2, 'Module loading should have thrown an error')
except FileNotFoundError as e:
self.assertEqual(str(e), source)
def test_load_terraform_registry(self):
registry = ModuleLoaderRegistry(True, DEFAULT_EXTERNAL_MODULES_DIR)
registry.root_dir = self.current_dir
source = "terraform-aws-modules/security-group/aws"
content = registry.load(current_dir=self.current_dir, source=source, source_version="~> 3.0")
assert content.loaded()
expected_content_path = os.path.join(
self.current_dir,
DEFAULT_EXTERNAL_MODULES_DIR,
"github.com/terraform-aws-modules/terraform-aws-security-group",
)
self.assertRegex(content.path(), f"^{expected_content_path}/v3.*")
def test_load_local_path(git_getter, source, expected_content_path, expected_exception):
# given
current_dir = Path(__file__).parent
registry = ModuleLoaderRegistry()
# when
with expected_exception:
content = registry.load(current_dir=str(current_dir), source=source, source_version="latest")
# then
assert content.loaded()
assert content.path() == str(current_dir / expected_content_path)
git_getter.assert_not_called()
def test_load_terraform_registry_check_cache(self):
registry = ModuleLoaderRegistry(download_external_modules=True)
registry.root_dir = self.current_dir
source1 = "git::https://github.com/bridgecrewio/checkov_not_working1.git"
registry.load(current_dir=self.current_dir, source=source1, source_version="latest")
self.assertIn(source1, registry.failed_urls_cache)
source2 = "git::https://github.com/bridgecrewio/checkov_not_working2.git"
registry.load(current_dir=self.current_dir, source=source2, source_version="latest")
self.assertIn(source1 in registry.failed_urls_cache and source2, registry.failed_urls_cache)
def _load_modules(self,
root_dir: str,
module_loader_registry: ModuleLoaderRegistry,
dir_filter: Callable[[str], bool],
module_load_context: Optional[str],
keys_referenced_as_modules: Set[str],
ignore_unresolved_params: bool = False) -> bool:
"""
Load modules which have not already been loaded and can be loaded (don't have unresolved parameters).
:param ignore_unresolved_params: If true, not-yet-loaded modules will be loaded even if they are
passed parameters that are not fully resolved.
:return: True if there were modules that were not loaded due to unresolved
parameters.
"""
all_module_definitions = {}
all_module_evaluations_context = {}
skipped_a_module = False
for file in list(self.out_definitions.keys()):
# Don't process a file in a directory other than the directory we're processing. For example,
# if we're down dealing with <top_dir>/<module>/something.tf, we don't want to rescan files
# up in <top_dir>.
if os.path.dirname(file) != root_dir:
continue
# Don't process a file reference which has already been processed
if file.endswith("]"):
continue
file_data = self.out_definitions.get(file)
if file_data is None:
continue
module_calls = file_data.get("module")
if not module_calls or not isinstance(module_calls, list):
continue
for module_index, module_call in enumerate(module_calls):
if not isinstance(module_call, dict):
continue
# There should only be one module reference per outer dict, but... safety first
for module_call_name, module_call_data in module_call.items():
if not isinstance(module_call_data, dict):
continue
module_address = (file, module_index, module_call_name)
if module_address in self._loaded_modules:
continue
# Variables being passed to module, "source" and "version" are reserved
specified_vars = {
k: v[0]
for k, v in module_call_data.items()
if k != "source" and k != "version"
}
if not ignore_unresolved_params:
has_unresolved_params = False
for k, v in specified_vars.items():
if not is_acceptable_module_param(
v) or not is_acceptable_module_param(k):
has_unresolved_params = True
break
if has_unresolved_params:
skipped_a_module = True
continue
self._loaded_modules.add(module_address)
source = module_call_data.get("source")
if not source or not isinstance(source, list):
continue
source = source[0]
if not isinstance(source, str):
logging.debug(
f"Skipping loading of {module_call_name} as source is not a string, it is: {source}"
)
continue
# Special handling for local sources to make sure we aren't double-parsing
if source.startswith("./") or source.startswith("../"):
source = os.path.normpath(
os.path.join(
os.path.dirname(
_remove_module_dependency_in_path(file)),
source))
version = module_call_data.get("version", "latest")
if version and isinstance(version, list):
version = version[0]
try:
with module_loader_registry.load(
root_dir, source, version) as content:
if not content.loaded():
continue
self._internal_dir_load(
directory=content.path(),
module_loader_registry=module_loader_registry,
dir_filter=dir_filter,
specified_vars=specified_vars,
module_load_context=module_load_context,
keys_referenced_as_modules=
keys_referenced_as_modules)
module_definitions = {
path: self.out_definitions[path]
for path in list(self.out_definitions.keys())
if os.path.dirname(path) == content.path()
}
if not module_definitions:
continue
# NOTE: Modules are put into the main TF definitions structure "as normal" with the
# notable exception of the file name. For loaded modules referrer information is
# appended to the file name to create this format:
# <file_name>[<referred_file>#<referrer_index>]
# For example:
# /the/path/module/my_module.tf[/the/path/main.tf#0]
# The referrer and index allow a module allow a module to be loaded multiple
# times with differing data.
#
# In addition, the referring block will have a "__resolved__" key added with a
# list pointing to the location of the module data that was resolved. For example:
# "__resolved__": ["/the/path/module/my_module.tf[/the/path/main.tf#0]"]
resolved_loc_list = module_call_data.get(
RESOLVED_MODULE_ENTRY_NAME)
if resolved_loc_list is None:
resolved_loc_list = []
module_call_data[
RESOLVED_MODULE_ENTRY_NAME] = resolved_loc_list
# NOTE: Modules can load other modules, so only append referrer information where it
# has not already been added.
keys = list(module_definitions.keys())
for key in keys:
if key.endswith("]") or file.endswith("]"):
continue
keys_referenced_as_modules.add(key)
new_key = f"{key}[{file}#{module_index}]"
module_definitions[
new_key] = module_definitions[key]
del module_definitions[key]
del self.out_definitions[key]
if new_key not in resolved_loc_list:
resolved_loc_list.append(new_key)
resolved_loc_list.sort(
) # For testing, need predictable ordering
deep_merge.merge(all_module_definitions,
module_definitions)
except Exception as e:
logging.warning(
"Unable to load module (source=\"%s\" version=\"%s\"): %s",
source, version, e)
pass
if all_module_definitions:
deep_merge.merge(self.out_definitions, all_module_definitions)
deep_merge.merge(self.out_evaluations_context,
all_module_evaluations_context)
return skipped_a_module
def _load_modules(self, root_dir: str,
module_loader_registry: ModuleLoaderRegistry,
dir_filter: Callable[[str], bool],
module_load_context: Optional[str]):
all_module_definitions = {}
all_module_evaluations_context = {}
for file in list(self.out_definitions.keys()):
file_data = self.out_definitions.get(file)
module_calls = file_data.get("module")
if not module_calls or not isinstance(module_calls, list):
continue
for module_index, module_call in enumerate(module_calls):
if not isinstance(module_call, dict):
continue
# There should only be one module reference per outer dict, but... safety first
for module_call_name, module_call_data in module_call.items():
if not isinstance(module_call_data, dict):
continue
source = module_call_data.get("source")
if not source or not isinstance(source, list):
continue
source = source[0]
# Special handling for local sources to make sure we aren't double-parsing
if source.startswith("./") or source.startswith("../"):
source = os.path.normpath(
os.path.join(
os.path.dirname(
_remove_module_dependency_in_path(file)),
source))
version = module_call_data.get("version", "latest")
if version and isinstance(version, list):
version = version[0]
try:
with module_loader_registry.load(
root_dir, source, version) as content:
if not content.loaded():
continue
# Variables being passed to module, "source" and "version" are reserved
specified_vars = {
k: v[0]
for k, v in module_call_data.items()
if k != "source" and k != "version"
}
if not dir_filter(os.path.abspath(content.path())):
continue
self._internal_dir_load(
directory=content.path(),
module_loader_registry=module_loader_registry,
dir_filter=dir_filter,
specified_vars=specified_vars,
module_load_context=module_load_context)
module_definitions = {
path: self.out_definitions[path]
for path in list(self.out_definitions.keys())
if os.path.dirname(path) == content.path()
}
if not module_definitions:
continue
# NOTE: Modules are put into the main TF definitions structure "as normal" with the
# notable exception of the file name. For loaded modules referrer information is
# appended to the file name to create this format:
# <file_name>[<referred_file>#<referrer_index>]
# For example:
# /the/path/module/my_module.tf[/the/path/main.tf#0]
# The referrer and index allow a module allow a module to be loaded multiple
# times with differing data.
#
# In addition, the referring block will have a "__resolved__" key added with a
# list pointing to the location of the module data that was resolved. For example:
# "__resolved__": ["/the/path/module/my_module.tf[/the/path/main.tf#0]"]
resolved_loc_list = module_call_data.get(
RESOLVED_MODULE_ENTRY_NAME)
if resolved_loc_list is None:
resolved_loc_list = []
module_call_data[
RESOLVED_MODULE_ENTRY_NAME] = resolved_loc_list
# NOTE: Modules can load other modules, so only append referrer information where it
# has not already been added.
keys = list(module_definitions.keys())
for key in keys:
if key.endswith("]"):
continue
new_key = f"{key}[{file}#{module_index}]"
module_definitions[new_key] = \
module_definitions[key]
del module_definitions[key]
del self.out_definitions[key]
resolved_loc_list.append(new_key)
deep_merge.merge(all_module_definitions,
module_definitions)
except Exception as e:
logging.warning(
"Unable to load module (source=\"%s\" version=\"%s\"): %s",
source, version, e)
pass
if all_module_definitions:
deep_merge.merge(self.out_definitions, all_module_definitions)
deep_merge.merge(self.out_evaluations_context,
all_module_evaluations_context)