def load(self, current_dir: str, source: str, source_version: Optional[str]) -> ModuleContent:
# Local path references start with "./" or "../"
if not source.startswith("./") and not source.startswith("../"):
return ModuleContent(None)
module_path = os.path.normpath(os.path.join(current_dir, source))
if not os.path.exists(module_path):
raise FileNotFoundError(module_path)
return ModuleContent(module_path)
def _load_module(self) -> ModuleContent:
if os.path.exists(self.dest_dir):
return ModuleContent(dir=None)
best_version = self._find_best_version()
request_download_url = os.path.join(self.REGISTRY_URL_PREFIX, self.module_source, best_version, 'download')
response = requests.get(url=request_download_url)
if response.status_code != HTTPStatus.OK and response.status_code != HTTPStatus.NO_CONTENT:
return ModuleContent(dir=None)
else:
return ModuleContent(dir=None, next_url=response.headers.get('X-Terraform-Get', ''))
def _load_module(self) -> ModuleContent:
try:
module_source = self.module_source.lstrip('git::')
if module_source.startswith('ssh:'):
return ModuleContent(dir=None)
git_getter = GitGetter(module_source, create_clone_and_result_dirs=False)
git_getter.temp_dir = self.dest_dir
git_getter.do_get()
return_dir = self.dest_dir
if self.inner_module:
return_dir = os.path.join(self.dest_dir, self.inner_module)
return ModuleContent(dir=return_dir)
except Exception as e:
self.logger.error(f'failed to get {self.module_source} because of {e}')
return ModuleContent(dir=None)
def load(self, current_dir: str, source: str,
source_version: Optional[str]) -> ModuleContent:
"""
Search all registered loaders for the first one which is able to load the module source type. For more
information, see `loader.ModuleLoader.load`.
"""
last_exception = None
for loader in self.loaders:
try:
content = loader.load(current_dir, source, source_version)
except Exception as e:
last_exception = e
continue
if content is None:
continue
elif not content.loaded():
content.cleanup()
continue
else:
return content
if last_exception is not None:
raise last_exception
return ModuleContent(None)
def load(self, current_dir: str, source: str,
source_version: Optional[str]) -> ModuleContent:
"""
Search all registered loaders for the first one which is able to load the module source type. For more
information, see `loader.ModuleLoader.load`.
"""
if os.name == 'nt':
# For windows, due to limitations in the allowed characters for path names, the hash of the source is used.
# https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
source_hash = hashlib.md5(source.encode()) # nosec
local_dir = os.path.join(self.root_dir,
self.external_modules_folder_name,
source_hash.hexdigest())
else:
local_dir = os.path.join(self.root_dir,
self.external_modules_folder_name, source)
inner_module = ""
next_url = source
last_exception = None
while next_url:
source = next_url
next_url = ""
if source in self.failed_urls_cache:
break
for loader in self.loaders:
if not self.download_external_modules and loader.is_external:
continue
try:
content = loader.load(
root_dir=self.root_dir,
current_dir=current_dir,
source=source,
source_version=source_version,
dest_dir=local_dir,
external_modules_folder_name=self.
external_modules_folder_name,
inner_module=inner_module,
)
except Exception as e:
last_exception = e
continue
if content.next_url:
next_url = content.next_url
if loader.inner_module:
local_dir = loader.dest_dir
inner_module = loader.inner_module
break
if content is None:
continue
elif not content.loaded():
if content.failed_url:
self.failed_urls_cache.add(content.failed_url)
content.cleanup()
continue
else:
return content
if last_exception is not None:
raise last_exception
return ModuleContent(None)
def load(
self,
root_dir: str,
current_dir: str,
source: str,
source_version: Optional[str],
dest_dir: str,
external_modules_folder_name: str,
inner_module: Optional[str] = None,
) -> ModuleContent:
"""
This function provides an opportunity for the loader to load a module's content if it chooses to do so.
There are three resulting states that can occur when calling this function:
1) the loader can't handle the source type, in which case a ModuleContent is returned for which
the `loaded()` method will return False.
2) the loader can handle the source type and loading is successful, in which case a ModuleContent
object is returned for which `loaded()` returns True and which provides the directory containing
the module files
3) the loader tried to load the module content but and error occurred, in which case an exception
is raised.
:param current_dir: Directory containing the reference to the module.
:param source: the raw source string from the module's `source` attribute (e.g.,
"hashicorp/consul/aws" or "git::https://example.com/vpc.git?ref=v1.2.0")
:param source_version: contains content from the module's `version` attribute, if provided
:param dest_dir: where to save the downloaded module
:return: A ModuleContent object which may or may not being loaded.
"""
self.root_dir = root_dir
self.module_source = source
self.current_dir = current_dir
self.version = str(source_version)
self.dest_dir = dest_dir
self.external_modules_folder_name = external_modules_folder_name
self.inner_module = inner_module
if not self._is_matching_loader():
return ModuleContent(dir=None)
module_path = self._find_module_path()
if os.path.exists(module_path):
return ModuleContent(dir=module_path)
self.logger.debug(
f"getting module {self.module_source} version: {self.version}")
return self._load_module()
def _load_module(self) -> ModuleContent:
module_path = os.path.normpath(os.path.join(self.current_dir, self.module_source))
if self.module_source.startswith(self.current_dir):
module_path = self.module_source
if not os.path.exists(module_path):
raise FileNotFoundError(module_path)
return ModuleContent(module_path)
def _load_module(self) -> ModuleContent:
try:
self._process_generic_git_repo()
module_source = self.module_source.replace("git::", "")
git_getter = GitGetter(module_source,
create_clone_and_result_dirs=False)
git_getter.temp_dir = self.dest_dir
git_getter.do_get()
return_dir = self.dest_dir
if self.inner_module:
return_dir = os.path.join(self.dest_dir, self.inner_module)
return ModuleContent(dir=return_dir)
except Exception as e:
self.logger.error(
f"failed to get {self.module_source} because of {e}")
return ModuleContent(dir=None, failed_url=self.module_source)
def load(self, current_dir: str, source: str,
source_version: Optional[str]) -> ModuleContent:
# Reference: https://www.terraform.io/docs/modules/sources.html#terraform-registry
# Format: [<HOSTNAME>/]<NAMESPACE>/<NAME>/<PROVIDER>
# Example: hashicorp/consul/aws
slash_count = source.count("/") != 2
if source.startswith("/") or source.endswith(
"/") or slash_count < 2 or slash_count > 3:
return ModuleContent(None)
tokens = source.split("/")
if len(tokens) == 3:
host = "registry.terraform.io"
namespace = tokens[0]
name = tokens[1]
provider = tokens[2]
else:
host = tokens[0]
namespace = tokens[1]
name = tokens[2]
provider = tokens[3]
# Info: https://www.terraform.io/docs/internals/module-registry-protocol.html#sample-request-1
base_url = f"https://{host}/{namespace}/{name}/{provider}"
with requests.session() as session:
if not source_version:
source_version = self._determine_latest_version(
base_url, session)
temp_dir = tempfile.TemporaryDirectory()
try:
self._download_source(base_url, source_version, session)
return ModuleContent(temp_dir)
except:
temp_dir.cleanup()
raise
def load(self, current_dir: str, source: str,
source_version: Optional[str]) -> ModuleContent:
"""
Search all registered loaders for the first one which is able to load the module source type. For more
information, see `loader.ModuleLoader.load`.
"""
local_dir = os.path.join(current_dir,
self.external_modules_folder_name, source)
inner_module = ''
next_url = source
last_exception = None
while next_url:
source = next_url
next_url = ''
if source in self.failed_urls_cache:
break
for loader in self.loaders:
if not self.download_external_modules and loader.is_external:
continue
try:
content = loader.load(current_dir, source, source_version,
local_dir, inner_module)
except Exception as e:
last_exception = e
continue
if content.next_url:
next_url = content.next_url
if loader.inner_module:
local_dir = loader.dest_dir
inner_module = loader.inner_module
break
if content is None:
continue
elif not content.loaded():
if content.failed_url:
self.failed_urls_cache.add(content.failed_url)
content.cleanup()
continue
else:
return content
if last_exception is not None:
raise last_exception
return ModuleContent(None)
