def get_module_vertices_mapping(self):
"""
For each vertex, if it's originated in a module import, add to the vertex the index of the
matching module vertex as 'source_module'
"""
block_dirs_to_modules = {}
for dir_name, paths_to_modules in self.module_dependency_map.items():
# for each directory, find the module vertex that imported it
if block_dirs_to_modules.get(dir_name):
continue
for path_to_module in paths_to_modules:
if not path_to_module:
continue
path_to_module = unify_dependency_path(path_to_module)
module_list = self.map_path_to_module.get(path_to_module, [])
for module_index in module_list:
module_vertex = self.vertices[module_index]
module_vertex_dir = os.path.dirname(module_vertex.path)
module_source = self.vertices[module_index].attributes.get(
'source', [''])[0]
if self._get_dest_module_path(module_vertex_dir,
module_source) == dir_name:
if not block_dirs_to_modules.get(dir_name):
block_dirs_to_modules[dir_name] = set()
block_dirs_to_modules[dir_name].add(module_index)
for vertex in self.vertices:
# match the right module vertex according to the vertex path directory
module_index = block_dirs_to_modules.get(
os.path.dirname(vertex.path), set())
if module_index:
vertex.source_module = module_index
def get_module_dependency_map(tf_definitions):
"""
:param tf_definitions, with paths in format 'dir/main.tf[module_dir/main.tf#0]'
:return module_dependency_map: mapping between directories and the location of its module definition:
{'dir': 'module_dir/main.tf'}
:return tf_definitions: with paths in format 'dir/main.tf'
"""
module_dependency_map = {}
copy_of_tf_definitions = {}
dep_index_mapping = {}
definitions_keys = list(tf_definitions.keys())
origin_keys = list(
filter(lambda k: not k.endswith(']'), definitions_keys))
unevaluated_keys = list(
filter(lambda k: k.endswith(']'), definitions_keys))
for file_path in origin_keys:
dir_name = os.path.dirname(file_path)
module_dependency_map[dir_name] = [[]]
copy_of_tf_definitions[file_path] = deepcopy(
tf_definitions[file_path])
next_level, unevaluated_keys = Parser.get_next_vertices(
origin_keys, unevaluated_keys)
while next_level:
for file_path in next_level:
path, module_dependency, module_dependency_num = remove_module_dependency_in_path(
file_path)
dir_name = os.path.dirname(path)
current_deps = deepcopy(
module_dependency_map[os.path.dirname(module_dependency)])
for dep in current_deps:
dep.append(module_dependency)
if dir_name not in module_dependency_map:
module_dependency_map[dir_name] = current_deps
elif current_deps not in module_dependency_map[dir_name]:
module_dependency_map[dir_name] += current_deps
copy_of_tf_definitions[path] = deepcopy(
tf_definitions[file_path])
origin_keys.append(path)
dep_index_mapping[path] = module_dependency_num
next_level, unevaluated_keys = Parser.get_next_vertices(
origin_keys, unevaluated_keys)
for key, dep_trails in module_dependency_map.items():
hashes = set()
deduped = []
for trail in dep_trails:
hash = unify_dependency_path(trail)
if hash in hashes:
continue
hashes.add(hash)
deduped.append(trail)
module_dependency_map[key] = deduped
return module_dependency_map, copy_of_tf_definitions, dep_index_mapping
def _add_to_blocks(self, block: Block) -> None:
dependencies = [dep_trail for dep_trail in self.module_dependency_map.get(os.path.dirname(block.path), [])]
module_dependency_num = ""
if not dependencies:
dependencies = [[]]
else:
module_dependency_num = self.dep_index_mapping.get(block.path, "")
for dep_trail in dependencies:
block = deepcopy(block)
block.module_dependency = unify_dependency_path(dep_trail)
block.module_dependency_num = module_dependency_num
self.blocks.append(block)
def get_module_vertices_mapping(self) -> None:
"""
For each vertex, if it's originated in a module import, add to the vertex the index of the
matching module vertex as 'source_module'
"""
block_dirs_to_modules: Dict[Tuple[str, str], Dict[str, Set[int]]] = defaultdict(dict)
for dir_name, paths_to_modules in self.module.module_dependency_map.items():
# for each directory, find the module vertex that imported it
for path_to_module in paths_to_modules:
if not path_to_module:
continue
path_to_module_str = unify_dependency_path(path_to_module)
if block_dirs_to_modules.get((dir_name, path_to_module_str)):
continue
module_list = self.map_path_to_module.get(path_to_module[-1], [])
for module_index in module_list:
module_vertex = self.vertices[module_index]
if module_vertex.module_dependency == unify_dependency_path(path_to_module[:-1]):
module_vertex_dir = self.get_dirname(module_vertex.path)
module_source = module_vertex.attributes.get("source", [""])[0]
module_version = module_vertex.attributes.get("version", ["latest"])[0]
dest_module_path = self._get_dest_module_path(
curr_module_dir=module_vertex_dir,
dest_module_source=module_source,
dest_module_version=module_version
)
if dest_module_path == dir_name:
module_dependency_num = self.module.module_address_map.get((module_vertex.path, module_vertex.name))
if module_dependency_num:
block_dirs_to_modules[(dir_name, path_to_module_str)].setdefault(module_dependency_num, set()).add(module_index)
for vertex in self.vertices:
# match the right module vertex according to the vertex path directory
module_dependency_nums = block_dirs_to_modules.get((self.get_dirname(vertex.path), vertex.module_dependency))
if module_dependency_nums:
module_indices = module_dependency_nums.get(vertex.module_dependency_num)
if module_indices:
vertex.source_module = module_indices
def _make_module_ref_absolute(match, dir_path) -> str:
module_location = match[1]
if not os.path.isabs(module_location):
module_location = os.path.join(dir_path, module_location)
module_referrer = match[2]
if PATH_SEPARATOR in module_referrer:
module_referrer_fixed = []
if '#' in module_referrer:
module_referrer = module_referrer[:-2]
for ref in module_referrer.split(PATH_SEPARATOR):
if not os.path.isabs(ref):
module_referrer_fixed.append(os.path.join(dir_path, ref))
module_referrer = unify_dependency_path(module_referrer_fixed)
else:
module_referrer = os.path.join(dir_path, module_referrer)
return f"{module_location}[{module_referrer}#{match[3]}]"
def _add_to_blocks(self, block: TerraformBlock) -> None:
dependencies = self.module_dependency_map.get(
os.path.dirname(block.path), [])
module_dependency_num = ""
if not dependencies:
dependencies = [[]]
for dep_idx, dep_trail in enumerate(dependencies):
if dep_idx > 0:
block = deepcopy(block)
block.module_dependency = unify_dependency_path(dep_trail)
if block.module_dependency:
module_dependency_numbers = self.dep_index_mapping.get(
(block.path, dep_trail[-1]), [])
for mod_idx, module_dep_num in enumerate(
module_dependency_numbers):
if mod_idx > 0:
block = deepcopy(block)
block.module_dependency_num = module_dep_num
self.blocks.append(block)
else:
block.module_dependency_num = module_dependency_num
self.blocks.append(block)
def _build_edges(self) -> None:
logging.info("Creating edges")
self.get_module_vertices_mapping()
aliases = self._get_aliases()
for origin_node_index, vertex in enumerate(self.vertices):
for attribute_key in vertex.attributes:
if attribute_key in reserved_attribute_names or attribute_has_nested_attributes(
attribute_key, vertex.attributes):
continue
referenced_vertices = get_referenced_vertices_in_value(
value=vertex.attributes[attribute_key],
aliases=aliases,
resources_types=self.get_resources_types_in_graph(),
)
for vertex_reference in referenced_vertices:
# for certain blocks such as data and resource, the block name is composed from several parts.
# the purpose of the loop is to avoid not finding the node if the name has several parts
sub_values = [
remove_index_pattern_from_str(sub_value)
for sub_value in vertex_reference.sub_parts
]
for i, _ in enumerate(sub_values):
reference_name = join_trimmed_strings(
char_to_join=".",
str_lst=sub_values,
num_to_trim=i)
if vertex.module_dependency:
dest_node_index = self._find_vertex_index_relative_to_path(
vertex_reference.block_type, reference_name,
vertex.path, vertex.module_dependency,
vertex.module_dependency_num)
if dest_node_index == -1:
dest_node_index = self._find_vertex_index_relative_to_path(
vertex_reference.block_type,
reference_name, vertex.path, vertex.path,
vertex.module_dependency_num)
else:
dest_node_index = self._find_vertex_index_relative_to_path(
vertex_reference.block_type, reference_name,
vertex.path, vertex.module_dependency,
vertex.module_dependency_num)
if dest_node_index > -1 and origin_node_index > -1:
if vertex_reference.block_type == BlockType.MODULE:
try:
self._connect_module(
sub_values, attribute_key,
self.vertices[dest_node_index],
origin_node_index)
except Exception as e:
logging.warning(
f"Module {self.vertices[dest_node_index]} does not have source attribute, skipping"
)
logging.warning(e, stack_info=True)
else:
self._create_edge(origin_node_index,
dest_node_index,
attribute_key)
break
if vertex.block_type == BlockType.MODULE and vertex.attributes.get(
'source'):
target_path = vertex.path
if vertex.module_dependency != "":
target_path = unify_dependency_path(
[vertex.module_dependency, vertex.path])
dest_module_path = self._get_dest_module_path(
curr_module_dir=self.get_dirname(vertex.path),
dest_module_source=vertex.attributes["source"][0],
dest_module_version=vertex.attributes.get(
"version", ["latest"])[0])
target_variables = [
index for index in self.vertices_by_module_dependency.get((
target_path,
self.module.module_address_map.get((
vertex.path,
vertex.name))), {}).get(BlockType.VARIABLE, [])
if self.get_dirname(self.vertices[index].path) ==
dest_module_path
]
for attribute, value in vertex.attributes.items():
if attribute in MODULE_RESERVED_ATTRIBUTES:
continue
target_variable = next(
(v for v in target_variables
if self.vertices[v].name == attribute), None)
if target_variable is not None:
self._create_edge(target_variable, origin_node_index,
"default")
elif vertex.block_type == BlockType.TF_VARIABLE:
# Assuming the tfvars file is in the same directory as the variables file (best practice)
target_variables = [
index for index in self.vertices_block_name_map.get(
BlockType.VARIABLE, {}).get(vertex.name, [])
if self.get_dirname(self.vertices[index].path) ==
self.get_dirname(vertex.path)
]
if len(target_variables) == 1:
self._create_edge(target_variables[0], origin_node_index,
"default")
def _build_edges(self):
logging.info('Creating edges')
self.get_module_vertices_mapping()
aliases = self._get_aliases()
for origin_node_index, vertex in enumerate(self.vertices):
for attribute_key in vertex.attributes:
if attribute_key in reserved_attribute_names or attribute_has_nested_attributes(
attribute_key, vertex.attributes):
continue
referenced_vertices = get_referenced_vertices_in_value(
value=vertex.attributes[attribute_key],
aliases=aliases,
resources_types=self.get_resources_types_in_graph())
for vertex_reference in referenced_vertices:
# for certain blocks such as data and resource, the block name is composed from several parts.
# the purpose of the loop is to avoid not finding the node if the name has several parts
sub_values = [
remove_index_pattern_from_str(sub_value)
for sub_value in vertex_reference.sub_parts
]
for i in range(len(sub_values)):
reference_name = join_trimmed_strings(
char_to_join=".",
str_lst=sub_values,
num_to_trim=i)
if vertex.module_dependency:
dest_node_index = self._find_vertex_index_relative_to_path(
vertex_reference.block_type, reference_name,
vertex.path, vertex.module_dependency)
if dest_node_index == -1:
dest_node_index = self._find_vertex_index_relative_to_path(
vertex_reference.block_type,
reference_name, vertex.path, vertex.path)
else:
dest_node_index = self._find_vertex_index_relative_to_path(
vertex_reference.block_type, reference_name,
vertex.path, vertex.module_dependency)
if dest_node_index > -1 and origin_node_index > -1:
if vertex_reference.block_type == BlockType.MODULE:
try:
self._connect_module(
sub_values, attribute_key,
self.vertices[dest_node_index],
origin_node_index)
except Exception as e:
logging.warning(
f'Module {self.vertices[dest_node_index]} does not have source attribute, skipping'
)
logging.warning(e, stack_info=True)
else:
self._create_edge(origin_node_index,
dest_node_index,
attribute_key)
break
if vertex.block_type == BlockType.MODULE:
target_path = vertex.path
if vertex.module_dependency != '':
target_path = unify_dependency_path(
[vertex.module_dependency, vertex.path])
target_variables = list(
filter(
lambda v: self.vertices[v].module_dependency ==
target_path,
self.vertices_by_block_type.get(
BlockType.VARIABLE, {})))
for attribute, value in vertex.attributes.items():
if attribute in MODULE_RESERVED_ATTRIBUTES:
continue
target_variable = None
for v in target_variables:
if self.vertices[v].name == attribute:
target_variable = v
break
if target_variable is not None:
self._create_edge(target_variable, origin_node_index,
'default')
elif vertex.block_type == BlockType.TF_VARIABLE:
if vertex.module_dependency != '':
target_path = unify_dependency_path(
[vertex.module_dependency, vertex.path])
# Assuming the tfvars file is in the same directory as the variables file (best practice)
target_variables = list(
filter(
lambda v: os.path.dirname(self.vertices[v].path) == os.
path.dirname(vertex.path),
self.vertices_block_name_map.get(
BlockType.VARIABLE, {}).get(vertex.name, [])))
if len(target_variables) == 1:
self._create_edge(target_variables[0], origin_node_index,
'default')
