def get_module_vertices_mapping(self): """ For each vertex, if it's originated in a module import, add to the vertex the index of the matching module vertex as 'source_module' """ block_dirs_to_modules = {} for dir_name, paths_to_modules in self.module_dependency_map.items(): # for each directory, find the module vertex that imported it if block_dirs_to_modules.get(dir_name): continue for path_to_module in paths_to_modules: if not path_to_module: continue path_to_module = unify_dependency_path(path_to_module) module_list = self.map_path_to_module.get(path_to_module, []) for module_index in module_list: module_vertex = self.vertices[module_index] module_vertex_dir = os.path.dirname(module_vertex.path) module_source = self.vertices[module_index].attributes.get( 'source', [''])[0] if self._get_dest_module_path(module_vertex_dir, module_source) == dir_name: if not block_dirs_to_modules.get(dir_name): block_dirs_to_modules[dir_name] = set() block_dirs_to_modules[dir_name].add(module_index) for vertex in self.vertices: # match the right module vertex according to the vertex path directory module_index = block_dirs_to_modules.get( os.path.dirname(vertex.path), set()) if module_index: vertex.source_module = module_index
def get_module_dependency_map(tf_definitions): """ :param tf_definitions, with paths in format 'dir/main.tf[module_dir/main.tf#0]' :return module_dependency_map: mapping between directories and the location of its module definition: {'dir': 'module_dir/main.tf'} :return tf_definitions: with paths in format 'dir/main.tf' """ module_dependency_map = {} copy_of_tf_definitions = {} dep_index_mapping = {} definitions_keys = list(tf_definitions.keys()) origin_keys = list( filter(lambda k: not k.endswith(']'), definitions_keys)) unevaluated_keys = list( filter(lambda k: k.endswith(']'), definitions_keys)) for file_path in origin_keys: dir_name = os.path.dirname(file_path) module_dependency_map[dir_name] = [[]] copy_of_tf_definitions[file_path] = deepcopy( tf_definitions[file_path]) next_level, unevaluated_keys = Parser.get_next_vertices( origin_keys, unevaluated_keys) while next_level: for file_path in next_level: path, module_dependency, module_dependency_num = remove_module_dependency_in_path( file_path) dir_name = os.path.dirname(path) current_deps = deepcopy( module_dependency_map[os.path.dirname(module_dependency)]) for dep in current_deps: dep.append(module_dependency) if dir_name not in module_dependency_map: module_dependency_map[dir_name] = current_deps elif current_deps not in module_dependency_map[dir_name]: module_dependency_map[dir_name] += current_deps copy_of_tf_definitions[path] = deepcopy( tf_definitions[file_path]) origin_keys.append(path) dep_index_mapping[path] = module_dependency_num next_level, unevaluated_keys = Parser.get_next_vertices( origin_keys, unevaluated_keys) for key, dep_trails in module_dependency_map.items(): hashes = set() deduped = [] for trail in dep_trails: hash = unify_dependency_path(trail) if hash in hashes: continue hashes.add(hash) deduped.append(trail) module_dependency_map[key] = deduped return module_dependency_map, copy_of_tf_definitions, dep_index_mapping
def _add_to_blocks(self, block: Block) -> None: dependencies = [dep_trail for dep_trail in self.module_dependency_map.get(os.path.dirname(block.path), [])] module_dependency_num = "" if not dependencies: dependencies = [[]] else: module_dependency_num = self.dep_index_mapping.get(block.path, "") for dep_trail in dependencies: block = deepcopy(block) block.module_dependency = unify_dependency_path(dep_trail) block.module_dependency_num = module_dependency_num self.blocks.append(block)
def get_module_vertices_mapping(self) -> None: """ For each vertex, if it's originated in a module import, add to the vertex the index of the matching module vertex as 'source_module' """ block_dirs_to_modules: Dict[Tuple[str, str], Dict[str, Set[int]]] = defaultdict(dict) for dir_name, paths_to_modules in self.module.module_dependency_map.items(): # for each directory, find the module vertex that imported it for path_to_module in paths_to_modules: if not path_to_module: continue path_to_module_str = unify_dependency_path(path_to_module) if block_dirs_to_modules.get((dir_name, path_to_module_str)): continue module_list = self.map_path_to_module.get(path_to_module[-1], []) for module_index in module_list: module_vertex = self.vertices[module_index] if module_vertex.module_dependency == unify_dependency_path(path_to_module[:-1]): module_vertex_dir = self.get_dirname(module_vertex.path) module_source = module_vertex.attributes.get("source", [""])[0] module_version = module_vertex.attributes.get("version", ["latest"])[0] dest_module_path = self._get_dest_module_path( curr_module_dir=module_vertex_dir, dest_module_source=module_source, dest_module_version=module_version ) if dest_module_path == dir_name: module_dependency_num = self.module.module_address_map.get((module_vertex.path, module_vertex.name)) if module_dependency_num: block_dirs_to_modules[(dir_name, path_to_module_str)].setdefault(module_dependency_num, set()).add(module_index) for vertex in self.vertices: # match the right module vertex according to the vertex path directory module_dependency_nums = block_dirs_to_modules.get((self.get_dirname(vertex.path), vertex.module_dependency)) if module_dependency_nums: module_indices = module_dependency_nums.get(vertex.module_dependency_num) if module_indices: vertex.source_module = module_indices
def _make_module_ref_absolute(match, dir_path) -> str: module_location = match[1] if not os.path.isabs(module_location): module_location = os.path.join(dir_path, module_location) module_referrer = match[2] if PATH_SEPARATOR in module_referrer: module_referrer_fixed = [] if '#' in module_referrer: module_referrer = module_referrer[:-2] for ref in module_referrer.split(PATH_SEPARATOR): if not os.path.isabs(ref): module_referrer_fixed.append(os.path.join(dir_path, ref)) module_referrer = unify_dependency_path(module_referrer_fixed) else: module_referrer = os.path.join(dir_path, module_referrer) return f"{module_location}[{module_referrer}#{match[3]}]"
def _add_to_blocks(self, block: TerraformBlock) -> None: dependencies = self.module_dependency_map.get( os.path.dirname(block.path), []) module_dependency_num = "" if not dependencies: dependencies = [[]] for dep_idx, dep_trail in enumerate(dependencies): if dep_idx > 0: block = deepcopy(block) block.module_dependency = unify_dependency_path(dep_trail) if block.module_dependency: module_dependency_numbers = self.dep_index_mapping.get( (block.path, dep_trail[-1]), []) for mod_idx, module_dep_num in enumerate( module_dependency_numbers): if mod_idx > 0: block = deepcopy(block) block.module_dependency_num = module_dep_num self.blocks.append(block) else: block.module_dependency_num = module_dependency_num self.blocks.append(block)
def _build_edges(self) -> None: logging.info("Creating edges") self.get_module_vertices_mapping() aliases = self._get_aliases() for origin_node_index, vertex in enumerate(self.vertices): for attribute_key in vertex.attributes: if attribute_key in reserved_attribute_names or attribute_has_nested_attributes( attribute_key, vertex.attributes): continue referenced_vertices = get_referenced_vertices_in_value( value=vertex.attributes[attribute_key], aliases=aliases, resources_types=self.get_resources_types_in_graph(), ) for vertex_reference in referenced_vertices: # for certain blocks such as data and resource, the block name is composed from several parts. # the purpose of the loop is to avoid not finding the node if the name has several parts sub_values = [ remove_index_pattern_from_str(sub_value) for sub_value in vertex_reference.sub_parts ] for i, _ in enumerate(sub_values): reference_name = join_trimmed_strings( char_to_join=".", str_lst=sub_values, num_to_trim=i) if vertex.module_dependency: dest_node_index = self._find_vertex_index_relative_to_path( vertex_reference.block_type, reference_name, vertex.path, vertex.module_dependency, vertex.module_dependency_num) if dest_node_index == -1: dest_node_index = self._find_vertex_index_relative_to_path( vertex_reference.block_type, reference_name, vertex.path, vertex.path, vertex.module_dependency_num) else: dest_node_index = self._find_vertex_index_relative_to_path( vertex_reference.block_type, reference_name, vertex.path, vertex.module_dependency, vertex.module_dependency_num) if dest_node_index > -1 and origin_node_index > -1: if vertex_reference.block_type == BlockType.MODULE: try: self._connect_module( sub_values, attribute_key, self.vertices[dest_node_index], origin_node_index) except Exception as e: logging.warning( f"Module {self.vertices[dest_node_index]} does not have source attribute, skipping" ) logging.warning(e, stack_info=True) else: self._create_edge(origin_node_index, dest_node_index, attribute_key) break if vertex.block_type == BlockType.MODULE and vertex.attributes.get( 'source'): target_path = vertex.path if vertex.module_dependency != "": target_path = unify_dependency_path( [vertex.module_dependency, vertex.path]) dest_module_path = self._get_dest_module_path( curr_module_dir=self.get_dirname(vertex.path), dest_module_source=vertex.attributes["source"][0], dest_module_version=vertex.attributes.get( "version", ["latest"])[0]) target_variables = [ index for index in self.vertices_by_module_dependency.get(( target_path, self.module.module_address_map.get(( vertex.path, vertex.name))), {}).get(BlockType.VARIABLE, []) if self.get_dirname(self.vertices[index].path) == dest_module_path ] for attribute, value in vertex.attributes.items(): if attribute in MODULE_RESERVED_ATTRIBUTES: continue target_variable = next( (v for v in target_variables if self.vertices[v].name == attribute), None) if target_variable is not None: self._create_edge(target_variable, origin_node_index, "default") elif vertex.block_type == BlockType.TF_VARIABLE: # Assuming the tfvars file is in the same directory as the variables file (best practice) target_variables = [ index for index in self.vertices_block_name_map.get( BlockType.VARIABLE, {}).get(vertex.name, []) if self.get_dirname(self.vertices[index].path) == self.get_dirname(vertex.path) ] if len(target_variables) == 1: self._create_edge(target_variables[0], origin_node_index, "default")
def _build_edges(self): logging.info('Creating edges') self.get_module_vertices_mapping() aliases = self._get_aliases() for origin_node_index, vertex in enumerate(self.vertices): for attribute_key in vertex.attributes: if attribute_key in reserved_attribute_names or attribute_has_nested_attributes( attribute_key, vertex.attributes): continue referenced_vertices = get_referenced_vertices_in_value( value=vertex.attributes[attribute_key], aliases=aliases, resources_types=self.get_resources_types_in_graph()) for vertex_reference in referenced_vertices: # for certain blocks such as data and resource, the block name is composed from several parts. # the purpose of the loop is to avoid not finding the node if the name has several parts sub_values = [ remove_index_pattern_from_str(sub_value) for sub_value in vertex_reference.sub_parts ] for i in range(len(sub_values)): reference_name = join_trimmed_strings( char_to_join=".", str_lst=sub_values, num_to_trim=i) if vertex.module_dependency: dest_node_index = self._find_vertex_index_relative_to_path( vertex_reference.block_type, reference_name, vertex.path, vertex.module_dependency) if dest_node_index == -1: dest_node_index = self._find_vertex_index_relative_to_path( vertex_reference.block_type, reference_name, vertex.path, vertex.path) else: dest_node_index = self._find_vertex_index_relative_to_path( vertex_reference.block_type, reference_name, vertex.path, vertex.module_dependency) if dest_node_index > -1 and origin_node_index > -1: if vertex_reference.block_type == BlockType.MODULE: try: self._connect_module( sub_values, attribute_key, self.vertices[dest_node_index], origin_node_index) except Exception as e: logging.warning( f'Module {self.vertices[dest_node_index]} does not have source attribute, skipping' ) logging.warning(e, stack_info=True) else: self._create_edge(origin_node_index, dest_node_index, attribute_key) break if vertex.block_type == BlockType.MODULE: target_path = vertex.path if vertex.module_dependency != '': target_path = unify_dependency_path( [vertex.module_dependency, vertex.path]) target_variables = list( filter( lambda v: self.vertices[v].module_dependency == target_path, self.vertices_by_block_type.get( BlockType.VARIABLE, {}))) for attribute, value in vertex.attributes.items(): if attribute in MODULE_RESERVED_ATTRIBUTES: continue target_variable = None for v in target_variables: if self.vertices[v].name == attribute: target_variable = v break if target_variable is not None: self._create_edge(target_variable, origin_node_index, 'default') elif vertex.block_type == BlockType.TF_VARIABLE: if vertex.module_dependency != '': target_path = unify_dependency_path( [vertex.module_dependency, vertex.path]) # Assuming the tfvars file is in the same directory as the variables file (best practice) target_variables = list( filter( lambda v: os.path.dirname(self.vertices[v].path) == os. path.dirname(vertex.path), self.vertices_block_name_map.get( BlockType.VARIABLE, {}).get(vertex.name, []))) if len(target_variables) == 1: self._create_edge(target_variables[0], origin_node_index, 'default')