def test_find_non_literal_values(self): aliases = {'aws': {CustomAttributes.BLOCK_TYPE: BlockType.PROVIDER}} str_value = 'aws.east1' expected = [ VertexReference(BlockType.PROVIDER, ['aws', 'east1'], 'aws.east1') ] self.assertEqual( expected, get_referenced_vertices_in_value(str_value, aliases, [])) str_values = [ 'var.x', 'format("-%s", var.x)', '../child', 'aws_instance.example.id', 'bc_c_${var.customer_name}', 'aws iam delete-role --role-name ${local.role_name} --profile ${var.profile} --region ${var.region}', 'length(aws_vpc.main) > 0 ? aws_vpc.main[0].cidr_block : ${var.x}', ] expected = [ [VertexReference(BlockType.VARIABLE, ['x'], 'var.x')], [VertexReference(BlockType.VARIABLE, ['x'], 'var.x')], [], [ VertexReference(BlockType.RESOURCE, ['aws_instance.example', 'id'], 'aws_instance.example.id') ], [ VertexReference(BlockType.VARIABLE, ['customer_name'], 'var.customer_name') ], [ VertexReference(BlockType.LOCALS, ['role_name'], 'local.role_name'), VertexReference(BlockType.VARIABLE, ['profile'], 'var.profile'), VertexReference(BlockType.VARIABLE, ['region'], 'var.region') ], [ VertexReference(BlockType.RESOURCE, ['aws_vpc.main'], 'aws_vpc.main'), VertexReference(BlockType.RESOURCE, ['aws_vpc.main', 'cidr_block'], 'aws_vpc.main.cidr_block'), VertexReference(BlockType.VARIABLE, ['x'], 'var.x') ], ] for i in range(0, len(str_values)): self.assertEqual( expected[i], get_referenced_vertices_in_value(str_values[i], aliases, ['aws_vpc', 'aws_instance']))
def scan_resource_conf(self, conf: Dict[str, List[Any]]) -> CheckResult: self.handle_dynamic_values(conf) excluded_key = self.get_excluded_key() if excluded_key is not None: if dpath.search(conf, excluded_key) != {}: value = dpath.get(conf, excluded_key) if isinstance(value, list) and len(value) == 1: value = value[0] if self.check_excluded_condition(value): return CheckResult.PASSED inspected_key = self.get_inspected_key() bad_values = self.get_forbidden_values() if dpath.search(conf, inspected_key) != {}: value = dpath.get(conf, inspected_key) if isinstance(value, list) and len(value) == 1: value = value[0] if get_referenced_vertices_in_value(value=value, aliases={}, resources_types=[]): # we don't provide resources_types as we want to stay provider agnostic return CheckResult.UNKNOWN if value in bad_values or ANY_VALUE in bad_values: return CheckResult.FAILED else: return CheckResult.PASSED return self.missing_attribute_result
def process_undetermined_values(self, undetermined_values): for undetermined in undetermined_values: module_vertex = self.vertices[undetermined.get('module_vertex_id')] value = module_vertex.attributes.get(undetermined.get('attribute_name')) if not get_referenced_vertices_in_value(value=value, aliases={}, resources_types=self.get_resources_types_in_graph()): self.update_vertex_attribute(undetermined.get('variable_vertex_id'), 'default', value, undetermined.get('module_vertex_id'), undetermined.get('attribute_name'))
def evaluate_vertex_attribute_from_edge(self, edge_list): multiple_edges = len(edge_list) > 1 edge = edge_list[0] origin_vertex_attributes = self.local_graph.vertices[edge.origin].attributes val_to_eval = deepcopy(origin_vertex_attributes.get(edge.label, '')) referenced_vertices = get_referenced_vertices_in_value(value=val_to_eval, aliases={}, resources_types=self.local_graph.get_resources_types_in_graph()) if not referenced_vertices: origin_vertex = self.local_graph.vertices[edge.origin] destination_vertex = self.local_graph.vertices[edge.dest] if origin_vertex.block_type == BlockType.VARIABLE and destination_vertex.block_type == BlockType.MODULE: self.update_evaluated_value(changed_attribute_key=edge.label, changed_attribute_value=destination_vertex.attributes[origin_vertex.name], vertex=edge.origin, change_origin_id=edge.dest, attribute_at_dest=edge.label) return if origin_vertex.block_type == BlockType.VARIABLE and destination_vertex.block_type == BlockType.TF_VARIABLE: self.update_evaluated_value(changed_attribute_key=edge.label, changed_attribute_value=destination_vertex.attributes['default'], vertex=edge.origin, change_origin_id=edge.dest, attribute_at_dest=edge.label) return modified_vertex_attributes = self.local_graph.vertices[edge.origin].attributes val_to_eval = deepcopy(modified_vertex_attributes.get(edge.label, '')) origin_val = deepcopy(val_to_eval) first_key_path = None if referenced_vertices: for edge in edge_list: dest_vertex_attributes = self.local_graph.get_vertex_attributes_by_index(edge.dest) key_path_in_dest_vertex, replaced_key = self.find_path_from_referenced_vertices(referenced_vertices, dest_vertex_attributes) if not key_path_in_dest_vertex or not replaced_key: continue if not first_key_path: first_key_path = key_path_in_dest_vertex evaluated_attribute_value = self.extract_value_from_vertex(key_path_in_dest_vertex, dest_vertex_attributes) if evaluated_attribute_value is not None: val_to_eval = self.replace_value(edge, val_to_eval, replaced_key, evaluated_attribute_value, True) if not multiple_edges and val_to_eval != origin_val: self.update_evaluated_value(changed_attribute_key=edge.label, changed_attribute_value=val_to_eval, vertex=edge.origin, change_origin_id=edge.dest, attribute_at_dest=key_path_in_dest_vertex) if multiple_edges and val_to_eval != origin_val: self.update_evaluated_value(changed_attribute_key=edge.label, changed_attribute_value=val_to_eval, vertex=edge.origin, change_origin_id=edge.dest, attribute_at_dest=first_key_path) # Avoid loops on output => output edges if self.local_graph.vertices[edge.origin].block_type == BlockType.OUTPUT and \ self.local_graph.vertices[edge.dest].block_type == BlockType.OUTPUT: if edge.origin not in self.done_edges_by_origin_vertex: self.done_edges_by_origin_vertex[edge.origin] = [] self.done_edges_by_origin_vertex[edge.origin].append(edge)
def _set_variables_values_from_modules(self) -> List[Undetermined]: undetermined_values: List[Undetermined] = [] for module_vertex_id in self.vertices_by_block_type.get( BlockType.MODULE, []): module_vertex = self.vertices[module_vertex_id] for attribute_name, attribute_value in module_vertex.attributes.items( ): matching_variables = self.vertices_block_name_map.get( BlockType.VARIABLE, {}).get(attribute_name, []) for variable_vertex_id in matching_variables: variable_dir = os.path.dirname( self.vertices[variable_vertex_id].path) # TODO: module_vertex.path is always a string and the retrieved dict value is a nested list # therefore this condition is always false. Fixing it results in some variables not being rendered. # see test: tests.graph.terraform.variable_rendering.test_render_scenario.TestRendererScenarios.test_account_dirs_and_modules if module_vertex.path in self.module_dependency_map.get( variable_dir, []): has_var_reference = get_referenced_vertices_in_value( value=attribute_value, aliases={}, resources_types=self.get_resources_types_in_graph( )) if has_var_reference: undetermined_values.append({ "module_vertex_id": module_vertex_id, "attribute_name": attribute_name, "variable_vertex_id": variable_vertex_id, }) var_default_value = self.vertices[ variable_vertex_id].attributes.get("default") if (not has_var_reference or not var_default_value or get_referenced_vertices_in_value( value=var_default_value, aliases={}, resources_types=self. get_resources_types_in_graph())): self.update_vertex_attribute( variable_vertex_id, "default", attribute_value, module_vertex_id, attribute_name) return undetermined_values
def _set_variables_values_from_modules(self): undetermined_values = [] for module_vertex_id in self.vertices_by_block_type.get( BlockType.MODULE, []): module_vertex = self.vertices[module_vertex_id] for attribute_name in module_vertex.attributes: matching_variables = self.vertices_block_name_map.get( BlockType.VARIABLE, {}).get(attribute_name, []) for variable_vertex_id in matching_variables: variable_vertex = self.vertices[variable_vertex_id] variable_dir = os.path.dirname(variable_vertex.path) if module_vertex.path in self.module_dependency_map.get( variable_dir, []): attribute_value = module_vertex.attributes[ attribute_name] has_var_reference = get_referenced_vertices_in_value( value=attribute_value, aliases={}, resources_types=self.get_resources_types_in_graph( )) if has_var_reference: undetermined_values.append({ 'module_vertex_id': module_vertex_id, 'attribute_name': attribute_name, 'variable_vertex_id': variable_vertex_id }) var_default_value = self.vertices[ variable_vertex_id].attributes.get("default") if not has_var_reference or not var_default_value or get_referenced_vertices_in_value( value=var_default_value, aliases={}, resources_types=self. get_resources_types_in_graph()): self.update_vertex_attribute( variable_vertex_id, 'default', attribute_value, module_vertex_id, attribute_name) return undetermined_values
def process_undetermined_values( self, undetermined_values: List[Undetermined]) -> None: for undetermined in undetermined_values: module_vertex = self.vertices[undetermined["module_vertex_id"]] value = module_vertex.attributes.get( undetermined["attribute_name"]) if not get_referenced_vertices_in_value( value=value, aliases={}, resources_types=self.get_resources_types_in_graph()): self.update_vertex_attribute( undetermined["variable_vertex_id"], "default", value, undetermined["module_vertex_id"], undetermined["attribute_name"], )
def scan_resource_conf(self, conf: Dict[str, List[Any]]) -> CheckResult: self.handle_dynamic_values(conf) inspected_key = self.get_inspected_key() expected_values = self.get_expected_values() if dpath.search(conf, inspected_key) != {}: # Inspected key exists value = dpath.get(conf, inspected_key) if isinstance(value, list) and len(value) == 1: value = value[0] if ANY_VALUE in expected_values and value is not None: # Key is found on the configuration - if it accepts any value, the check is PASSED return CheckResult.PASSED if self._is_variable_dependant(value): # If the tested attribute is variable-dependant, then result is PASSED return CheckResult.PASSED if value in expected_values: return CheckResult.PASSED if get_referenced_vertices_in_value(value=value, aliases={}, resources_types=[]): # we don't provide resources_types as we want to stay provider agnostic return CheckResult.UNKNOWN return CheckResult.FAILED else: # Look for the configuration in a bottom-up fashion inspected_attributes = self._filter_key_path(inspected_key) for attribute in reversed(inspected_attributes): for sub_key, sub_conf in dpath.search(conf, f"**/{attribute}", yielded=True): filtered_sub_key = self._filter_key_path(sub_key) # Only proceed with check if full path for key is similar - not partial match if inspected_attributes == filtered_sub_key: if self._is_nesting_key(inspected_attributes, filtered_sub_key): if isinstance(sub_conf, list) and len(sub_conf) == 1: sub_conf = sub_conf[0] if sub_conf in self.get_expected_values(): return CheckResult.PASSED if self._is_variable_dependant(sub_conf): # If the tested attribute is variable-dependant, then result is PASSED return CheckResult.PASSED return self.missing_block_result
def test_find_var_blocks(self): cases: List[Tuple[str, List[VertexReference]]] = [ ("${local.one}", [ VertexReference(BlockType.LOCALS, sub_parts=["one"], origin_value="local.one") ]), ("${local.NAME[foo]}-${local.TAIL}${var.gratuitous_var_default}", [ VertexReference(BlockType.LOCALS, sub_parts=["NAME"], origin_value="local.NAME"), VertexReference(BlockType.LOCALS, sub_parts=["TAIL"], origin_value="local.TAIL"), VertexReference(BlockType.VARIABLE, sub_parts=["gratuitous_var_default"], origin_value="var.gratuitous_var_default"), ]), # Ordered returning of sub-vars and then outer var. ( "${merge(local.common_tags,local.common_data_tags,{'Name': 'my-thing-${var.ENVIRONMENT}-${var.REGION}'})}", [ VertexReference(BlockType.LOCALS, sub_parts=["common_tags"], origin_value="local.common_tags"), VertexReference(BlockType.LOCALS, sub_parts=["common_data_tags"], origin_value="local.common_data_tags"), VertexReference(BlockType.VARIABLE, sub_parts=["ENVIRONMENT"], origin_value="var.ENVIRONMENT"), VertexReference(BlockType.VARIABLE, sub_parts=["REGION"], origin_value="var.REGION"), ], ), ( "${merge(${local.common_tags},${local.common_data_tags},{'Name': 'my-thing-${var.ENVIRONMENT}-${var.REGION}'})}", [ VertexReference(BlockType.LOCALS, sub_parts=["common_tags"], origin_value="local.common_tags"), VertexReference(BlockType.LOCALS, sub_parts=["common_data_tags"], origin_value="local.common_data_tags"), VertexReference(BlockType.VARIABLE, sub_parts=["ENVIRONMENT"], origin_value="var.ENVIRONMENT"), VertexReference(BlockType.VARIABLE, sub_parts=["REGION"], origin_value="var.REGION"), ], ), ('${merge(var.tags, map("Name", "${var.name}", "data_classification", "none"))}', [ VertexReference(BlockType.VARIABLE, sub_parts=["tags"], origin_value="var.tags"), VertexReference(BlockType.VARIABLE, sub_parts=["name"], origin_value="var.name"), ]), ('${var.metadata_http_tokens_required ? "required" : "optional"}', [ VertexReference( BlockType.VARIABLE, sub_parts=["metadata_http_tokens_required"], origin_value="var.metadata_http_tokens_required"), ]), ('${local.NAME[${module.bucket.bucket_name}]}-${local.TAIL}${var.gratuitous_var_default}', [ VertexReference(BlockType.LOCALS, sub_parts=["NAME"], origin_value="local.NAME"), VertexReference(BlockType.MODULE, sub_parts=["bucket", "bucket_name"], origin_value="module.bucket.bucket_name"), VertexReference(BlockType.LOCALS, sub_parts=["TAIL"], origin_value="local.TAIL"), VertexReference(BlockType.VARIABLE, sub_parts=["gratuitous_var_default"], origin_value="var.gratuitous_var_default"), ]), ] for case in cases: actual = get_referenced_vertices_in_value(value=case[0], aliases={}, resources_types=[]) assert actual == case[1], \ f"Case \"{case[0]}\" failed ❌:\n" \ f" Expected: \n{pprint.pformat([str(c) for c in case[1]], indent=2)}\n\n" \ f" Actual: \n{pprint.pformat([str(c) for c in actual], indent=2)}" print(f"Case \"{case[0]}: ✅")
def _build_edges(self): logging.info('Creating edges') self.get_module_vertices_mapping() aliases = self._get_aliases() for origin_node_index, vertex in enumerate(self.vertices): for attribute_key in vertex.attributes: if attribute_key in reserved_attribute_names or attribute_has_nested_attributes( attribute_key, vertex.attributes): continue referenced_vertices = get_referenced_vertices_in_value( value=vertex.attributes[attribute_key], aliases=aliases, resources_types=self.get_resources_types_in_graph()) for vertex_reference in referenced_vertices: # for certain blocks such as data and resource, the block name is composed from several parts. # the purpose of the loop is to avoid not finding the node if the name has several parts sub_values = [ remove_index_pattern_from_str(sub_value) for sub_value in vertex_reference.sub_parts ] for i in range(len(sub_values)): reference_name = join_trimmed_strings( char_to_join=".", str_lst=sub_values, num_to_trim=i) if vertex.module_dependency: dest_node_index = self._find_vertex_index_relative_to_path( vertex_reference.block_type, reference_name, vertex.path, vertex.module_dependency) if dest_node_index == -1: dest_node_index = self._find_vertex_index_relative_to_path( vertex_reference.block_type, reference_name, vertex.path, vertex.path) else: dest_node_index = self._find_vertex_index_relative_to_path( vertex_reference.block_type, reference_name, vertex.path, vertex.module_dependency) if dest_node_index > -1 and origin_node_index > -1: if vertex_reference.block_type == BlockType.MODULE: try: self._connect_module( sub_values, attribute_key, self.vertices[dest_node_index], origin_node_index) except Exception as e: logging.warning( f'Module {self.vertices[dest_node_index]} does not have source attribute, skipping' ) logging.warning(e, stack_info=True) else: self._create_edge(origin_node_index, dest_node_index, attribute_key) break if vertex.block_type == BlockType.MODULE: target_path = vertex.path if vertex.module_dependency != '': target_path = unify_dependency_path( [vertex.module_dependency, vertex.path]) target_variables = list( filter( lambda v: self.vertices[v].module_dependency == target_path, self.vertices_by_block_type.get( BlockType.VARIABLE, {}))) for attribute, value in vertex.attributes.items(): if attribute in MODULE_RESERVED_ATTRIBUTES: continue target_variable = None for v in target_variables: if self.vertices[v].name == attribute: target_variable = v break if target_variable is not None: self._create_edge(target_variable, origin_node_index, 'default') elif vertex.block_type == BlockType.TF_VARIABLE: if vertex.module_dependency != '': target_path = unify_dependency_path( [vertex.module_dependency, vertex.path]) # Assuming the tfvars file is in the same directory as the variables file (best practice) target_variables = list( filter( lambda v: os.path.dirname(self.vertices[v].path) == os. path.dirname(vertex.path), self.vertices_block_name_map.get( BlockType.VARIABLE, {}).get(vertex.name, []))) if len(target_variables) == 1: self._create_edge(target_variables[0], origin_node_index, 'default')