def test_find_non_literal_values(self):
aliases = {'aws': {CustomAttributes.BLOCK_TYPE: BlockType.PROVIDER}}
str_value = 'aws.east1'
expected = [
VertexReference(BlockType.PROVIDER, ['aws', 'east1'], 'aws.east1')
]
self.assertEqual(
expected, get_referenced_vertices_in_value(str_value, aliases, []))
str_values = [
'var.x',
'format("-%s", var.x)',
'../child',
'aws_instance.example.id',
'bc_c_${var.customer_name}',
'aws iam delete-role --role-name ${local.role_name} --profile ${var.profile} --region ${var.region}',
'length(aws_vpc.main) > 0 ? aws_vpc.main[0].cidr_block : ${var.x}',
]
expected = [
[VertexReference(BlockType.VARIABLE, ['x'], 'var.x')],
[VertexReference(BlockType.VARIABLE, ['x'], 'var.x')],
[],
[
VertexReference(BlockType.RESOURCE,
['aws_instance.example', 'id'],
'aws_instance.example.id')
],
[
VertexReference(BlockType.VARIABLE, ['customer_name'],
'var.customer_name')
],
[
VertexReference(BlockType.LOCALS, ['role_name'],
'local.role_name'),
VertexReference(BlockType.VARIABLE, ['profile'],
'var.profile'),
VertexReference(BlockType.VARIABLE, ['region'], 'var.region')
],
[
VertexReference(BlockType.RESOURCE, ['aws_vpc.main'],
'aws_vpc.main'),
VertexReference(BlockType.RESOURCE,
['aws_vpc.main', 'cidr_block'],
'aws_vpc.main.cidr_block'),
VertexReference(BlockType.VARIABLE, ['x'], 'var.x')
],
]
for i in range(0, len(str_values)):
self.assertEqual(
expected[i],
get_referenced_vertices_in_value(str_values[i], aliases,
['aws_vpc', 'aws_instance']))
def scan_resource_conf(self, conf: Dict[str, List[Any]]) -> CheckResult:
self.handle_dynamic_values(conf)
excluded_key = self.get_excluded_key()
if excluded_key is not None:
if dpath.search(conf, excluded_key) != {}:
value = dpath.get(conf, excluded_key)
if isinstance(value, list) and len(value) == 1:
value = value[0]
if self.check_excluded_condition(value):
return CheckResult.PASSED
inspected_key = self.get_inspected_key()
bad_values = self.get_forbidden_values()
if dpath.search(conf, inspected_key) != {}:
value = dpath.get(conf, inspected_key)
if isinstance(value, list) and len(value) == 1:
value = value[0]
if get_referenced_vertices_in_value(value=value, aliases={}, resources_types=[]):
# we don't provide resources_types as we want to stay provider agnostic
return CheckResult.UNKNOWN
if value in bad_values or ANY_VALUE in bad_values:
return CheckResult.FAILED
else:
return CheckResult.PASSED
return self.missing_attribute_result
def process_undetermined_values(self, undetermined_values):
for undetermined in undetermined_values:
module_vertex = self.vertices[undetermined.get('module_vertex_id')]
value = module_vertex.attributes.get(undetermined.get('attribute_name'))
if not get_referenced_vertices_in_value(value=value, aliases={},
resources_types=self.get_resources_types_in_graph()):
self.update_vertex_attribute(undetermined.get('variable_vertex_id'), 'default', value,
undetermined.get('module_vertex_id'), undetermined.get('attribute_name'))
def evaluate_vertex_attribute_from_edge(self, edge_list):
multiple_edges = len(edge_list) > 1
edge = edge_list[0]
origin_vertex_attributes = self.local_graph.vertices[edge.origin].attributes
val_to_eval = deepcopy(origin_vertex_attributes.get(edge.label, ''))
referenced_vertices = get_referenced_vertices_in_value(value=val_to_eval, aliases={},
resources_types=self.local_graph.get_resources_types_in_graph())
if not referenced_vertices:
origin_vertex = self.local_graph.vertices[edge.origin]
destination_vertex = self.local_graph.vertices[edge.dest]
if origin_vertex.block_type == BlockType.VARIABLE and destination_vertex.block_type == BlockType.MODULE:
self.update_evaluated_value(changed_attribute_key=edge.label,
changed_attribute_value=destination_vertex.attributes[origin_vertex.name], vertex=edge.origin,
change_origin_id=edge.dest, attribute_at_dest=edge.label)
return
if origin_vertex.block_type == BlockType.VARIABLE and destination_vertex.block_type == BlockType.TF_VARIABLE:
self.update_evaluated_value(changed_attribute_key=edge.label,
changed_attribute_value=destination_vertex.attributes['default'],
vertex=edge.origin,
change_origin_id=edge.dest, attribute_at_dest=edge.label)
return
modified_vertex_attributes = self.local_graph.vertices[edge.origin].attributes
val_to_eval = deepcopy(modified_vertex_attributes.get(edge.label, ''))
origin_val = deepcopy(val_to_eval)
first_key_path = None
if referenced_vertices:
for edge in edge_list:
dest_vertex_attributes = self.local_graph.get_vertex_attributes_by_index(edge.dest)
key_path_in_dest_vertex, replaced_key = self.find_path_from_referenced_vertices(referenced_vertices,
dest_vertex_attributes)
if not key_path_in_dest_vertex or not replaced_key:
continue
if not first_key_path:
first_key_path = key_path_in_dest_vertex
evaluated_attribute_value = self.extract_value_from_vertex(key_path_in_dest_vertex,
dest_vertex_attributes)
if evaluated_attribute_value is not None:
val_to_eval = self.replace_value(edge, val_to_eval, replaced_key, evaluated_attribute_value, True)
if not multiple_edges and val_to_eval != origin_val:
self.update_evaluated_value(changed_attribute_key=edge.label,
changed_attribute_value=val_to_eval, vertex=edge.origin, change_origin_id=edge.dest, attribute_at_dest=key_path_in_dest_vertex)
if multiple_edges and val_to_eval != origin_val:
self.update_evaluated_value(changed_attribute_key=edge.label,
changed_attribute_value=val_to_eval, vertex=edge.origin, change_origin_id=edge.dest, attribute_at_dest=first_key_path)
# Avoid loops on output => output edges
if self.local_graph.vertices[edge.origin].block_type == BlockType.OUTPUT and \
self.local_graph.vertices[edge.dest].block_type == BlockType.OUTPUT:
if edge.origin not in self.done_edges_by_origin_vertex:
self.done_edges_by_origin_vertex[edge.origin] = []
self.done_edges_by_origin_vertex[edge.origin].append(edge)
def _set_variables_values_from_modules(self) -> List[Undetermined]:
undetermined_values: List[Undetermined] = []
for module_vertex_id in self.vertices_by_block_type.get(
BlockType.MODULE, []):
module_vertex = self.vertices[module_vertex_id]
for attribute_name, attribute_value in module_vertex.attributes.items(
):
matching_variables = self.vertices_block_name_map.get(
BlockType.VARIABLE, {}).get(attribute_name, [])
for variable_vertex_id in matching_variables:
variable_dir = os.path.dirname(
self.vertices[variable_vertex_id].path)
# TODO: module_vertex.path is always a string and the retrieved dict value is a nested list
# therefore this condition is always false. Fixing it results in some variables not being rendered.
# see test: tests.graph.terraform.variable_rendering.test_render_scenario.TestRendererScenarios.test_account_dirs_and_modules
if module_vertex.path in self.module_dependency_map.get(
variable_dir, []):
has_var_reference = get_referenced_vertices_in_value(
value=attribute_value,
aliases={},
resources_types=self.get_resources_types_in_graph(
))
if has_var_reference:
undetermined_values.append({
"module_vertex_id":
module_vertex_id,
"attribute_name":
attribute_name,
"variable_vertex_id":
variable_vertex_id,
})
var_default_value = self.vertices[
variable_vertex_id].attributes.get("default")
if (not has_var_reference or not var_default_value
or get_referenced_vertices_in_value(
value=var_default_value,
aliases={},
resources_types=self.
get_resources_types_in_graph())):
self.update_vertex_attribute(
variable_vertex_id, "default", attribute_value,
module_vertex_id, attribute_name)
return undetermined_values
def _set_variables_values_from_modules(self):
undetermined_values = []
for module_vertex_id in self.vertices_by_block_type.get(
BlockType.MODULE, []):
module_vertex = self.vertices[module_vertex_id]
for attribute_name in module_vertex.attributes:
matching_variables = self.vertices_block_name_map.get(
BlockType.VARIABLE, {}).get(attribute_name, [])
for variable_vertex_id in matching_variables:
variable_vertex = self.vertices[variable_vertex_id]
variable_dir = os.path.dirname(variable_vertex.path)
if module_vertex.path in self.module_dependency_map.get(
variable_dir, []):
attribute_value = module_vertex.attributes[
attribute_name]
has_var_reference = get_referenced_vertices_in_value(
value=attribute_value,
aliases={},
resources_types=self.get_resources_types_in_graph(
))
if has_var_reference:
undetermined_values.append({
'module_vertex_id':
module_vertex_id,
'attribute_name':
attribute_name,
'variable_vertex_id':
variable_vertex_id
})
var_default_value = self.vertices[
variable_vertex_id].attributes.get("default")
if not has_var_reference or not var_default_value or get_referenced_vertices_in_value(
value=var_default_value,
aliases={},
resources_types=self.
get_resources_types_in_graph()):
self.update_vertex_attribute(
variable_vertex_id, 'default', attribute_value,
module_vertex_id, attribute_name)
return undetermined_values
def process_undetermined_values(
self, undetermined_values: List[Undetermined]) -> None:
for undetermined in undetermined_values:
module_vertex = self.vertices[undetermined["module_vertex_id"]]
value = module_vertex.attributes.get(
undetermined["attribute_name"])
if not get_referenced_vertices_in_value(
value=value,
aliases={},
resources_types=self.get_resources_types_in_graph()):
self.update_vertex_attribute(
undetermined["variable_vertex_id"],
"default",
value,
undetermined["module_vertex_id"],
undetermined["attribute_name"],
)
def scan_resource_conf(self, conf: Dict[str, List[Any]]) -> CheckResult:
self.handle_dynamic_values(conf)
inspected_key = self.get_inspected_key()
expected_values = self.get_expected_values()
if dpath.search(conf, inspected_key) != {}:
# Inspected key exists
value = dpath.get(conf, inspected_key)
if isinstance(value, list) and len(value) == 1:
value = value[0]
if ANY_VALUE in expected_values and value is not None:
# Key is found on the configuration - if it accepts any value, the check is PASSED
return CheckResult.PASSED
if self._is_variable_dependant(value):
# If the tested attribute is variable-dependant, then result is PASSED
return CheckResult.PASSED
if value in expected_values:
return CheckResult.PASSED
if get_referenced_vertices_in_value(value=value,
aliases={},
resources_types=[]):
# we don't provide resources_types as we want to stay provider agnostic
return CheckResult.UNKNOWN
return CheckResult.FAILED
else:
# Look for the configuration in a bottom-up fashion
inspected_attributes = self._filter_key_path(inspected_key)
for attribute in reversed(inspected_attributes):
for sub_key, sub_conf in dpath.search(conf,
f"**/{attribute}",
yielded=True):
filtered_sub_key = self._filter_key_path(sub_key)
# Only proceed with check if full path for key is similar - not partial match
if inspected_attributes == filtered_sub_key:
if self._is_nesting_key(inspected_attributes,
filtered_sub_key):
if isinstance(sub_conf,
list) and len(sub_conf) == 1:
sub_conf = sub_conf[0]
if sub_conf in self.get_expected_values():
return CheckResult.PASSED
if self._is_variable_dependant(sub_conf):
# If the tested attribute is variable-dependant, then result is PASSED
return CheckResult.PASSED
return self.missing_block_result
def test_find_var_blocks(self):
cases: List[Tuple[str, List[VertexReference]]] = [
("${local.one}", [
VertexReference(BlockType.LOCALS,
sub_parts=["one"],
origin_value="local.one")
]),
("${local.NAME[foo]}-${local.TAIL}${var.gratuitous_var_default}", [
VertexReference(BlockType.LOCALS,
sub_parts=["NAME"],
origin_value="local.NAME"),
VertexReference(BlockType.LOCALS,
sub_parts=["TAIL"],
origin_value="local.TAIL"),
VertexReference(BlockType.VARIABLE,
sub_parts=["gratuitous_var_default"],
origin_value="var.gratuitous_var_default"),
]),
# Ordered returning of sub-vars and then outer var.
(
"${merge(local.common_tags,local.common_data_tags,{'Name': 'my-thing-${var.ENVIRONMENT}-${var.REGION}'})}",
[
VertexReference(BlockType.LOCALS,
sub_parts=["common_tags"],
origin_value="local.common_tags"),
VertexReference(BlockType.LOCALS,
sub_parts=["common_data_tags"],
origin_value="local.common_data_tags"),
VertexReference(BlockType.VARIABLE,
sub_parts=["ENVIRONMENT"],
origin_value="var.ENVIRONMENT"),
VertexReference(BlockType.VARIABLE,
sub_parts=["REGION"],
origin_value="var.REGION"),
],
),
(
"${merge(${local.common_tags},${local.common_data_tags},{'Name': 'my-thing-${var.ENVIRONMENT}-${var.REGION}'})}",
[
VertexReference(BlockType.LOCALS,
sub_parts=["common_tags"],
origin_value="local.common_tags"),
VertexReference(BlockType.LOCALS,
sub_parts=["common_data_tags"],
origin_value="local.common_data_tags"),
VertexReference(BlockType.VARIABLE,
sub_parts=["ENVIRONMENT"],
origin_value="var.ENVIRONMENT"),
VertexReference(BlockType.VARIABLE,
sub_parts=["REGION"],
origin_value="var.REGION"),
],
),
('${merge(var.tags, map("Name", "${var.name}", "data_classification", "none"))}',
[
VertexReference(BlockType.VARIABLE,
sub_parts=["tags"],
origin_value="var.tags"),
VertexReference(BlockType.VARIABLE,
sub_parts=["name"],
origin_value="var.name"),
]),
('${var.metadata_http_tokens_required ? "required" : "optional"}',
[
VertexReference(
BlockType.VARIABLE,
sub_parts=["metadata_http_tokens_required"],
origin_value="var.metadata_http_tokens_required"),
]),
('${local.NAME[${module.bucket.bucket_name}]}-${local.TAIL}${var.gratuitous_var_default}',
[
VertexReference(BlockType.LOCALS,
sub_parts=["NAME"],
origin_value="local.NAME"),
VertexReference(BlockType.MODULE,
sub_parts=["bucket", "bucket_name"],
origin_value="module.bucket.bucket_name"),
VertexReference(BlockType.LOCALS,
sub_parts=["TAIL"],
origin_value="local.TAIL"),
VertexReference(BlockType.VARIABLE,
sub_parts=["gratuitous_var_default"],
origin_value="var.gratuitous_var_default"),
]),
]
for case in cases:
actual = get_referenced_vertices_in_value(value=case[0],
aliases={},
resources_types=[])
assert actual == case[1], \
f"Case \"{case[0]}\" failed ❌:\n" \
f" Expected: \n{pprint.pformat([str(c) for c in case[1]], indent=2)}\n\n" \
f" Actual: \n{pprint.pformat([str(c) for c in actual], indent=2)}"
print(f"Case \"{case[0]}: ✅")
def _build_edges(self):
logging.info('Creating edges')
self.get_module_vertices_mapping()
aliases = self._get_aliases()
for origin_node_index, vertex in enumerate(self.vertices):
for attribute_key in vertex.attributes:
if attribute_key in reserved_attribute_names or attribute_has_nested_attributes(
attribute_key, vertex.attributes):
continue
referenced_vertices = get_referenced_vertices_in_value(
value=vertex.attributes[attribute_key],
aliases=aliases,
resources_types=self.get_resources_types_in_graph())
for vertex_reference in referenced_vertices:
# for certain blocks such as data and resource, the block name is composed from several parts.
# the purpose of the loop is to avoid not finding the node if the name has several parts
sub_values = [
remove_index_pattern_from_str(sub_value)
for sub_value in vertex_reference.sub_parts
]
for i in range(len(sub_values)):
reference_name = join_trimmed_strings(
char_to_join=".",
str_lst=sub_values,
num_to_trim=i)
if vertex.module_dependency:
dest_node_index = self._find_vertex_index_relative_to_path(
vertex_reference.block_type, reference_name,
vertex.path, vertex.module_dependency)
if dest_node_index == -1:
dest_node_index = self._find_vertex_index_relative_to_path(
vertex_reference.block_type,
reference_name, vertex.path, vertex.path)
else:
dest_node_index = self._find_vertex_index_relative_to_path(
vertex_reference.block_type, reference_name,
vertex.path, vertex.module_dependency)
if dest_node_index > -1 and origin_node_index > -1:
if vertex_reference.block_type == BlockType.MODULE:
try:
self._connect_module(
sub_values, attribute_key,
self.vertices[dest_node_index],
origin_node_index)
except Exception as e:
logging.warning(
f'Module {self.vertices[dest_node_index]} does not have source attribute, skipping'
)
logging.warning(e, stack_info=True)
else:
self._create_edge(origin_node_index,
dest_node_index,
attribute_key)
break
if vertex.block_type == BlockType.MODULE:
target_path = vertex.path
if vertex.module_dependency != '':
target_path = unify_dependency_path(
[vertex.module_dependency, vertex.path])
target_variables = list(
filter(
lambda v: self.vertices[v].module_dependency ==
target_path,
self.vertices_by_block_type.get(
BlockType.VARIABLE, {})))
for attribute, value in vertex.attributes.items():
if attribute in MODULE_RESERVED_ATTRIBUTES:
continue
target_variable = None
for v in target_variables:
if self.vertices[v].name == attribute:
target_variable = v
break
if target_variable is not None:
self._create_edge(target_variable, origin_node_index,
'default')
elif vertex.block_type == BlockType.TF_VARIABLE:
if vertex.module_dependency != '':
target_path = unify_dependency_path(
[vertex.module_dependency, vertex.path])
# Assuming the tfvars file is in the same directory as the variables file (best practice)
target_variables = list(
filter(
lambda v: os.path.dirname(self.vertices[v].path) == os.
path.dirname(vertex.path),
self.vertices_block_name_map.get(
BlockType.VARIABLE, {}).get(vertex.name, [])))
if len(target_variables) == 1:
self._create_edge(target_variables[0], origin_node_index,
'default')
