def render_variables_from_local_graph(self) -> None:
# find vertices with out-degree = 0 and in-degree > 0
end_vertices_indexes = self.local_graph.get_vertices_with_degrees_conditions(
out_degree_cond=lambda degree: degree == 0,
in_degree_cond=lambda degree: degree > 0)
# all the edges entering `end_vertices`
edges_to_render = self.local_graph.get_in_edges(end_vertices_indexes)
loops = 0
while len(edges_to_render) > 0:
logging.info(f"evaluating {len(edges_to_render)} edges")
# group edges that have the same origin and label together
edges_groups = self.group_edges_by_origin_and_label(
edges_to_render)
if self.run_async:
run_function_multithreaded(
func=self._edge_evaluation_task,
data=edges_groups,
max_group_size=1,
num_of_workers=self.max_workers,
)
else:
for edge_group in edges_groups:
self._edge_evaluation_task([edge_group])
for edge in edges_to_render:
origin = edge.origin
if origin not in self.done_edges_by_origin_vertex:
self.done_edges_by_origin_vertex[origin] = []
self.done_edges_by_origin_vertex[origin].append(edge)
for edge in edges_to_render:
origin_vertex_index = edge.origin
out_edges = self.local_graph.out_edges.get(
origin_vertex_index, [])
if all(e in self.done_edges_by_origin_vertex.get(
origin_vertex_index, []) for e in out_edges):
end_vertices_indexes.append(origin_vertex_index)
edges_to_render = self.local_graph.get_in_edges(
end_vertices_indexes)
edges_to_render = list(
set([
edge for edge in edges_to_render
if edge not in self.done_edges_by_origin_vertex.get(
edge.origin, [])
]))
loops += 1
if loops >= 50:
logging.warning(
f"Reached 50 graph edge iterations, breaking. Module: {self.local_graph.module.source_dir}"
)
break
self.local_graph.update_vertices_configs()
logging.info("done evaluating edges")
self.evaluate_non_rendered_values()
def run(self, root_folder, external_checks_dir=None, files=None, runner_filter=RunnerFilter(),
collect_skip_comments=True) -> Report:
secrets = SecretsCollection()
with transient_settings({
# Only run scans with only these plugins.
'plugins_used': [
{
'name': 'AWSKeyDetector'
},
{
'name': 'ArtifactoryDetector'
},
{
'name': 'AzureStorageKeyDetector'
},
{
'name': 'BasicAuthDetector'
},
{
'name': 'CloudantDetector'
},
{
'name': 'IbmCloudIamDetector'
},
{
'name': 'MailchimpDetector'
},
{
'name': 'PrivateKeyDetector'
},
{
'name': 'SlackDetector'
},
{
'name': 'SoftlayerDetector'
},
{
'name': 'SquareOAuthDetector'
},
{
'name': 'StripeDetector'
},
{
'name': 'TwilioKeyDetector'
},
]
}):
report = Report(self.check_type)
# Implement non IaC files (including .terraform dir)
files_to_scan = files or []
excluded_paths = (runner_filter.excluded_paths or []) + ignored_directories + [DEFAULT_EXTERNAL_MODULES_DIR]
if root_folder:
for root, d_names, f_names in os.walk(root_folder):
filter_ignored_paths(root, d_names, runner_filter.excluded_paths)
filter_ignored_paths(root, f_names, runner_filter.excluded_paths)
for file in f_names:
if file not in PROHIBITED_FILES and f".{file.split('.')[-1]}" in SUPPORTED_FILE_EXTENSIONS:
files_to_scan.append(os.path.join(root, file))
logging.info(f'Secrets scanning will scan {len(files_to_scan)} files')
# TODO: re-enable filter when re-adding `SecretKeyword` plugin
scan.get_settings().disable_filters(*['detect_secrets.filters.heuristic.is_indirect_reference'])
def _scan_file(file_paths: List[str]):
for file_path in file_paths:
start = time.time()
try:
secrets.scan_file(file_path)
except Exception as err:
logging.warning(f"Secret scanning:could not process file {file_path}, {err}")
end = time.time()
scan_time = end - start
if scan_time > 10:
logging.info(f'Scanned {file_path}, took {scan_time} seconds')
run_function_multithreaded(_scan_file, files_to_scan, 1, num_of_workers=os.cpu_count())
for _, secret in iter(secrets):
check_id = SECRET_TYPE_TO_ID.get(secret.type)
if not check_id:
continue
result = {'result': CheckResult.FAILED}
line_text = linecache.getline(os.path.join(root_folder, secret.filename),
secret.line_number) if root_folder else linecache.getline(secret.filename,
secret.line_number)
if line_text != "" and line_text.split()[0] == 'git_commit':
continue
result = self.search_for_suppression(check_id, root_folder, secret, runner_filter.skip_checks,
CHECK_ID_TO_SECRET_TYPE) or result
report.add_record(Record(
check_id=check_id,
check_name=secret.type,
check_result=result,
code_block=[(secret.line_number, line_text)],
file_path=f'/{os.path.relpath(secret.filename, root_folder)}',
file_line_range=[secret.line_number, secret.line_number + 1],
resource=secret.secret_hash,
check_class=None,
evaluations=None,
file_abs_path=os.path.abspath(secret.filename)
))
return report