def scan(argv):
args = parse_args(argv)
automaton = None
word_list_hash = None
if args.word_list_file:
automaton, word_list_hash = build_automaton(args.word_list_file)
# Plugins are *always* rescanned with fresh settings, because
# we want to get the latest updates.
plugins = initialize.from_parser_builder(
plugins_dict=args.plugins,
custom_plugin_paths=args.custom_plugin_paths,
exclude_lines_regex=args.exclude_lines,
automaton=automaton,
should_verify_secrets=not args.no_verify,
)
return baseline.initialize(
path=args.path,
plugins=plugins,
custom_plugin_paths=args.custom_plugin_paths,
exclude_files_regex=args.exclude_files,
exclude_lines_regex=args.exclude_lines,
word_list_file=args.word_list_file,
word_list_hash=word_list_hash,
should_scan_all_files=args.all_files,
)
def main(argv=None):
if len(sys.argv) == 1: # pragma: no cover
sys.argv.append('-h')
args = parse_args(argv)
if args.verbose: # pragma: no cover
log.set_debug_level(args.verbose)
if args.action == 'scan':
# Plugins are *always* rescanned with fresh settings, because
# we want to get the latest updates.
plugins = initialize.from_parser_builder(args.plugins)
if args.string:
line = args.string
if isinstance(args.string, bool):
line = sys.stdin.read().splitlines()[0]
_scan_string(line, plugins)
else:
baseline_dict = _perform_scan(
args,
plugins,
)
if args.import_filename:
write_baseline_to_file(
filename=args.import_filename[0],
data=baseline_dict,
)
else:
print(baseline.format_baseline_for_output(baseline_dict, ), )
elif args.action == 'audit':
if not args.diff:
audit.audit_baseline(args.filename[0])
return 0
if len(args.filename) != 2:
print(
'Must specify two files to compare!',
file=sys.stderr,
)
return 1
try:
audit.compare_baselines(args.filename[0], args.filename[1])
except audit.RedundantComparisonError:
print(
'No difference, because it\'s the same file!',
file=sys.stderr,
)
return 0
def main(argv=None):
args = parse_args(argv)
if args.verbose: # pragma: no cover
log.set_debug_level(args.verbose)
try:
# If baseline is provided, we first want to make sure
# it's valid, before doing any further computation.
baseline_collection = get_baseline(args.baseline[0])
except (IOError, ValueError):
# Error logs handled within logic.
return 1
plugins = initialize.from_parser_builder(args.plugins)
results = find_secrets_in_files(args, plugins)
if baseline_collection:
original_results = results
results = get_secrets_not_in_baseline(
results,
baseline_collection,
)
if len(results.data) > 0:
pretty_print_diagnostics(results)
return 1
if not baseline_collection:
return 0
# Only attempt baseline modifications if we don't find any new secrets
baseline_modified = trim_baseline_of_removed_secrets(
original_results,
baseline_collection,
args.filenames,
)
if VERSION != baseline_collection.version:
baseline_collection.plugins = plugins
baseline_collection.version = VERSION
baseline_modified = True
if baseline_modified:
write_baseline_to_file(
filename=args.baseline[0],
data=baseline_collection.format_for_baseline_output(),
)
log.error(
'The baseline file was updated.\n'
'Probably to keep line numbers of secrets up-to-date.\n'
'Please `git add {}`, thank you.\n\n'.format(args.baseline[0]), )
return 1
return 0
def scan(self, exclude_files_regex=None, exclude_lines_regex=None, scan_head=False):
"""Fetches latest changes, and scans the git diff between last_commit_hash
and HEAD.
:raises: subprocess.CalledProcessError
:type exclude_files_regex: str|None
:param exclude_files_regex: A regex matching filenames to skip over.
:type exclude_lines: str|None
:param exclude_lines: A regex matching lines to skip over.
:rtype: SecretsCollection
:returns: secrets found.
"""
self.storage.fetch_new_changes()
default_plugins = initialize_plugins.from_parser_builder(
self.plugin_config,
exclude_lines_regex=exclude_lines_regex,
)
# TODO Issue 17: Ignoring self.exclude_regex, using the server scan CLI arg
secrets = SecretsCollection(
plugins=default_plugins,
exclude_files=exclude_files_regex,
exclude_lines=exclude_lines_regex,
)
scan_from_this_commit = git.get_empty_tree_commit_hash() if scan_head else self.last_commit_hash
try:
diff_name_only = self.storage.get_diff_name_only(scan_from_this_commit)
# do a per-file diff + scan so we don't get a OOM if the the commit-diff is too large
for filename in diff_name_only:
file_diff = self.storage.get_diff(scan_from_this_commit, filename)
secrets.scan_diff(
file_diff,
baseline_filename=self.baseline_filename,
last_commit_hash=scan_from_this_commit,
repo_name=self.name,
)
except subprocess.CalledProcessError:
self.update()
return secrets
if self.baseline_filename:
baseline = self.storage.get_baseline_file(self.baseline_filename)
if baseline:
baseline_collection = SecretsCollection.load_baseline_from_string(baseline)
secrets = get_secrets_not_in_baseline(secrets, baseline_collection)
return secrets
def __init__(self):
self.password_replacement_regex = re.compile("|".join(self.PASSWORD_REPLACEMENTS), flags=re.I)
active_plugins = {}
for plugin in PluginOptions.all_plugins:
related_args = {}
for related_arg_tuple in plugin.related_args:
flag_name, default_value = related_arg_tuple
related_args[flag_name[2:].replace("-", "_")] = default_value
active_plugins[plugin.classname] = related_args
self._plugins = initialize.from_parser_builder(active_plugins, exclude_lines_regex=None, automaton=False, should_verify_secrets=True)
def scan(self, exclude_files_regex=None, exclude_lines_regex=None):
"""Fetches latest changes, and scans the git diff between last_commit_hash
and HEAD.
:raises: subprocess.CalledProcessError
:type exclude_files_regex: str|None
:param exclude_files_regex: A regex matching filenames to skip over.
:type exclude_lines: str|None
:param exclude_lines: A regex matching lines to skip over.
:rtype: SecretsCollection
:returns: secrets found.
"""
self.storage.fetch_new_changes()
default_plugins = initialize_plugins.from_parser_builder(
self.plugin_config,
exclude_lines_regex=exclude_lines_regex,
)
# TODO Issue 17: Ignoring self.exclude_regex, using the server scan CLI arg
secrets = SecretsCollection(
plugins=default_plugins,
exclude_files=exclude_files_regex,
exclude_lines=exclude_lines_regex,
)
try:
diff = self.storage.get_diff(self.last_commit_hash)
except subprocess.CalledProcessError:
self.update()
return secrets
secrets.scan_diff(
diff,
baseline_filename=self.baseline_filename,
last_commit_hash=self.last_commit_hash,
repo_name=self.name,
)
if self.baseline_filename:
baseline = self.storage.get_baseline_file(self.baseline_filename)
if baseline:
baseline_collection = SecretsCollection.load_baseline_from_string(
baseline)
secrets = get_secrets_not_in_baseline(secrets,
baseline_collection)
return secrets
def ScanForSecrets(text: str) -> bool:
"""Scan for secrets in the given text.
Args:
text: The text to scan.
Returns:
True (always).
Raises:
TextContainsSecret: If the text contains a secret.
"""
args = detect_secrets_main.parse_args(["scan"])
plugins = detect_secrets_initialize.from_parser_builder(
args.plugins, exclude_lines_regex="",
)
for plugin in plugins:
if plugin.analyze_string(text, 0, "does_not_matter"):
raise TextContainsSecret(plugin.__class__.__name__)
return True
def RejectSecrets(text: str) -> str:
"""Test for secrets such as private keys in a text.
Args:
text: The text to check.
Returns:
The unmodified text.
Raises:
ValueError: In case the text contains secrets.
"""
args = secrets_main.parse_args(["scan"])
plugins = secrets_init.from_parser_builder(args.plugins,
exclude_lines_regex="")
for plugin in plugins:
if plugin.analyze_string(text, 0, "does_not_matter"):
raise ValueError(plugin.__class__.__name__)
return text
def get_project_secrets(project_results_dir: str, project_name: str) -> dict:
argv = ["scan", f"{project_results_dir}/{project_name}"]
args = parse_args(argv)
automaton = None
word_list_hash = None
if args.word_list_file:
automaton, word_list_hash = build_automaton(args.word_list_file)
# Plugins are *always* rescanned with fresh settings, because
# we want to get the latest updates.
plugins = initialize.from_parser_builder(
plugins_dict=args.plugins,
custom_plugin_paths=args.custom_plugin_paths,
exclude_lines_regex=args.exclude_lines,
automaton=automaton,
should_verify_secrets=not args.no_verify,
)
baseline_dict = _perform_scan(args, plugins, automaton, word_list_hash,)
return baseline_dict
def main(argv=sys.argv[1:]):
if len(sys.argv) == 1: # pragma: no cover
sys.argv.append('--help')
args = parse_args(argv)
if args.verbose: # pragma: no cover
log.set_debug_level(args.verbose)
if args.action == 'scan':
automaton = None
word_list_hash = None
if args.word_list_file:
automaton, word_list_hash = build_automaton(args.word_list_file)
# Plugins are *always* rescanned with fresh settings, because
# we want to get the latest updates.
plugins = initialize.from_parser_builder(
plugins_dict=args.plugins,
custom_plugin_paths=args.custom_plugin_paths,
exclude_lines_regex=args.exclude_lines,
automaton=automaton,
should_verify_secrets=not args.no_verify,
)
if args.string:
line = args.string
if isinstance(args.string, bool):
line = sys.stdin.read().splitlines()[0]
_scan_string(line, plugins)
else:
baseline_dict = _perform_scan(
args,
plugins,
automaton,
word_list_hash,
)
if args.import_filename:
write_baseline_to_file(
filename=args.import_filename[0],
data=baseline_dict,
)
else:
print(baseline.format_baseline_for_output(baseline_dict, ), )
elif args.action == 'audit':
if not args.diff and not args.display_results:
audit.audit_baseline(args.filename[0])
return 0
if args.display_results:
audit.print_audit_results(args.filename[0])
return 0
if len(args.filename) != 2:
print(
'Must specify two files to compare!',
file=sys.stderr,
)
return 1
try:
audit.compare_baselines(args.filename[0], args.filename[1])
except audit.RedundantComparisonError:
print(
'No difference, because it\'s the same file!',
file=sys.stderr,
)
return 0
def main(argv=sys.argv[1:]):
args = parse_args(argv)
if args.verbose: # pragma: no cover
log.set_debug_level(args.verbose)
try:
# If baseline is provided, we first want to make sure
# it's valid, before doing any further computation.
baseline_collection = get_baseline(args.baseline[0])
except (IOError, TypeError, ValueError):
# Error logs handled within logic.
return 1
automaton = None
word_list_hash = None
if args.word_list_file:
automaton, word_list_hash = build_automaton(args.word_list_file)
plugins = initialize.from_parser_builder(
plugins_dict=args.plugins,
custom_plugin_paths=args.custom_plugin_paths,
exclude_lines_regex=args.exclude_lines,
automaton=automaton,
should_verify_secrets=not args.no_verify,
)
# Merge plugins from baseline
if baseline_collection:
plugins = initialize.merge_plugins_from_baseline(
baseline_plugins=baseline_collection.plugins,
args=args,
automaton=automaton,
)
baseline_collection.plugins = plugins
results = find_secrets_in_files(args, plugins)
if baseline_collection:
original_results = results
results = get_secrets_not_in_baseline(
results,
baseline_collection,
)
if len(results.data) > 0:
pretty_print_diagnostics(results)
return 1
if not baseline_collection:
return 0
# Only attempt baseline modifications if we don't find any new secrets
baseline_modified = trim_baseline_of_removed_secrets(
original_results,
baseline_collection,
args.filenames,
)
if VERSION != baseline_collection.version:
baseline_collection.version = VERSION
baseline_modified = True
# adding this line as we don't want the modification of baseline file.
baseline_modified = False
if baseline_modified:
write_baseline_to_file(
filename=args.baseline[0],
data=baseline_collection.format_for_baseline_output(),
)
log.error(
'The baseline file was updated.\n'
'Probably to keep line numbers of secrets up-to-date.\n'
'Please `git add {}`, thank you.\n\n'.format(args.baseline[0]), )
return 3
return 0
def main(argv=None):
version_check()
args = parse_args(argv)
if args.verbose: # pragma: no cover
log.set_debug_level(args.verbose)
try:
# If baseline is provided, we first want to make sure
# it's valid, before doing any further computation.
baseline_collection = get_baseline(
args.baseline[0],
plugin_filenames=args.plugin_filenames,
)
except (IOError, TypeError, ValueError):
# Error logs handled within logic.
return 1
automaton = None
word_list_hash = None
if args.word_list_file:
automaton, word_list_hash = build_automaton(args.word_list_file)
plugins = initialize.from_parser_builder(
args.plugins,
exclude_lines_regex=args.exclude_lines,
automaton=automaton,
should_verify_secrets=not args.no_verify,
plugin_filenames=args.plugin_filenames,
)
# Merge plugins from baseline
if baseline_collection:
plugins = initialize.merge_plugins_from_baseline(
baseline_collection.plugins,
args,
automaton,
)
baseline_collection.plugins = plugins
results_collection = find_secrets_in_files(args, plugins)
if baseline_collection:
original_results_collection = results_collection
results_collection = get_secrets_not_in_baseline(
results_collection,
baseline_collection,
)
if len(results_collection.data) > 0:
pretty_print_diagnostics_for_new_secrets(results_collection)
return 1
# if no baseline been supplied
if not baseline_collection:
return 0
# Only attempt baseline modifications if we don't find any new secrets
baseline_modified = trim_baseline_of_removed_secrets(
original_results_collection,
baseline_collection,
args.filenames,
)
if VERSION != baseline_collection.version:
baseline_collection.version = VERSION
baseline_modified = True
if baseline_modified:
write_baseline_to_file(
filename=args.baseline[0],
data=baseline_collection.format_for_baseline_output(),
)
log.error(
'The baseline file was updated.\n'
'Probably to keep line numbers of secrets up-to-date.\n'
'Please `git add {}`, thank you.\n\n'.format(args.baseline[0]),
)
return 3
# check if there are verified but haven't been audited secrets
verified_non_audited = get_verified_non_audited_secrets_from_baseline(
baseline_collection,
)
if len(verified_non_audited.data) > 0:
pretty_print_diagnostics_for_verified_non_audited(verified_non_audited)
return 2
# check if there are non-audited secrets
if args.fail_on_non_audited:
non_audited = get_non_audited_secrets_from_baseline(
baseline_collection,
)
if len(non_audited.data) > 0:
pretty_print_diagnostics_for_non_audited(non_audited)
return 4
return 0
