def scan(argv): args = parse_args(argv) automaton = None word_list_hash = None if args.word_list_file: automaton, word_list_hash = build_automaton(args.word_list_file) # Plugins are *always* rescanned with fresh settings, because # we want to get the latest updates. plugins = initialize.from_parser_builder( plugins_dict=args.plugins, custom_plugin_paths=args.custom_plugin_paths, exclude_lines_regex=args.exclude_lines, automaton=automaton, should_verify_secrets=not args.no_verify, ) return baseline.initialize( path=args.path, plugins=plugins, custom_plugin_paths=args.custom_plugin_paths, exclude_files_regex=args.exclude_files, exclude_lines_regex=args.exclude_lines, word_list_file=args.word_list_file, word_list_hash=word_list_hash, should_scan_all_files=args.all_files, )
def main(argv=None): if len(sys.argv) == 1: # pragma: no cover sys.argv.append('-h') args = parse_args(argv) if args.verbose: # pragma: no cover log.set_debug_level(args.verbose) if args.action == 'scan': # Plugins are *always* rescanned with fresh settings, because # we want to get the latest updates. plugins = initialize.from_parser_builder(args.plugins) if args.string: line = args.string if isinstance(args.string, bool): line = sys.stdin.read().splitlines()[0] _scan_string(line, plugins) else: baseline_dict = _perform_scan( args, plugins, ) if args.import_filename: write_baseline_to_file( filename=args.import_filename[0], data=baseline_dict, ) else: print(baseline.format_baseline_for_output(baseline_dict, ), ) elif args.action == 'audit': if not args.diff: audit.audit_baseline(args.filename[0]) return 0 if len(args.filename) != 2: print( 'Must specify two files to compare!', file=sys.stderr, ) return 1 try: audit.compare_baselines(args.filename[0], args.filename[1]) except audit.RedundantComparisonError: print( 'No difference, because it\'s the same file!', file=sys.stderr, ) return 0
def main(argv=None): args = parse_args(argv) if args.verbose: # pragma: no cover log.set_debug_level(args.verbose) try: # If baseline is provided, we first want to make sure # it's valid, before doing any further computation. baseline_collection = get_baseline(args.baseline[0]) except (IOError, ValueError): # Error logs handled within logic. return 1 plugins = initialize.from_parser_builder(args.plugins) results = find_secrets_in_files(args, plugins) if baseline_collection: original_results = results results = get_secrets_not_in_baseline( results, baseline_collection, ) if len(results.data) > 0: pretty_print_diagnostics(results) return 1 if not baseline_collection: return 0 # Only attempt baseline modifications if we don't find any new secrets baseline_modified = trim_baseline_of_removed_secrets( original_results, baseline_collection, args.filenames, ) if VERSION != baseline_collection.version: baseline_collection.plugins = plugins baseline_collection.version = VERSION baseline_modified = True if baseline_modified: write_baseline_to_file( filename=args.baseline[0], data=baseline_collection.format_for_baseline_output(), ) log.error( 'The baseline file was updated.\n' 'Probably to keep line numbers of secrets up-to-date.\n' 'Please `git add {}`, thank you.\n\n'.format(args.baseline[0]), ) return 1 return 0
def scan(self, exclude_files_regex=None, exclude_lines_regex=None, scan_head=False): """Fetches latest changes, and scans the git diff between last_commit_hash and HEAD. :raises: subprocess.CalledProcessError :type exclude_files_regex: str|None :param exclude_files_regex: A regex matching filenames to skip over. :type exclude_lines: str|None :param exclude_lines: A regex matching lines to skip over. :rtype: SecretsCollection :returns: secrets found. """ self.storage.fetch_new_changes() default_plugins = initialize_plugins.from_parser_builder( self.plugin_config, exclude_lines_regex=exclude_lines_regex, ) # TODO Issue 17: Ignoring self.exclude_regex, using the server scan CLI arg secrets = SecretsCollection( plugins=default_plugins, exclude_files=exclude_files_regex, exclude_lines=exclude_lines_regex, ) scan_from_this_commit = git.get_empty_tree_commit_hash() if scan_head else self.last_commit_hash try: diff_name_only = self.storage.get_diff_name_only(scan_from_this_commit) # do a per-file diff + scan so we don't get a OOM if the the commit-diff is too large for filename in diff_name_only: file_diff = self.storage.get_diff(scan_from_this_commit, filename) secrets.scan_diff( file_diff, baseline_filename=self.baseline_filename, last_commit_hash=scan_from_this_commit, repo_name=self.name, ) except subprocess.CalledProcessError: self.update() return secrets if self.baseline_filename: baseline = self.storage.get_baseline_file(self.baseline_filename) if baseline: baseline_collection = SecretsCollection.load_baseline_from_string(baseline) secrets = get_secrets_not_in_baseline(secrets, baseline_collection) return secrets
def __init__(self): self.password_replacement_regex = re.compile("|".join(self.PASSWORD_REPLACEMENTS), flags=re.I) active_plugins = {} for plugin in PluginOptions.all_plugins: related_args = {} for related_arg_tuple in plugin.related_args: flag_name, default_value = related_arg_tuple related_args[flag_name[2:].replace("-", "_")] = default_value active_plugins[plugin.classname] = related_args self._plugins = initialize.from_parser_builder(active_plugins, exclude_lines_regex=None, automaton=False, should_verify_secrets=True)
def scan(self, exclude_files_regex=None, exclude_lines_regex=None): """Fetches latest changes, and scans the git diff between last_commit_hash and HEAD. :raises: subprocess.CalledProcessError :type exclude_files_regex: str|None :param exclude_files_regex: A regex matching filenames to skip over. :type exclude_lines: str|None :param exclude_lines: A regex matching lines to skip over. :rtype: SecretsCollection :returns: secrets found. """ self.storage.fetch_new_changes() default_plugins = initialize_plugins.from_parser_builder( self.plugin_config, exclude_lines_regex=exclude_lines_regex, ) # TODO Issue 17: Ignoring self.exclude_regex, using the server scan CLI arg secrets = SecretsCollection( plugins=default_plugins, exclude_files=exclude_files_regex, exclude_lines=exclude_lines_regex, ) try: diff = self.storage.get_diff(self.last_commit_hash) except subprocess.CalledProcessError: self.update() return secrets secrets.scan_diff( diff, baseline_filename=self.baseline_filename, last_commit_hash=self.last_commit_hash, repo_name=self.name, ) if self.baseline_filename: baseline = self.storage.get_baseline_file(self.baseline_filename) if baseline: baseline_collection = SecretsCollection.load_baseline_from_string( baseline) secrets = get_secrets_not_in_baseline(secrets, baseline_collection) return secrets
def ScanForSecrets(text: str) -> bool: """Scan for secrets in the given text. Args: text: The text to scan. Returns: True (always). Raises: TextContainsSecret: If the text contains a secret. """ args = detect_secrets_main.parse_args(["scan"]) plugins = detect_secrets_initialize.from_parser_builder( args.plugins, exclude_lines_regex="", ) for plugin in plugins: if plugin.analyze_string(text, 0, "does_not_matter"): raise TextContainsSecret(plugin.__class__.__name__) return True
def RejectSecrets(text: str) -> str: """Test for secrets such as private keys in a text. Args: text: The text to check. Returns: The unmodified text. Raises: ValueError: In case the text contains secrets. """ args = secrets_main.parse_args(["scan"]) plugins = secrets_init.from_parser_builder(args.plugins, exclude_lines_regex="") for plugin in plugins: if plugin.analyze_string(text, 0, "does_not_matter"): raise ValueError(plugin.__class__.__name__) return text
def get_project_secrets(project_results_dir: str, project_name: str) -> dict: argv = ["scan", f"{project_results_dir}/{project_name}"] args = parse_args(argv) automaton = None word_list_hash = None if args.word_list_file: automaton, word_list_hash = build_automaton(args.word_list_file) # Plugins are *always* rescanned with fresh settings, because # we want to get the latest updates. plugins = initialize.from_parser_builder( plugins_dict=args.plugins, custom_plugin_paths=args.custom_plugin_paths, exclude_lines_regex=args.exclude_lines, automaton=automaton, should_verify_secrets=not args.no_verify, ) baseline_dict = _perform_scan(args, plugins, automaton, word_list_hash,) return baseline_dict
def main(argv=sys.argv[1:]): if len(sys.argv) == 1: # pragma: no cover sys.argv.append('--help') args = parse_args(argv) if args.verbose: # pragma: no cover log.set_debug_level(args.verbose) if args.action == 'scan': automaton = None word_list_hash = None if args.word_list_file: automaton, word_list_hash = build_automaton(args.word_list_file) # Plugins are *always* rescanned with fresh settings, because # we want to get the latest updates. plugins = initialize.from_parser_builder( plugins_dict=args.plugins, custom_plugin_paths=args.custom_plugin_paths, exclude_lines_regex=args.exclude_lines, automaton=automaton, should_verify_secrets=not args.no_verify, ) if args.string: line = args.string if isinstance(args.string, bool): line = sys.stdin.read().splitlines()[0] _scan_string(line, plugins) else: baseline_dict = _perform_scan( args, plugins, automaton, word_list_hash, ) if args.import_filename: write_baseline_to_file( filename=args.import_filename[0], data=baseline_dict, ) else: print(baseline.format_baseline_for_output(baseline_dict, ), ) elif args.action == 'audit': if not args.diff and not args.display_results: audit.audit_baseline(args.filename[0]) return 0 if args.display_results: audit.print_audit_results(args.filename[0]) return 0 if len(args.filename) != 2: print( 'Must specify two files to compare!', file=sys.stderr, ) return 1 try: audit.compare_baselines(args.filename[0], args.filename[1]) except audit.RedundantComparisonError: print( 'No difference, because it\'s the same file!', file=sys.stderr, ) return 0
def main(argv=sys.argv[1:]): args = parse_args(argv) if args.verbose: # pragma: no cover log.set_debug_level(args.verbose) try: # If baseline is provided, we first want to make sure # it's valid, before doing any further computation. baseline_collection = get_baseline(args.baseline[0]) except (IOError, TypeError, ValueError): # Error logs handled within logic. return 1 automaton = None word_list_hash = None if args.word_list_file: automaton, word_list_hash = build_automaton(args.word_list_file) plugins = initialize.from_parser_builder( plugins_dict=args.plugins, custom_plugin_paths=args.custom_plugin_paths, exclude_lines_regex=args.exclude_lines, automaton=automaton, should_verify_secrets=not args.no_verify, ) # Merge plugins from baseline if baseline_collection: plugins = initialize.merge_plugins_from_baseline( baseline_plugins=baseline_collection.plugins, args=args, automaton=automaton, ) baseline_collection.plugins = plugins results = find_secrets_in_files(args, plugins) if baseline_collection: original_results = results results = get_secrets_not_in_baseline( results, baseline_collection, ) if len(results.data) > 0: pretty_print_diagnostics(results) return 1 if not baseline_collection: return 0 # Only attempt baseline modifications if we don't find any new secrets baseline_modified = trim_baseline_of_removed_secrets( original_results, baseline_collection, args.filenames, ) if VERSION != baseline_collection.version: baseline_collection.version = VERSION baseline_modified = True # adding this line as we don't want the modification of baseline file. baseline_modified = False if baseline_modified: write_baseline_to_file( filename=args.baseline[0], data=baseline_collection.format_for_baseline_output(), ) log.error( 'The baseline file was updated.\n' 'Probably to keep line numbers of secrets up-to-date.\n' 'Please `git add {}`, thank you.\n\n'.format(args.baseline[0]), ) return 3 return 0
def main(argv=None): version_check() args = parse_args(argv) if args.verbose: # pragma: no cover log.set_debug_level(args.verbose) try: # If baseline is provided, we first want to make sure # it's valid, before doing any further computation. baseline_collection = get_baseline( args.baseline[0], plugin_filenames=args.plugin_filenames, ) except (IOError, TypeError, ValueError): # Error logs handled within logic. return 1 automaton = None word_list_hash = None if args.word_list_file: automaton, word_list_hash = build_automaton(args.word_list_file) plugins = initialize.from_parser_builder( args.plugins, exclude_lines_regex=args.exclude_lines, automaton=automaton, should_verify_secrets=not args.no_verify, plugin_filenames=args.plugin_filenames, ) # Merge plugins from baseline if baseline_collection: plugins = initialize.merge_plugins_from_baseline( baseline_collection.plugins, args, automaton, ) baseline_collection.plugins = plugins results_collection = find_secrets_in_files(args, plugins) if baseline_collection: original_results_collection = results_collection results_collection = get_secrets_not_in_baseline( results_collection, baseline_collection, ) if len(results_collection.data) > 0: pretty_print_diagnostics_for_new_secrets(results_collection) return 1 # if no baseline been supplied if not baseline_collection: return 0 # Only attempt baseline modifications if we don't find any new secrets baseline_modified = trim_baseline_of_removed_secrets( original_results_collection, baseline_collection, args.filenames, ) if VERSION != baseline_collection.version: baseline_collection.version = VERSION baseline_modified = True if baseline_modified: write_baseline_to_file( filename=args.baseline[0], data=baseline_collection.format_for_baseline_output(), ) log.error( 'The baseline file was updated.\n' 'Probably to keep line numbers of secrets up-to-date.\n' 'Please `git add {}`, thank you.\n\n'.format(args.baseline[0]), ) return 3 # check if there are verified but haven't been audited secrets verified_non_audited = get_verified_non_audited_secrets_from_baseline( baseline_collection, ) if len(verified_non_audited.data) > 0: pretty_print_diagnostics_for_verified_non_audited(verified_non_audited) return 2 # check if there are non-audited secrets if args.fail_on_non_audited: non_audited = get_non_audited_secrets_from_baseline( baseline_collection, ) if len(non_audited.data) > 0: pretty_print_diagnostics_for_non_audited(non_audited) return 4 return 0