def test_configure_regexes_does_not_clone_if_local_rules_repo_defined( self, mock_clone ): runner = CliRunner() with runner.isolated_filesystem(): config.configure_regexes(rules_repo=".") mock_clone.assert_not_called()
def test_configure_regexes_clones_git_rules_repo(self, mock_clone): runner = CliRunner() with runner.isolated_filesystem(): mock_clone.return_value = (pathlib.Path(".").resolve(), "origin") config.configure_regexes( rules_repo="[email protected]:godaddy/tartufo.git") mock_clone.assert_called_once_with( "[email protected]:godaddy/tartufo.git")
def test_configure_regexes_grabs_all_json_from_rules_repo_by_default( self, mock_pathlib): runner = CliRunner() with runner.isolated_filesystem(): repo_path = mock_pathlib.Path.return_value repo_path.is_dir.return_value = True repo_path.glob.return_value = [] config.configure_regexes(rules_repo=".") repo_path.glob.assert_called_once_with("*.json")
def test_configure_regexes_grabs_specified_rules_files_from_repo( self, mock_pathlib): runner = CliRunner() with runner.isolated_filesystem(): repo_path = mock_pathlib.Path.return_value repo_path.is_dir.return_value = True repo_path.glob.return_value = [] config.configure_regexes(rules_repo=".", rules_repo_files=("tartufo.json", )) repo_path.glob.assert_called_once_with("tartufo.json")
def test_configure_regexes_rules_files_with_defaults(self): rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json" rules_files = (rules_path.open(), ) with config.DEFAULT_PATTERN_FILE.open() as handle: expected_regexes = config.load_rules_from_file(handle) expected_regexes.add( Rule( name="RSA private key 2", pattern=re.compile("-----BEGIN EC PRIVATE KEY-----"), path_pattern=None, re_match_type=MatchType.Match, re_match_scope=None, )) expected_regexes.add( Rule( name="Complex Rule", pattern=re.compile("complex-rule"), path_pattern=re.compile("/tmp/[a-z0-9A-Z]+\\.(py|js|json)"), re_match_type=MatchType.Match, re_match_scope=None, )) actual_regexes = config.configure_regexes(include_default=True, rules_files=rules_files) self.assertEqual( expected_regexes, actual_regexes, f"The regexes dictionary should match the test rules (expected: {expected_regexes}, actual: {actual_regexes})", )
def test_rule_patterns_without_defaults(self): rule_patterns = [ { "reason": "RSA private key 2", "pattern": "-----BEGIN EC PRIVATE KEY-----", }, { "reason": "Complex Rule", "pattern": "complex-rule", "path-pattern": "/tmp/[a-z0-9A-Z]+\\.(py|js|json)", }, ] expected = { Rule( name="RSA private key 2", pattern=re.compile("-----BEGIN EC PRIVATE KEY-----"), path_pattern=re.compile(""), re_match_type=MatchType.Search, re_match_scope=None, ), Rule( name="Complex Rule", pattern=re.compile("complex-rule"), path_pattern=re.compile("/tmp/[a-z0-9A-Z]+\\.(py|js|json)"), re_match_type=MatchType.Search, re_match_scope=None, ), } actual = config.configure_regexes(rule_patterns=rule_patterns, include_default=False) self.assertEqual(actual, expected)
def test_configure_regexes_includes_rules_from_rules_repo(self): rules_path = pathlib.Path(__file__).parent / "data" actual_regexes = config.configure_regexes( include_default=False, rules_repo=str(rules_path), rules_repo_files=["testRules.json"], ) expected_regexes = { Rule( name="RSA private key 2", pattern=re.compile("-----BEGIN EC PRIVATE KEY-----"), path_pattern=None, re_match_type=MatchType.Match, re_match_scope=None, ), Rule( name="Complex Rule", pattern=re.compile("complex-rule"), path_pattern=re.compile("/tmp/[a-z0-9A-Z]+\\.(py|js|json)"), re_match_type=MatchType.Match, re_match_scope=None, ), } self.assertEqual( expected_regexes, actual_regexes, f"The regexes dictionary should match the test rules (expected: {expected_regexes}, actual: {actual_regexes})", )
def test_configure_regexes_rules_files_with_defaults(self): rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json" rules_files = (rules_path.open(),) expected_regexes = copy.copy(config.DEFAULT_REGEXES) expected_regexes["RSA private key 2"] = Rule( name="RSA private key 2", pattern=re.compile("-----BEGIN EC PRIVATE KEY-----"), path_pattern=None, ) expected_regexes["Complex Rule"] = Rule( name="Complex Rule", pattern=re.compile("complex-rule"), path_pattern=re.compile("/tmp/[a-z0-9A-Z]+\\.(py|js|json)"), ) actual_regexes = config.configure_regexes( include_default=True, rules_files=rules_files ) self.assertEqual( expected_regexes, actual_regexes, "The regexes dictionary should match the test rules " "(expected: {}, actual: {})".format(expected_regexes, actual_regexes), )
def test_configure_regexes_returns_just_default_regexes_by_default(self): actual_regexes = config.configure_regexes() self.assertEqual( config.DEFAULT_REGEXES, actual_regexes, "The regexes dictionary should not have been changed when no rules files are specified", )
def test_configure_regexes_returns_just_default_regexes_by_default(self): actual_regexes = config.configure_regexes() with config.DEFAULT_PATTERN_FILE.open() as handle: expected_regexes = config.load_rules_from_file(handle) self.assertEqual( expected_regexes, actual_regexes, "The regexes dictionary should not have been changed when no rules files are specified", )
def rules_regexes(self) -> Dict[str, Rule]: """Get a dictionary of regular expressions to scan the code for. :raises types.TartufoConfigException: If there was a problem compiling the rules :rtype: Dict[str, Pattern] """ if self._rules_regexes is None: try: self._rules_regexes = config.configure_regexes( self.global_options.default_regexes, self.global_options.rules, self.global_options.git_rules_repo, self.global_options.git_rules_files, ) except (ValueError, re.error) as exc: raise types.ConfigException(str(exc)) from exc return self._rules_regexes
def test_configure_regexes_rules_files_without_defaults(self): rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json" rules_files = (rules_path.open(),) expected_regexes = { "RSA private key 2": re.compile("-----BEGIN EC PRIVATE KEY-----") } actual_regexes = config.configure_regexes( include_default=False, rules_files=rules_files ) self.assertEqual( expected_regexes, actual_regexes, "The regexes dictionary should match the test rules " "(expected: {}, actual: {})".format(expected_regexes, actual_regexes), )
def rules_regexes(self) -> Set[Rule]: """Get a set of regular expressions to scan the code for. :raises types.ConfigException: If there was a problem compiling the rules """ if self._rules_regexes is None: self.logger.info("Initializing regex rules") try: self._rules_regexes = config.configure_regexes( include_default=self.global_options.default_regexes, rules_files=self.global_options.rules, rule_patterns=self.global_options.rule_patterns, rules_repo=self.global_options.git_rules_repo, rules_repo_files=self.global_options.git_rules_files, ) except (ValueError, re.error) as exc: self.logger.exception("Error loading regex rules", exc_info=exc) raise types.ConfigException(str(exc)) from exc self.logger.debug("Regex rules were initialized as: %s", self._rules_regexes) return self._rules_regexes
def test_config_exception_is_raised_if_pattern_is_missing(self): with self.assertRaisesRegex(ConfigException, "Invalid rule-pattern"): config.configure_regexes(rule_patterns=[{"reason": "foo"}])
def test_loading_rules_from_file_raises_deprecation_warning(self): rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json" rules_files = (rules_path.open(), ) with self.assertWarnsRegex(DeprecationWarning, "has been deprecated and will be removed."): config.configure_regexes(rules_files=rules_files)
def main(ctx: click.Context, **kwargs: config.OptionTypes) -> None: """Find secrets hidden in the depths of git. Tartufo will, by default, scan the entire history of a git repository for any text which looks like a secret, password, credential, etc. It can also be made to work in pre-commit mode, for scanning blobs of text as a pre-commit hook. """ if not any((kwargs["entropy"], kwargs["regex"])): err("No analysis requested.") ctx.exit(1) if not any((kwargs["pre_commit"], kwargs["repo_path"], kwargs["git_url"])): err("You must specify one of --pre-commit, --repo-path, or git_url.") ctx.exit(1) if kwargs["regex"]: try: rules_regexes = config.configure_regexes( cast(bool, kwargs["default_regexes"]), cast(Tuple[TextIO, ...], kwargs["rules"]), cast(Optional[str], kwargs["git_rules_repo"]), cast(Tuple[str, ...], kwargs["git_rules_files"]), ) except ValueError as exc: err(str(exc)) ctx.exit(1) if not rules_regexes: err("Regex checks requested, but no regexes found.") ctx.exit(1) else: rules_regexes = {} # read & compile path inclusion/exclusion patterns path_inclusions = [] # type: List[Pattern] path_exclusions = [] # type: List[Pattern] paths_file = cast(TextIO, kwargs["include_paths"]) if paths_file: path_inclusions = config.compile_path_rules(paths_file.readlines()) paths_file = cast(TextIO, kwargs["exclude_paths"]) if paths_file: path_exclusions = config.compile_path_rules(paths_file.readlines()) if kwargs["pre_commit"]: output = scanner.find_staged( cast(str, kwargs["repo_path"]), cast(bool, kwargs["json"]), cast(bool, kwargs["regex"]), cast(bool, kwargs["entropy"]), custom_regexes=rules_regexes, suppress_output=False, path_inclusions=path_inclusions, path_exclusions=path_exclusions, ) else: remove_repo = False if kwargs["git_url"]: repo_path = util.clone_git_repo(cast(str, kwargs["git_url"])) remove_repo = True else: repo_path = cast(str, kwargs["repo_path"]) output = scanner.scan_repo(repo_path, rules_regexes, path_inclusions, path_exclusions, kwargs) if remove_repo: shutil.rmtree(repo_path, onerror=util.del_rw) if kwargs["cleanup"]: util.clean_outputs(output) else: issues_path = output.get("issues_path", None) if issues_path: print("Results have been saved in {}".format(issues_path)) if output.get("found_issues", False): ctx.exit(1) ctx.exit(0)