def test_configure_regexes_does_not_clone_if_local_rules_repo_defined(
self, mock_clone
):
runner = CliRunner()
with runner.isolated_filesystem():
config.configure_regexes(rules_repo=".")
mock_clone.assert_not_called()
def test_configure_regexes_clones_git_rules_repo(self, mock_clone):
runner = CliRunner()
with runner.isolated_filesystem():
mock_clone.return_value = (pathlib.Path(".").resolve(), "origin")
config.configure_regexes(
rules_repo="[email protected]:godaddy/tartufo.git")
mock_clone.assert_called_once_with(
"[email protected]:godaddy/tartufo.git")
def test_configure_regexes_grabs_all_json_from_rules_repo_by_default(
self, mock_pathlib):
runner = CliRunner()
with runner.isolated_filesystem():
repo_path = mock_pathlib.Path.return_value
repo_path.is_dir.return_value = True
repo_path.glob.return_value = []
config.configure_regexes(rules_repo=".")
repo_path.glob.assert_called_once_with("*.json")
def test_configure_regexes_grabs_specified_rules_files_from_repo(
self, mock_pathlib):
runner = CliRunner()
with runner.isolated_filesystem():
repo_path = mock_pathlib.Path.return_value
repo_path.is_dir.return_value = True
repo_path.glob.return_value = []
config.configure_regexes(rules_repo=".",
rules_repo_files=("tartufo.json", ))
repo_path.glob.assert_called_once_with("tartufo.json")
def test_configure_regexes_rules_files_with_defaults(self):
rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json"
rules_files = (rules_path.open(), )
with config.DEFAULT_PATTERN_FILE.open() as handle:
expected_regexes = config.load_rules_from_file(handle)
expected_regexes.add(
Rule(
name="RSA private key 2",
pattern=re.compile("-----BEGIN EC PRIVATE KEY-----"),
path_pattern=None,
re_match_type=MatchType.Match,
re_match_scope=None,
))
expected_regexes.add(
Rule(
name="Complex Rule",
pattern=re.compile("complex-rule"),
path_pattern=re.compile("/tmp/[a-z0-9A-Z]+\\.(py|js|json)"),
re_match_type=MatchType.Match,
re_match_scope=None,
))
actual_regexes = config.configure_regexes(include_default=True,
rules_files=rules_files)
self.assertEqual(
expected_regexes,
actual_regexes,
f"The regexes dictionary should match the test rules (expected: {expected_regexes}, actual: {actual_regexes})",
)
def test_rule_patterns_without_defaults(self):
rule_patterns = [
{
"reason": "RSA private key 2",
"pattern": "-----BEGIN EC PRIVATE KEY-----",
},
{
"reason": "Complex Rule",
"pattern": "complex-rule",
"path-pattern": "/tmp/[a-z0-9A-Z]+\\.(py|js|json)",
},
]
expected = {
Rule(
name="RSA private key 2",
pattern=re.compile("-----BEGIN EC PRIVATE KEY-----"),
path_pattern=re.compile(""),
re_match_type=MatchType.Search,
re_match_scope=None,
),
Rule(
name="Complex Rule",
pattern=re.compile("complex-rule"),
path_pattern=re.compile("/tmp/[a-z0-9A-Z]+\\.(py|js|json)"),
re_match_type=MatchType.Search,
re_match_scope=None,
),
}
actual = config.configure_regexes(rule_patterns=rule_patterns,
include_default=False)
self.assertEqual(actual, expected)
def test_configure_regexes_includes_rules_from_rules_repo(self):
rules_path = pathlib.Path(__file__).parent / "data"
actual_regexes = config.configure_regexes(
include_default=False,
rules_repo=str(rules_path),
rules_repo_files=["testRules.json"],
)
expected_regexes = {
Rule(
name="RSA private key 2",
pattern=re.compile("-----BEGIN EC PRIVATE KEY-----"),
path_pattern=None,
re_match_type=MatchType.Match,
re_match_scope=None,
),
Rule(
name="Complex Rule",
pattern=re.compile("complex-rule"),
path_pattern=re.compile("/tmp/[a-z0-9A-Z]+\\.(py|js|json)"),
re_match_type=MatchType.Match,
re_match_scope=None,
),
}
self.assertEqual(
expected_regexes,
actual_regexes,
f"The regexes dictionary should match the test rules (expected: {expected_regexes}, actual: {actual_regexes})",
)
def test_configure_regexes_rules_files_with_defaults(self):
rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json"
rules_files = (rules_path.open(),)
expected_regexes = copy.copy(config.DEFAULT_REGEXES)
expected_regexes["RSA private key 2"] = Rule(
name="RSA private key 2",
pattern=re.compile("-----BEGIN EC PRIVATE KEY-----"),
path_pattern=None,
)
expected_regexes["Complex Rule"] = Rule(
name="Complex Rule",
pattern=re.compile("complex-rule"),
path_pattern=re.compile("/tmp/[a-z0-9A-Z]+\\.(py|js|json)"),
)
actual_regexes = config.configure_regexes(
include_default=True, rules_files=rules_files
)
self.assertEqual(
expected_regexes,
actual_regexes,
"The regexes dictionary should match the test rules "
"(expected: {}, actual: {})".format(expected_regexes, actual_regexes),
)
def test_configure_regexes_returns_just_default_regexes_by_default(self):
actual_regexes = config.configure_regexes()
self.assertEqual(
config.DEFAULT_REGEXES,
actual_regexes,
"The regexes dictionary should not have been changed when no rules files are specified",
)
def test_configure_regexes_returns_just_default_regexes_by_default(self):
actual_regexes = config.configure_regexes()
with config.DEFAULT_PATTERN_FILE.open() as handle:
expected_regexes = config.load_rules_from_file(handle)
self.assertEqual(
expected_regexes,
actual_regexes,
"The regexes dictionary should not have been changed when no rules files are specified",
)
def rules_regexes(self) -> Dict[str, Rule]:
"""Get a dictionary of regular expressions to scan the code for.
:raises types.TartufoConfigException: If there was a problem compiling the rules
:rtype: Dict[str, Pattern]
"""
if self._rules_regexes is None:
try:
self._rules_regexes = config.configure_regexes(
self.global_options.default_regexes,
self.global_options.rules,
self.global_options.git_rules_repo,
self.global_options.git_rules_files,
)
except (ValueError, re.error) as exc:
raise types.ConfigException(str(exc)) from exc
return self._rules_regexes
def test_configure_regexes_rules_files_without_defaults(self):
rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json"
rules_files = (rules_path.open(),)
expected_regexes = {
"RSA private key 2": re.compile("-----BEGIN EC PRIVATE KEY-----")
}
actual_regexes = config.configure_regexes(
include_default=False, rules_files=rules_files
)
self.assertEqual(
expected_regexes,
actual_regexes,
"The regexes dictionary should match the test rules "
"(expected: {}, actual: {})".format(expected_regexes, actual_regexes),
)
def rules_regexes(self) -> Set[Rule]:
"""Get a set of regular expressions to scan the code for.
:raises types.ConfigException: If there was a problem compiling the rules
"""
if self._rules_regexes is None:
self.logger.info("Initializing regex rules")
try:
self._rules_regexes = config.configure_regexes(
include_default=self.global_options.default_regexes,
rules_files=self.global_options.rules,
rule_patterns=self.global_options.rule_patterns,
rules_repo=self.global_options.git_rules_repo,
rules_repo_files=self.global_options.git_rules_files,
)
except (ValueError, re.error) as exc:
self.logger.exception("Error loading regex rules",
exc_info=exc)
raise types.ConfigException(str(exc)) from exc
self.logger.debug("Regex rules were initialized as: %s",
self._rules_regexes)
return self._rules_regexes
def test_config_exception_is_raised_if_pattern_is_missing(self):
with self.assertRaisesRegex(ConfigException, "Invalid rule-pattern"):
config.configure_regexes(rule_patterns=[{"reason": "foo"}])
def test_loading_rules_from_file_raises_deprecation_warning(self):
rules_path = pathlib.Path(__file__).parent / "data" / "testRules.json"
rules_files = (rules_path.open(), )
with self.assertWarnsRegex(DeprecationWarning,
"has been deprecated and will be removed."):
config.configure_regexes(rules_files=rules_files)
def main(ctx: click.Context, **kwargs: config.OptionTypes) -> None:
"""Find secrets hidden in the depths of git.
Tartufo will, by default, scan the entire history of a git repository
for any text which looks like a secret, password, credential, etc. It can
also be made to work in pre-commit mode, for scanning blobs of text as a
pre-commit hook.
"""
if not any((kwargs["entropy"], kwargs["regex"])):
err("No analysis requested.")
ctx.exit(1)
if not any((kwargs["pre_commit"], kwargs["repo_path"], kwargs["git_url"])):
err("You must specify one of --pre-commit, --repo-path, or git_url.")
ctx.exit(1)
if kwargs["regex"]:
try:
rules_regexes = config.configure_regexes(
cast(bool, kwargs["default_regexes"]),
cast(Tuple[TextIO, ...], kwargs["rules"]),
cast(Optional[str], kwargs["git_rules_repo"]),
cast(Tuple[str, ...], kwargs["git_rules_files"]),
)
except ValueError as exc:
err(str(exc))
ctx.exit(1)
if not rules_regexes:
err("Regex checks requested, but no regexes found.")
ctx.exit(1)
else:
rules_regexes = {}
# read & compile path inclusion/exclusion patterns
path_inclusions = [] # type: List[Pattern]
path_exclusions = [] # type: List[Pattern]
paths_file = cast(TextIO, kwargs["include_paths"])
if paths_file:
path_inclusions = config.compile_path_rules(paths_file.readlines())
paths_file = cast(TextIO, kwargs["exclude_paths"])
if paths_file:
path_exclusions = config.compile_path_rules(paths_file.readlines())
if kwargs["pre_commit"]:
output = scanner.find_staged(
cast(str, kwargs["repo_path"]),
cast(bool, kwargs["json"]),
cast(bool, kwargs["regex"]),
cast(bool, kwargs["entropy"]),
custom_regexes=rules_regexes,
suppress_output=False,
path_inclusions=path_inclusions,
path_exclusions=path_exclusions,
)
else:
remove_repo = False
if kwargs["git_url"]:
repo_path = util.clone_git_repo(cast(str, kwargs["git_url"]))
remove_repo = True
else:
repo_path = cast(str, kwargs["repo_path"])
output = scanner.scan_repo(repo_path, rules_regexes, path_inclusions,
path_exclusions, kwargs)
if remove_repo:
shutil.rmtree(repo_path, onerror=util.del_rw)
if kwargs["cleanup"]:
util.clean_outputs(output)
else:
issues_path = output.get("issues_path", None)
if issues_path:
print("Results have been saved in {}".format(issues_path))
if output.get("found_issues", False):
ctx.exit(1)
ctx.exit(0)
