def test_cache_catches_last_found_secrets(client):
"""
GIVEN an empty cache and an empty config matches-ignore section
WHEN I run a scan with multiple secrets
THEN cache last_found_secrets is updated with these secrets and saved
"""
c = Commit()
c._patch = _MULTIPLE_SECRETS
config = Config()
setattr(config, "matches_ignore", set())
cache = Cache()
cache.purge()
assert cache.last_found_secrets == set()
with my_vcr.use_cassette("multiple_secrets"):
c.scan(
client=client,
cache=cache,
matches_ignore=config.matches_ignore,
all_policies=True,
verbose=False,
)
assert config.matches_ignore == set()
assert cache.last_found_secrets == FOUND_SECRETS
cache.load_cache()
assert cache.last_found_secrets == FOUND_SECRETS
def test_cache_catches_last_found_secrets(client, isolated_fs):
"""
GIVEN an empty cache and an empty config matches-ignore section
WHEN I run a scan with multiple secrets
THEN cache last_found_secrets is updated with these secrets and saved
"""
c = Commit()
c._patch = _MULTIPLE_SECRETS
config = Config()
setattr(config, "matches_ignore", [])
cache = Cache()
cache.purge()
assert cache.last_found_secrets == list()
with my_vcr.use_cassette("multiple_secrets"):
c.scan(
client=client,
cache=cache,
matches_ignore=config.matches_ignore,
all_policies=True,
verbose=False,
)
assert config.matches_ignore == list()
cache_found_secrets = sorted(cache.last_found_secrets, key=compare_matches_ignore)
found_secrets = sorted(FOUND_SECRETS, key=compare_matches_ignore)
assert [found_secret["match"] for found_secret in cache_found_secrets] == [
found_secret["match"] for found_secret in found_secrets
]
ignore_last_found(config, cache)
for ignore in config.matches_ignore:
assert "test.txt" in ignore["name"]
cache.load_cache()
def test_cache_catches_nothing(client):
"""
GIVEN a cache of last found secrets same as config ignored-matches
WHEN I run a scan (therefore finding no secret)
THEN config matches is unchanged and cache is empty
"""
c = Commit()
c._patch = _MULTIPLE_SECRETS
config = Config()
config.matches_ignore = FOUND_SECRETS
cache = Cache()
cache.last_found_secrets = FOUND_SECRETS
with my_vcr.use_cassette("multiple_secrets"):
results = c.scan(
client=client,
cache=cache,
matches_ignore=config.matches_ignore,
all_policies=True,
verbose=False,
)
assert results == []
assert config.matches_ignore == FOUND_SECRETS
assert cache.last_found_secrets == []
def test_scan_patch(client, cache, name, input_patch, expected):
c = Commit()
c._patch = input_patch
with my_vcr.use_cassette(name):
results = c.scan(
client=client,
cache=cache,
matches_ignore={},
all_policies=True,
verbose=False,
)
for result in results:
if result.scan.policy_breaks:
assert len(
result.scan.policy_breaks[0].matches) == expected.matches
if expected.first_match:
assert (result.scan.policy_breaks[0].matches[0].match ==
expected.first_match)
else:
assert result.scan.policy_breaks == []
if expected.want:
assert result.content == expected.want["content"]
assert result.filename == expected.want["filename"]
assert result.filemode == expected.want["filemode"]
def test_cache_old_config_no_new_secret(client, isolated_fs):
"""
GIVEN a cache of last found secrets same as config ignored-matches
and config ignored-matches is a list of strings
WHEN I run a scan (therefore finding no secret)
THEN config matches is unchanged and cache is empty
"""
c = Commit()
c._patch = _MULTIPLE_SECRETS
config = Config()
config.matches_ignore = [d["match"] for d in FOUND_SECRETS]
cache = Cache()
cache.last_found_secrets = FOUND_SECRETS
with my_vcr.use_cassette("multiple_secrets"):
results = c.scan(
client=client,
cache=cache,
matches_ignore=config.matches_ignore,
all_policies=True,
verbose=False,
)
assert results == []
assert config.matches_ignore == [d["match"] for d in FOUND_SECRETS]
assert cache.last_found_secrets == []
def test_patch_separation_ignore():
c = Commit()
c._patch = PATCH_SEPARATION
file_to_ignore = ".env"
c.filter_set = {os.path.join(os.getcwd(), file_to_ignore)}
files = list(c.get_files())
assert len(files) == 3
assert not (any(entry.filename == file_to_ignore for entry in files))
def test_patch_separation():
c = Commit()
c._patch = PATCH_SEPARATION
files = list(c.get_files())
assert len(files) == 4
assert c.info.author == "Testificate Jose"
assert c.info.email == "*****@*****.**"
assert c.info.date == "Fri Oct 18 13:20:00 2012 +0100"
def test_patch_max_size():
c = Commit()
c._patch = """
diff --git a/tests/test_scannable.py b/.env
new file mode 100644
index 0000000..0000000
--- /dev/null
+++ b/.env
@@ -0,0 +1,112 @@
CHECK_ENVIRONMENT=true
"""
c._patch += "a" * MAX_FILE_SIZE
files = list(c.get_files())
assert len(files) == 0
def precommit_cmd(ctx: click.Context,
precommit_args: List[str]) -> int: # pragma: no cover
"""
scan as a pre-commit git hook.
"""
config = ctx.obj["config"]
output_handler = TextHandler(show_secrets=config.show_secrets,
verbose=config.verbose,
output=None)
try:
check_git_dir()
results = Commit(filter_set=ctx.obj["filter_set"]).scan(
client=ctx.obj["client"],
cache=ctx.obj["cache"],
matches_ignore=config.matches_ignore,
all_policies=config.all_policies,
verbose=config.verbose,
)
return output_handler.process_scan(
ScanCollection(id="cached", type="pre-commit", results=results))[1]
except click.exceptions.Abort:
return 0
except Exception as error:
if config.verbose:
traceback.print_exc()
raise click.ClickException(str(error))
def scan_commit(
commit: Commit,
client: GGClient,
cache: Cache,
verbose: bool,
matches_ignore: Iterable[IgnoredMatch],
all_policies: bool,
mode_header: str,
banlisted_detectors: Optional[Set[str]] = None,
) -> ScanCollection: # pragma: no cover
results = commit.scan(
client=client,
cache=cache,
matches_ignore=matches_ignore,
banlisted_detectors=banlisted_detectors,
all_policies=all_policies,
verbose=verbose,
)
return ScanCollection(
commit.sha or "unknown",
type="commit",
results=results,
optional_header=commit.optional_header,
extra_info=commit.info._asdict(),
)
def test_json_output(client, name, input_patch, expected, snapshot):
c = Commit()
c._patch = input_patch
handler = JSONHandler(verbose=True, show_secrets=False)
with my_vcr.use_cassette(name):
results = c.scan(
client=client, matches_ignore={}, all_policies=True, verbose=False
)
flat_results, exit_code = handler.process_scan(
scan=ScanCollection(id="path", type="test", results=results), top=True
)
assert exit_code == expected
json_flat_results = JSONScanCollectionSchema().dumps(flat_results)
snapshot.assert_match(JSONScanCollectionSchema().loads(json_flat_results))
def test_request_headers(scan_mock: Mock, client):
c = Commit()
c._patch = _SIMPLE_SECRET
with Context(Command("bar"), info_name="bar") as ctx:
ctx.parent = Context(Group("foo"), info_name="foo")
c.scan(
client=client,
cache=Cache(),
matches_ignore={},
all_policies=True,
verbose=False,
)
scan_mock.assert_called_with(
ANY,
{
"GGShield-Version": __version__,
"GGShield-Command-Path": "foo bar",
},
)
def test_json_output(client, cache, name, input_patch, expected, snapshot):
c = Commit()
c._patch = input_patch
handler = JSONOutputHandler(verbose=True, show_secrets=False)
with my_vcr.use_cassette(name):
results = c.scan(
client=client,
cache=cache,
matches_ignore={},
all_policies=True,
verbose=False,
banlisted_detectors=None,
)
scan = ScanCollection(id="path", type="test", results=results)
json_flat_results = handler._process_scan_impl(scan)
exit_code = OutputHandler._get_exit_code(scan)
assert exit_code == expected
snapshot.assert_match(
JSONScanCollectionSchema().loads(json_flat_results))
def scan_commit_range(
client: GGClient,
cache: Cache,
commit_list: List[str],
output_handler: OutputHandler,
verbose: bool,
exclusion_regexes: Set[re.Pattern],
matches_ignore: Iterable[IgnoredMatch],
all_policies: bool,
scan_id: str,
mode_header: str,
banlisted_detectors: Optional[Set[str]] = None,
) -> int: # pragma: no cover
"""
Scan every commit in a range.
:param client: Public Scanning API client
:param commit_list: List of commits sha to scan
:param verbose: Display successfull scan's message
"""
return_code = 0
with concurrent.futures.ThreadPoolExecutor(
max_workers=min(CPU_COUNT, 4)) as executor:
future_to_process = [
executor.submit(
scan_commit,
Commit(sha, exclusion_regexes),
client,
cache,
verbose,
matches_ignore,
all_policies,
mode_header,
banlisted_detectors,
) for sha in commit_list
]
scans: List[ScanCollection] = []
with click.progressbar(
iterable=concurrent.futures.as_completed(future_to_process),
length=len(future_to_process),
label=format_text("Scanning Commits", STYLE["progress"]),
) as completed_futures:
for future in completed_futures:
scans.append(future.result())
return_code = output_handler.process_scan(
ScanCollection(id=scan_id, type="commit-range", scans=scans))
return return_code
def scan_commit_range(
client: GGClient,
cache: Cache,
commit_list: List[str],
output_handler: OutputHandler,
verbose: bool,
filter_set: Set[str],
matches_ignore: Iterable[str],
all_policies: bool,
scan_id: str,
) -> int: # pragma: no cover
"""
Scan every commit in a range.
:param client: Public Scanning API client
:param commit_range: Range of commits to scan (A...B)
:param verbose: Display successfull scan's message
"""
return_code = 0
with concurrent.futures.ThreadPoolExecutor(
max_workers=min(CPU_COUNT, 4)) as executor:
future_to_process = [
executor.submit(
scan_commit,
Commit(sha, filter_set),
client,
cache,
verbose,
matches_ignore,
all_policies,
) for sha in commit_list
]
scans: List[ScanCollection] = []
with click.progressbar(
length=len(future_to_process),
label=format_text("Scanning Commits", STYLE["progress"]),
) as bar:
processed = 0
for future in concurrent.futures.as_completed(future_to_process):
scans.append(future.result())
processed += 1
bar.update(processed)
return_code = output_handler.process_scan(
ScanCollection(id=scan_id, type="commit-range", scans=scans))[1]
return return_code
def scan_commit(
commit: Commit,
client: GGClient,
verbose: bool,
matches_ignore: Iterable[str],
all_policies: bool,
) -> ScanCollection: # pragma: no cover
results = commit.scan(
client=client,
matches_ignore=matches_ignore,
all_policies=all_policies,
verbose=verbose,
)
return ScanCollection(
commit.sha or "unknown",
type="commit",
results=results,
optional_header=commit.optional_header,
extra_info=commit.info._asdict(),
)
def precommit_cmd(ctx: click.Context,
precommit_args: List[str]) -> int: # pragma: no cover
"""
scan as a pre-commit git hook.
"""
config = ctx.obj["config"]
output_handler = TextOutputHandler(show_secrets=config.show_secrets,
verbose=config.verbose,
output=None)
try:
check_git_dir()
results = Commit(exclusion_regexes=ctx.obj["exclusion_regexes"]).scan(
client=ctx.obj["client"],
cache=ctx.obj["cache"],
matches_ignore=config.matches_ignore,
all_policies=config.all_policies,
verbose=config.verbose,
banlisted_detectors=config.banlisted_detectors,
)
return output_handler.process_scan(
ScanCollection(id="cached", type="pre-commit", results=results))
except Exception as error:
return handle_exception(error, config.verbose)
def test_get_filename():
line = "a/test.txt b/test.txt"
assert Commit().get_filename(line) == "test.txt"
def test_get_filemode_delete():
line = "deleted file mode 100644\n"
assert Commit().get_filemode(line) == Filemode.DELETE
def test_get_filemode_modify():
line = "index 3d47bfe..ee93988 100644\n"
assert Commit().get_filemode(line) == Filemode.MODIFY
def test_get_filemode_new():
line = "new file mode 100644\n"
assert Commit().get_filemode(line) == Filemode.NEW
def test_get_filemode_rename():
line = "similarity index 99%\n"
assert Commit().get_filemode(line) == Filemode.RENAME