def test_load_baseline_without_any_valid_fields(self, mock_log):
with pytest.raises(IOError):
SecretsCollection.load_baseline_from_string(
json.dumps({
'junk': 'dictionary',
}), )
assert mock_log.error_messages == 'Incorrectly formatted baseline!\n'
def test_load_baseline_without_exclude(self, mock_log):
with pytest.raises(IOError):
SecretsCollection.load_baseline_from_string(
json.dumps({
'plugins_used': (),
'results': {},
}), )
assert mock_log.error_messages == 'Incorrectly formatted baseline!\n'
def test_load_baseline_with_invalid_input(self, mock_log):
with pytest.raises(IOError):
SecretsCollection.load_baseline_from_string(
json.dumps({
'junk': 'dictionary',
}), )
assert mock_log.getLogger().error.called
def test_load_baseline_with_invalid_input(self, mock_log):
with pytest.raises(IOError):
SecretsCollection.load_baseline_from_string(
json.dumps({
'junk': 'dictionary',
}),
)
assert mock_log.error_messages == 'Incorrectly formatted baseline!\n'
def test_load_baseline_from_string_with_point_twelve_point_seven_and_later_string(
self,
mock_gmtime,
):
"""
We use load_baseline_from_string as a proxy to testing load_baseline_from_dict,
because it's the most entry into the private function.
"""
original = self.get_point_twelve_point_seven_and_later_baseline_dict(
mock_gmtime)
word_list = """
roller\n
"""
with mock_open_base(
data=word_list,
namespace='detect_secrets.util.open',
):
secrets = SecretsCollection.load_baseline_from_string(
json.dumps(original), ).format_for_baseline_output()
# v0.12.7+ assertions
assert original['word_list']['file'] == secrets['word_list']['file']
# Original hash is thrown out and replaced with new word list hash
assert (secrets['word_list']['hash'] == hashlib.sha1(
'roller'.encode('utf-8')).hexdigest() !=
original['word_list']['hash'])
# Regular assertions
assert original['exclude']['files'] == secrets['exclude']['files']
assert secrets['exclude']['lines'] is None
assert original['results'] == secrets['results']
def get_baseline(baseline_filename):
"""
:raises: IOError
:raises: ValueError
"""
if not baseline_filename:
return
raise_exception_if_baseline_file_is_not_up_to_date(baseline_filename)
baseline_string = _get_baseline_string_from_file(baseline_filename)
baseline_version = json.loads(baseline_string).get('version')
try:
raise_exception_if_baseline_version_is_outdated(baseline_version, )
except ValueError:
log.error(
'The supplied baseline may be incompatible with the current\n'
'version of detect-secrets. Please recreate your baseline to\n'
'avoid potential mis-configurations.\n\n'
'Current Version: %s\n'
'Baseline Version: %s',
VERSION,
baseline_version if baseline_version else '0.0.0',
)
raise
return SecretsCollection.load_baseline_from_string(baseline_string)
def get_baseline(baseline_filename):
"""
:raises: IOError
:raises: ValueError
"""
if not baseline_filename:
return
raise_exception_if_baseline_file_is_not_up_to_date(baseline_filename)
baseline_string = _get_baseline_string_from_file(baseline_filename)
baseline_version = json.loads(baseline_string).get('version')
try:
raise_exception_if_baseline_version_is_outdated(
baseline_version,
)
except ValueError:
log.error(
'The supplied baseline may be incompatible with the current\n'
'version of detect-secrets. Please recreate your baseline to\n'
'avoid potential mis-configurations.\n\n'
'Current Version: %s\n'
'Baseline Version: %s',
VERSION,
baseline_version if baseline_version else '0.0.0',
)
raise
return SecretsCollection.load_baseline_from_string(baseline_string)
def test_load_baseline_from_string(self, mock_gmtime):
"""
We use load_baseline_from_string as a proxy to testing _load_baseline_from_dict,
because it's the most entry into the private function.
"""
original = self.get_baseline_dict(mock_gmtime)
secrets = SecretsCollection.load_baseline_from_string(
json.dumps(original), ).format_for_baseline_output()
self.assert_loaded_collection_is_original_collection(original, secrets)
def get_baseline(baseline_filename):
"""
:raises: IOError
:raises: ValueError
"""
if not baseline_filename:
return
raise_exception_if_baseline_file_is_unstaged(baseline_filename)
return SecretsCollection.load_baseline_from_string(
_get_baseline_string_from_file(baseline_filename, ), )
def scan(self, exclude_files_regex=None, exclude_lines_regex=None, scan_head=False):
"""Fetches latest changes, and scans the git diff between last_commit_hash
and HEAD.
:raises: subprocess.CalledProcessError
:type exclude_files_regex: str|None
:param exclude_files_regex: A regex matching filenames to skip over.
:type exclude_lines: str|None
:param exclude_lines: A regex matching lines to skip over.
:rtype: SecretsCollection
:returns: secrets found.
"""
self.storage.fetch_new_changes()
default_plugins = initialize_plugins.from_parser_builder(
self.plugin_config,
exclude_lines_regex=exclude_lines_regex,
)
# TODO Issue 17: Ignoring self.exclude_regex, using the server scan CLI arg
secrets = SecretsCollection(
plugins=default_plugins,
exclude_files=exclude_files_regex,
exclude_lines=exclude_lines_regex,
)
scan_from_this_commit = git.get_empty_tree_commit_hash() if scan_head else self.last_commit_hash
try:
diff_name_only = self.storage.get_diff_name_only(scan_from_this_commit)
# do a per-file diff + scan so we don't get a OOM if the the commit-diff is too large
for filename in diff_name_only:
file_diff = self.storage.get_diff(scan_from_this_commit, filename)
secrets.scan_diff(
file_diff,
baseline_filename=self.baseline_filename,
last_commit_hash=scan_from_this_commit,
repo_name=self.name,
)
except subprocess.CalledProcessError:
self.update()
return secrets
if self.baseline_filename:
baseline = self.storage.get_baseline_file(self.baseline_filename)
if baseline:
baseline_collection = SecretsCollection.load_baseline_from_string(baseline)
secrets = get_secrets_not_in_baseline(secrets, baseline_collection)
return secrets
def test_load_baseline_from_string(self, mock_gmtime):
"""
We use load_baseline_from_string as a proxy to testing _load_baseline_from_dict,
because it's the most entry into the private function.
"""
original = self.get_baseline_dict(mock_gmtime)
secrets = SecretsCollection.load_baseline_from_string(
json.dumps(original),
).format_for_baseline_output()
self.assert_loaded_collection_is_original_collection(original, secrets)
def get_baseline(baseline_filename):
"""
:raises: IOError
:raises: ValueError
"""
if not baseline_filename:
return
return SecretsCollection.load_baseline_from_string(
_get_baseline_string_from_file(
baseline_filename,
),
)
def test_load_baseline_from_string_with_point_twelve_to_twelve_six_string(
self, mock_gmtime):
"""
We use load_baseline_from_string as a proxy to testing load_baseline_from_dict,
because it's the most entry into the private function.
"""
original = self.get_point_twelve_and_later_baseline_dict(mock_gmtime)
secrets = SecretsCollection.load_baseline_from_string(
json.dumps(original), ).format_for_baseline_output()
assert original['exclude']['files'] == secrets['exclude']['files']
assert secrets['exclude']['lines'] is None
assert original['results'] == secrets['results']
def scan(self, exclude_files_regex=None, exclude_lines_regex=None):
"""Fetches latest changes, and scans the git diff between last_commit_hash
and HEAD.
:raises: subprocess.CalledProcessError
:type exclude_files_regex: str|None
:param exclude_files_regex: A regex matching filenames to skip over.
:type exclude_lines: str|None
:param exclude_lines: A regex matching lines to skip over.
:rtype: SecretsCollection
:returns: secrets found.
"""
self.storage.fetch_new_changes()
default_plugins = initialize_plugins.from_parser_builder(
self.plugin_config,
exclude_lines_regex=exclude_lines_regex,
)
# TODO Issue 17: Ignoring self.exclude_regex, using the server scan CLI arg
secrets = SecretsCollection(
plugins=default_plugins,
exclude_files=exclude_files_regex,
exclude_lines=exclude_lines_regex,
)
try:
diff = self.storage.get_diff(self.last_commit_hash)
except subprocess.CalledProcessError:
self.update()
return secrets
secrets.scan_diff(
diff,
baseline_filename=self.baseline_filename,
last_commit_hash=self.last_commit_hash,
repo_name=self.name,
)
if self.baseline_filename:
baseline = self.storage.get_baseline_file(self.baseline_filename)
if baseline:
baseline_collection = SecretsCollection.load_baseline_from_string(
baseline)
secrets = get_secrets_not_in_baseline(secrets,
baseline_collection)
return secrets
def get_baseline(baseline_filename, plugin_filenames=None):
"""
:type baseline_filename: string
:param baseline_filename: name of the baseline file
:type plugin_filenames: tuple
:param plugin_filenames: list of plugins to import
:raises: IOError
:raises: ValueError
"""
if not baseline_filename:
return
raise_exception_if_baseline_file_is_unstaged(baseline_filename)
return SecretsCollection.load_baseline_from_string(
_get_baseline_string_from_file(baseline_filename, ),
plugin_filenames=plugin_filenames,
)
def scan(self):
"""Clones the repo, and scans the git diff between last_commit_hash and HEAD.
:raises: subprocess.CalledProcessError
"""
self.clone_and_pull_repo()
diff = self._get_latest_changes()
baseline = self._get_baseline()
default_plugins = initialize(self.plugin_config)
secrets = SecretsCollection(default_plugins, self.exclude_regex)
secrets.scan_diff(diff,
baseline_filename=baseline,
last_commit_hash=self.last_commit_hash,
repo_name=self.name)
if baseline:
baseline_collection = SecretsCollection.load_baseline_from_string(
baseline)
secrets = get_secrets_not_in_baseline(secrets, baseline_collection)
return secrets