def test_basic():
file = mock_file_object(
textwrap.dedent("""
keyA: string
keyB: string # with comments
keyC: !!binary YWJjZGVm
keyD: !!binary YWJjZGVm # with comments
keyE: !!binary invalidBinar
dict:
keyD: nested string
num: 1 # don't care
""")[1:-1],
)
assert YAMLTransformer().parse_file(file) == [
'keyA: "string"',
'keyB: "string" # with comments',
'',
'keyC: "abcdef"',
'keyD: "abcdef" # with comments',
'',
'',
'',
'keyD: "nested string"',
]
def test_ignored_lines(self, content_to_format):
file_content = content_to_format.format(secret=self.secret)
f = mock_file_object(file_content)
results = self.logic.analyze(f, 'does_not_matter')
assert len(results) == 0
def test_ignored_lines(self, content_to_format):
file_content = content_to_format.format(secret=self.secret)
f = mock_file_object(file_content)
results = self.logic.analyze(f, 'does_not_matter')
assert len(results) == 0
def test_analyze(self, file_content):
logic = PrivateKeyDetector()
f = mock_file_object(file_content)
output = logic.analyze(f, 'mock_filename')
assert len(output) == 1
for potential_secret in output:
assert 'mock_filename' == potential_secret.filename
def test_analyze(self, file_content):
logic = PrivateKeyDetector()
f = mock_file_object(file_content)
output = logic.analyze(f, 'mock_filename')
assert len(output) == 1
for potential_secret in output:
assert 'mock_filename' == potential_secret.filename
def test_analyze(self, file_content):
logic = SlackDetector(should_verify=False)
f = mock_file_object(file_content)
output = logic.analyze(f, 'mock_filename')
assert len(output) == 1
for potential_secret in output:
assert 'mock_filename' == potential_secret.filename
def test_analyze(self, file_content, should_flag):
logic = AWSKeyDetector()
f = mock_file_object(file_content)
output = logic.analyze(f, 'mock_filename')
assert len(output) == (1 if should_flag else 0)
for potential_secret in output:
assert 'mock_filename' == potential_secret.filename
def test_multiline_block_scalar_literal_style(block_chomping):
file = mock_file_object(
textwrap.dedent("""
multiline: >
this will be skipped
""")[1:-1], )
assert YAMLTransformer().parse_file(file) == ['']
def test_analyze_should_abide_by_no_verify_flag(self):
with self.create_test_plugin(VerifiedResult.VERIFIED_FALSE) as plugin:
plugin.should_verify = False
file = mock_file_object('does not matter')
result = plugin.analyze(file, 'does not matter')
# If it is verified, this value should be 0.
assert len(result) == 1
def test_analyze_yaml_negatives(self, file_content, file_extension):
logic = KeywordDetector()
f = mock_file_object(file_content)
output = logic.analyze(
f,
'mock_filename{}'.format(file_extension),
)
assert len(output) == 0
def test_analyze(self, file_content):
logic = KeywordDetector()
f = mock_file_object(file_content)
output = logic.analyze(f, 'mock_filename')
assert len(output) == 1
for potential_secret in output:
assert 'mock_filename' == potential_secret.filename
generated = list(logic.secret_generator(file_content))
assert len(generated) == len(output)
def test_possible_secret_format(yaml_value, expected_value):
content = 'key: {yaml_value}'.format(yaml_value=yaml_value)
f = mock_file_object(content)
result = YAMLFileParser(f).json()
assert result['key'] == {
'__value__': expected_value,
'__line__': mock.ANY,
'__original_key__': mock.ANY,
}
def test_analyze_verified_false_with_output_verified_false_flag(self):
with self.create_test_plugin(VerifiedResult.VERIFIED_FALSE) as plugin:
file = mock_file_object('does not matter')
result = plugin.analyze(
file,
'does not matter',
output_verified_false=True,
)
assert len(result) == 1
def test_analyze_multiple_strings_same_line(self, content_to_format, expected_results):
content = content_to_format.format(
non_secret=self.non_secret,
secret=self.secret,
)
f = mock_file_object(content)
results = self.logic.analyze(f, 'does_not_matter')
assert len(results) == expected_results
def test_analyze_yaml_negatives(self, file_content, file_extension):
logic = KeywordDetector()
# Make it start with `{{`, (and end with `}}`) so it hits our false-positive check
f = mock_file_object(file_content.replace('m{', '{'))
output = logic.analyze(
f,
'mock_filename{}'.format(file_extension),
)
assert len(output) == 0
def test_analyze_objective_c_positives(self, file_content):
logic = KeywordDetector()
f = mock_file_object(file_content)
output = logic.analyze(f, 'mock_filename.m')
assert len(output) == 1
for potential_secret in output:
assert 'mock_filename.m' == potential_secret.filename
assert (potential_secret.secret_hash == PotentialSecret.
hash_secret('m{{h}o)p${e]nob(ody[finds>-_$#thisone}}'))
def test_analyze_multiple_strings_same_line(self, content_to_format, expected_results):
content = content_to_format.format(
non_secret=self.non_secret,
secret=self.secret,
)
f = mock_file_object(content)
results = self.logic.analyze(f, 'does_not_matter')
assert len(results) == expected_results
def test_get_ignored_lines(self):
content = """keyA: value
keyB: \"another_value\" # pragma: whitelist secret
keyC: yet_another_value
"""
f = mock_file_object(content)
ignored_lines = YamlFileParser(f).get_ignored_lines()
assert ignored_lines == {2}
def test_pattern(self, content_to_format, should_be_caught):
content = content_to_format.format(
non_secret=self.non_secret,
secret=self.secret,
)
f = mock_file_object(content)
results = self.logic.analyze(f, 'does_not_matter')
assert len(results) == bool(should_be_caught)
def test_pattern(self, content_to_format, should_be_caught):
content = content_to_format.format(
non_secret=self.non_secret,
secret=self.secret,
)
f = mock_file_object(content)
results = self.logic.analyze(f, 'does_not_matter')
assert len(results) == bool(should_be_caught)
def test_analyze_example_negatives(self, file_content):
logic = KeywordDetector()
# Make it start with `<`, (and end with `>`) so it hits our false-positive check
f = mock_file_object(
file_content.replace('m{', '<').replace('}', '>'), )
output = logic.analyze(
f,
'mock_filename.example',
)
assert len(output) == 0
def test_get_ignored_lines(self):
content = """keyA: value
keyB: \"another_value\" # pragma: allowlist secret
keyC: \"another_value\" # pragma: whitelist secret (backwards compatibility test)
keyD: yet_another_value
"""
f = mock_file_object(content)
ignored_lines = YamlFileParser(f).get_ignored_lines()
assert ignored_lines == {2, 3}
def test_analyze_quotes_required_positives(self, file_content,
file_extension):
logic = KeywordDetector()
f = mock_file_object(file_content)
mock_filename = 'mock_filename{}'.format(file_extension)
output = logic.analyze(f, mock_filename)
assert len(output) == 1
for potential_secret in output:
assert mock_filename == potential_secret.filename
assert (potential_secret.secret_hash == PotentialSecret.
hash_secret('m{{h}o)p${e]nob(ody[finds>-_$#thisone}}'))
def test_add_header():
file = mock_file_object(
textwrap.dedent("""
key = value
# comment
tea = chai
""")[1:-1], )
assert list(IniFileParser(file, add_header=True)) == [
('key', 'value', 1),
('tea', 'chai', 4),
]
def test_multi_line(block_scalar_style, block_chomping):
# NOTE: Referenced https://yaml-multiline.info/ for the many ways to do multi line strings
file = mock_file_object(
textwrap.dedent(f"""
key: {block_scalar_style}{block_chomping} # comment
multi
#line
string
""")[1:-1], )
assert [item.line for item in YAMLFileParser(file)] == [
f'key: {block_scalar_style}{block_chomping} # comment',
]
def test_analyze_standard_positives_with_automaton(self, file_content):
automaton = ahocorasick.Automaton()
word = 'thisone'
automaton.add_word(word, word)
automaton.make_automaton()
logic = KeywordDetector(automaton=automaton)
f = mock_file_object(file_content)
output = logic.analyze(f, 'mock_filename')
# All skipped due to automaton
assert len(output) == 0
def test_not_first():
file = mock_file_object(
textwrap.dedent("""
[section]
key =
value1
# comment
value2
""")[1:-1], )
assert list(IniFileParser(file)) == [
('key', 'value1', 3),
('key', 'value2', 6),
]
def test_possible_secret_format(
self,
yaml_value,
expected_value,
expected_is_binary,
):
content = 'key: {yaml_value}'.format(yaml_value=yaml_value)
f = mock_file_object(content)
result = YamlFileParser(f).json()
assert result['key'] == {
'__value__': expected_value,
'__is_binary__': expected_is_binary,
'__line__': mock.ANY,
}
def test_multiline_block_scalar_folded_style(block_chomping):
# NOTE(2020-11-07|domanchi): For YAML parsing, we don't really care about "literal" style
# (the one with `|`) since that will keep new lines, and our assumption is that secrets
# won't have new lines.
#
# However, "folded" style may be used to keep a certain line limit with very long secrets,
# so we should probably handle that.
file = mock_file_object(
textwrap.dedent(f"""
multiline: |{block_chomping} # example
this is
a basic multiline string
""")[1:-1], )
assert YAMLTransformer().parse_file(file) == [
'multiline: this is a basic multiline string # example',
]
def test_analyze_standard_positives_with_automaton(self, file_content):
automaton = ahocorasick.Automaton()
word = 'thisone'
if is_python_2(): # pragma: no cover
# Due to pyahocorasick
word = word.encode('utf-8')
automaton.add_word(word, word)
automaton.make_automaton()
logic = KeywordDetector(automaton=automaton)
f = mock_file_object(file_content)
output = logic.analyze(f, 'mock_filename')
# All skipped due to automaton
assert len(output) == 0
def test_basic():
file = mock_file_object(
textwrap.dedent("""
keyA: string
dict:
keyB: 123
""")[1:-1], )
assert YAMLFileParser(file).json() == {
'keyA': {
'__value__': 'string',
'__line__': 1,
'__original_key__': 'keyA',
},
# Ignores non-string or non-binary
'dict': {
'keyB': 123,
},
}
def test_basic():
file = mock_file_object(
textwrap.dedent("""
[section]
key = value
rice = fried
# comment
tea = chai
[other]
water = unflavored
""")[1:-1], )
assert list(IniFileParser(file)) == [
('key', 'value', 2),
('rice', 'fried', 3),
('tea', 'chai', 6),
('water', 'unflavored', 10),
]
def test_transformer(transformer):
file = mock_file_object(
textwrap.dedent("""
[section]
keyA = value
keyB = "double"
keyC = 'single'
keyD = o'brian
keyE = "chai" tea
""")[1:-1], )
assert transformer().parse_file(file) == [
'',
'keyA = "value"',
'',
'keyB = "double"',
'keyC = "single"',
'',
'keyD = "o\'brian"',
'keyE = "\\\"chai\\\" tea"',
]
def test_analyze_with_line_exclude(self, file_content):
logic = KeywordDetector(keyword_exclude='thisone')
f = mock_file_object(file_content)
output = logic.analyze(f, 'mock_filename.foo')
assert len(output) == 0
