def test_parse_rule(rulefile, expectation):
rules = WhisperRules()
rulefile = Path(rule_path(rulefile))
rule_id, rule = load_yaml_from_file(rulefile).popitem()
with expectation:
parsed_rule = rules.parse_rule(rule_id, rule)
assert parsed_rule["message"] == rule["message"]
def test_check(ruleslist, expectation):
filepath = FIXTURE_PATH.joinpath("ruleslist.yml")
rules = WhisperRules(ruleslist=ruleslist)
result = 0
for key, value, _ in Yml().pairs(filepath):
if rules.check(key, value, filepath):
result += 1
assert result == expectation
def test_load_rule(dups, expectation):
rules = WhisperRules()
rulefile = Path(rule_path("valid.yml"))
rule_id, rule = load_yaml_from_file(rulefile).popitem()
with expectation:
for _ in range(dups):
rules.load_rule(rule_id, rule)
assert rule_id in rules.rules
assert rules.rules[rule_id] == rule
def test_load_rules(rulefile, expectation):
rules = WhisperRules(rule_path("empty.yml"))
rulefile = rule_path(rulefile)
ruleyaml = load_yaml_from_file(Path(rulefile))
with expectation:
rules.load_rules(rulefile)
assert len(rules.rules) == len(ruleyaml)
for rule_id in ruleyaml.keys():
assert rule_id in rules.rules
def cli_info():
rule_ids = list(WhisperRules().rules.keys())
rule_ids.sort()
cli_parser().print_help()
print("\navailable rules:")
for rule_id in rule_ids:
print(f" {rule_id}")
def test_cli_info():
mock_print = StringIO()
with patch("sys.stdout", mock_print):
cli_info()
result = mock_print.getvalue()
assert "available rules" in result
for rule_id in WhisperRules().rules.keys():
assert rule_id in result
class Xml:
def __init__(self):
self.breadcrumbs = []
self.rules = WhisperRules()
def pairs(self, filepath: Path):
def _traverse(tree):
"""Traverse XML document"""
for event, element in tree:
if event == "start":
self.breadcrumbs.append(element.tag)
elif event == "end":
self.breadcrumbs.pop()
continue
# Format: <elem key="value">
for key, value in element.attrib.items():
yield key, value, self.breadcrumbs
# Format: <elem name="jdbc:mysql://host?k1=v1&k2=v2">
if self.rules.match("uri", value):
for k, v in Uri().pairs(value):
yield k, v, self.breadcrumbs
# Format: <key>value</key>
if not element.text:
continue
yield element.tag, element.text, self.breadcrumbs
# Format: <elem>key=value</elem>
if "=" in element.text:
item = element.text.split("=")
if len(item) == 2:
yield item[0], item[1], self.breadcrumbs
# Format: <key>name</key><value>string</value>
found_key = None
found_value = None
for item in element:
if str(item.tag).lower() == "key":
found_key = item.text
elif str(item.tag).lower() == "value":
found_value = item.text
if found_key and found_value:
yield found_key, found_value, self.breadcrumbs
try:
parser = ElementTree.XMLParser(recover=True)
tree = ElementTree.parse(filepath.as_posix(), parser)
tree = ElementTree.iterwalk(tree, events=("start", "end"))
yield from _traverse(tree)
except Exception as e:
debug(f"{type(e)} in {filepath}")
class Plaintext:
def __init__(self):
self.rules = WhisperRules()
def pairs(self, filepath: Path):
lines = filepath.open("r").readlines()
for idx in range(len(lines)):
line = lines[idx]
if not strip_string(line):
continue
for value in line.split():
if self.rules.match("uri", value):
yield from Uri().pairs(value)
class StructuredDocument:
def __init__(self):
self.breadcrumbs = []
self.rules = WhisperRules()
def traverse(self, code, key=None):
"""Recursively traverse YAML/JSON document"""
if isinstance(code, dict):
yield from self.cloudformation(code)
for k, v in code.items():
self.breadcrumbs.append(k)
yield k, v, self.breadcrumbs
yield from self.traverse(v, key=k)
self.breadcrumbs.pop()
# Special key/value format
elements = list(code.keys())
if "key" in elements and "value" in elements:
yield code["key"], code["value"], self.breadcrumbs
elif isinstance(code, list):
for item in code:
yield key, item, self.breadcrumbs
yield from self.traverse(item, key=key)
elif isinstance(code, str):
if "=" in code:
item = code.split("=", 1)
if len(item) == 2:
yield item[0], item[1], self.breadcrumbs
if self.rules.match("uri", code):
for k, v in Uri().pairs(code):
yield k, v, self.breadcrumbs
def cloudformation(self, code):
"""
AWS CloudFormation format
"""
if self.breadcrumbs:
return # Not tree root
if "AWSTemplateFormatVersion" not in code:
return # Not CF format
if "Parameters" not in code:
return # No parameters
for key, values in code["Parameters"].items():
if "Default" not in values:
continue # No default value
yield key, values["Default"]
def test_load_rules_from_dict(rulefile, rules_added):
rules = WhisperRules()
rules_len = len(rules.rules)
custom_rules = load_yaml_from_file(Path(rule_path(rulefile)))
rules.load_rules_from_dict(custom_rules)
assert len(rules.rules) == rules_len + rules_added
def test_load_rules_from_file(rulefile, expectation):
rules = WhisperRules()
rules_len = len(rules.rules)
with expectation:
rules.load_rules_from_file(Path(rule_path(rulefile)))
assert len(rules.rules) == rules_len + 1
def __init__(self, config):
self.exclude = config["exclude"]
self.breadcrumbs = []
self.rules = WhisperRules()
self.rules.load_rules_from_dict(config["rules"])
def test_check_similar(rule, key, value, expectation):
rules = WhisperRules()
result = rules.check_similar(rule, key, value)
assert result == expectation
def test_check_regex(rule, value, expectation):
rules = WhisperRules()
result = rules.check_regex(rule, "value", value)
assert result == expectation
def __init__(self):
self.breadcrumbs = []
self.rules = WhisperRules()
def test_match(value, result):
rules = WhisperRules(rule_path("valid.yml"))
assert rules.match("valid", value) == result
def test_decode_if_base64(test, value, expectation):
rules = WhisperRules()
rule = {"isBase64": test}
result = rules.decode_if_base64(rule, value)
assert result == expectation
def __init__(self, args):
self.exclude = args.config["exclude"]
self.breadcrumbs = []
self.rules = WhisperRules(ruleslist=args.rules)
self.rules.load_rules_from_dict(args.config["rules"])
def test_is_ascii(value, expectation):
rules = WhisperRules()
result = rules.is_ascii(value)
assert result == expectation
class WhisperSecrets:
def __init__(self, config):
self.exclude = config["exclude"]
self.breadcrumbs = []
self.rules = WhisperRules()
self.rules.load_rules_from_dict(config["rules"])
def is_static(self, key: str, value: str) -> bool:
"""
Check if given value is static (hardcoded).
If value is dynamic, it's not a hardcoded secret.
"""
if not isinstance(value, str):
return False # Not string
if not value:
return False # Empty
if value.startswith("$") and "$" not in value[2:]:
return False # Variable
if "{{" in value and "}}" in value:
return False # Variable
if value.startswith("{") and value.endswith("}"):
if len(value) > 50:
if self.rules.match("base64", value[1:-1]):
return True # Token
return False # Variable
if value.startswith("${") and value.endswith("}"):
return False # Variable
if value.startswith("<") and value.endswith(">"):
return False # Placeholder
if value == "null":
return False # IaC
if re.match(r"\![A-Za-z]+ .+", value):
return False # IaC !Ref !Sub ...
if key:
s_key = simple_string(key)
s_value = simple_string(value)
if s_key == s_value:
return False # Placeholder
if s_value.endswith(s_key):
return False # Placeholder
for ex in self.exclude["keys"]:
if ex.match(key):
return False # Exclude keys
for ex in self.exclude["values"]:
if ex.match(value):
return False # Exclude values
return True # Hardcoded static value
def is_excluded(self, breadcrumbs: list) -> bool:
for crumb in breadcrumbs:
for ex in self.exclude["keys"]:
if ex.match(str(crumb)):
return True
return False
def detect_secrets(self,
key: str,
value: str,
filepath: Path,
breadcrumbs: list = []) -> Optional[Secret]:
if not key:
key = ""
else:
key = strip_string(key)
if isinstance(value, str):
value = strip_string(value)
elif isinstance(value, int):
value = str(value)
else:
return None # Neither text nor digits
if not self.is_static(key, value):
return None # Not static
if self.is_excluded(breadcrumbs):
return None # Excluded via config
return self.rules.check(key, value, filepath)
def scan(self, filename: str) -> Optional[Secret]:
plugin = WhisperPlugins(filename)
if not plugin:
return
yield self.detect_secrets("file", plugin.filepath.as_posix(),
plugin.filepath)
for ret in plugin.pairs():
if len(ret) == 2: # key, value
yield self.detect_secrets(ret[0], ret[1], plugin.filepath)
elif len(ret) == 3: # key, value, breadcrumbs
yield self.detect_secrets(ret[0],
ret[1],
plugin.filepath,
breadcrumbs=ret[2])
def test_match(value, expectation):
rules = WhisperRules(rulespath=rule_path("valid.yml"))
assert rules.match("valid", value) == expectation
def __init__(self):
self.rules = WhisperRules()
def __init__(self, args):
self.exclude = args.config["exclude"]
self.breadcrumbs = [] # Tracks key path
self.foundlines = {} # Avoids dup line reports
self.rules = WhisperRules(ruleslist=args.rules)
self.rules.load_rules_from_dict(args.config["rules"])
