def test_evaluate_python_bad_return_type() -> None:
results = {
PatternId("all_execs"): [
PatternMatchMock(400, 500,
{"$X": {
"abstract_content": "cmd_pattern"
}}),
PatternMatchMock(800, 900,
{"$X": {
"abstract_content": "other_pattern"
}}),
]
}
expression = [
BooleanRuleExpression(OPERATORS.AND, PatternId("all_execs"), None,
"all_execs"),
BooleanRuleExpression(
OPERATORS.WHERE_PYTHON,
PatternId("p1"),
None,
"str(vars['$X'])",
),
]
with pytest.raises(SemgrepError):
evaluate_expression(expression, results, allow_exec=True)
def test_evaluate_python() -> None:
"""Test evaluating the subpattern `where-python: <python_expression>`,
in which a rule can provide an arbitrary Python expression that will be
evaluated against the currently matched metavariables.
NOTE: the Python expression must evaluate to True or False.
NOTE: Assume patterns are applied in the order specified, top to bottom.
This is implementing: https://github.com/returntocorp/semgrep/issues/101.
let allExecs = exec($X)
let filteredExecs = where-python: "vars['$X'].startswith('cmd')"
000-100 var exec = require('child_process').exec;
100-200 var cmd_pattern = "user_input";
200-300 var other_pattern = "hardcoded_string";
300-400 // should match
400-500 exec(cmd_pattern, function(error, stdout, stderr){
500-600 console.log(stdout);
600-700 });
700-800 // should not match
800-900 exec(other_pattern, function(error, stdout, stderr){
900-1000 console.log(stdout);
1100-1200 });
patterns:
pattern: exec($X)
where-python: "vars['$X'].startswith('cmd')"
OUTPUT: [400-500]
"""
results = {
PatternId("all_execs"): [
PatternMatchMock(400, 500, {"$X": {"abstract_content": "cmd_pattern"}}),
PatternMatchMock(800, 900, {"$X": {"abstract_content": "other_pattern"}}),
]
}
expression = [
BooleanRuleExpression(OPERATORS.AND, PatternId("all_execs"), None, "all_execs"),
BooleanRuleExpression(
OPERATORS.WHERE_PYTHON,
PatternId("p1"),
None,
"vars['$X'].startswith('cmd')",
),
]
result = evaluate_expression(expression, results, flags={RCE_RULE_FLAG: True})
assert result == set([Range(400, 500, {})]), f"{result}"
def _build_boolean_expression(
self, rule: YamlTree[YamlMap]
) -> BooleanRuleExpression:
"""
Build a boolean expression from the yml lines in the rule
"""
rule_raw = rule.value
_rule_id = cast(str, rule_raw["id"].unroll())
rule_id = PatternId(_rule_id)
for pattern_name in pattern_names_for_operator(OPERATORS.AND):
pattern = rule_raw.get(pattern_name)
if pattern:
self._pattern_spans[rule_id] = pattern.span
return BooleanRuleExpression(
OPERATORS.AND, rule_id, None, pattern.value
)
for pattern_name in pattern_names_for_operator(OPERATORS.REGEX):
pattern = rule_raw.get(pattern_name)
if pattern:
self._pattern_spans[rule_id] = pattern.span
return BooleanRuleExpression(
OPERATORS.REGEX, rule_id, None, pattern.value
)
for pattern_name in pattern_names_for_operator(OPERATORS.AND_ALL):
patterns = rule_raw.get(pattern_name)
if patterns:
return BooleanRuleExpression(
operator=OPERATORS.AND_ALL,
pattern_id=None,
children=list(self._parse_boolean_expression(patterns)),
operand=None,
)
for pattern_name in pattern_names_for_operator(OPERATORS.AND_EITHER):
patterns = rule_raw.get(pattern_name)
if patterns:
return BooleanRuleExpression(
operator=OPERATORS.AND_EITHER,
pattern_id=None,
children=list(self._parse_boolean_expression(patterns)),
operand=None,
)
required_operator = [
OPERATORS.AND_ALL,
OPERATORS.AND_EITHER,
OPERATORS.REGEX,
OPERATORS.AND,
]
raise Exception("Internal error: bad schema")
def testB() -> None:
"""
Our algebra needs to express "AND-INSIDE" as opposed to "AND-NOT-INSIDE", so that cases
like this one can still fire (we explicitly don't want to ignore the nested expression
just because it's inside).
let pattern1 = subprocess.Popen($X, safeflag=True)
let pattern2 = subprocess.Popen($X, ...)
let pattern3 = importlib.Popen($X, ...)
import subprocess
subprocess.Popen(subprocess.Popen(bad), safeFlag=True)
----------------------------------------------------- P1, P2 R1
--------------------- P2 R2
and-not P1 --> remove all ranges == P1 exactly (R1 is removed)
and-or (P2, P3) --> remove any ranges not exactly == P2 or P3. (R2 stays)
OUTPUT: R2
"""
results: Dict[PatternId, List[PatternMatch]] = {
PatternId("pattern1"): [PatternMatchMock(0, 100)],
PatternId("pattern2"): [PatternMatchMock(30, 70), PatternMatchMock(0, 100)],
PatternId("pattern3"): [],
}
expression = [
RuleExpr(OPERATORS.AND_NOT, "pattern1"),
BooleanRuleExpression(
OPERATORS.AND_EITHER,
None,
[RuleExpr(OPERATORS.AND, "pattern2"), RuleExpr(OPERATORS.AND, "pattern3")],
),
]
result = evaluate_expression(expression, results)
assert result == set([Range(30, 70, {})]), f"{result}"
def RuleExpr(
operator: Operator,
fake_pattern_name: str,
children: Optional[List[BooleanRuleExpression]] = None,
) -> BooleanRuleExpression:
return BooleanRuleExpression(operator, PatternId(fake_pattern_name),
children, "fake-pattern-text-here")
def test_single_pattern_match_filtering() -> None:
results = {
PatternId("pattern1"):
[PatternMatchMock(30, 100, {
"$X": "x1",
"$Y": "y1"
})],
PatternId("pattern2"):
[PatternMatchMock(30, 100, {
"$X": "x1",
"$Y": "y2"
})],
PatternId("pattern3"): [PatternMatchMock(30, 100, {"$X": "x1"})],
}
expression = [
BooleanRuleExpression(
OPERATORS.AND_EITHER,
None,
[
RuleExpr(OPERATORS.AND, "pattern1"),
RuleExpr(OPERATORS.AND, "pattern2")
],
),
RuleExpr(OPERATORS.AND_NOT, "pattern3"),
]
result = evaluate_expression(expression, results)
assert result == set(), f"{result}"
def _build_boolean_expression(
self, rule: YamlTree[YamlMap]
) -> BooleanRuleExpression:
"""
Build a boolean expression from the yml lines in the rule
"""
rule_raw = rule.value
_rule_id = cast(str, rule_raw["id"].unroll())
rule_id = PatternId(_rule_id)
for pattern_name in pattern_names_for_operator(OPERATORS.AND):
pattern = rule_raw.get(pattern_name)
if pattern:
self._pattern_spans[rule_id] = pattern.span
return BooleanRuleExpression(
OPERATORS.AND, rule_id, None, pattern.value
)
for pattern_name in pattern_names_for_operator(OPERATORS.REGEX):
pattern = rule_raw.get(pattern_name)
if pattern:
self._pattern_spans[rule_id] = pattern.span
return BooleanRuleExpression(
OPERATORS.REGEX, rule_id, None, pattern.value
)
for pattern_name in pattern_names_for_operator(OPERATORS.AND_ALL):
patterns = rule_raw.get(pattern_name)
if patterns:
return BooleanRuleExpression(
operator=OPERATORS.AND_ALL,
pattern_id=None,
children=list(self._parse_boolean_expression(patterns)),
operand=None,
)
for pattern_name in pattern_names_for_operator(OPERATORS.AND_EITHER):
patterns = rule_raw.get(pattern_name)
if patterns:
return BooleanRuleExpression(
operator=OPERATORS.AND_EITHER,
pattern_id=None,
children=list(self._parse_boolean_expression(patterns)),
operand=None,
)
raise SemgrepError(f"rule with id '{_rule_id}' is missing top-level operator")
def _parse_boolean_expression(
self,
rule_patterns: YamlTree[List[YamlTree]],
pattern_id_idx: int = 0,
prefix: str = "",
) -> Iterator[BooleanRuleExpression]:
"""
Move through the expression from the YML, yielding tuples of (operator, unique-id-for-pattern, pattern)
"""
for rule_index, pattern_tree in enumerate(rule_patterns.value):
pattern = pattern_tree.value
for boolean_operator_yaml, sub_pattern in pattern.items():
boolean_operator: str = boolean_operator_yaml.value
operator = operator_for_pattern_name(boolean_operator_yaml)
if operator in set(OPERATORS_WITH_CHILDREN):
if isinstance(sub_pattern.value, list):
sub_expression = self._parse_boolean_expression(
sub_pattern, 0, f"{prefix}.{rule_index}.{pattern_id_idx}"
)
yield BooleanRuleExpression(
operator=operator,
pattern_id=None,
children=list(sub_expression),
operand=None,
)
else:
raise Exception("Internal error: bad schema")
else:
pattern_text, pattern_span = sub_pattern.value, sub_pattern.span
if isinstance(pattern_text, YamlMap):
pattern_text = {
k.value: v.value for k, v in pattern_text.items()
}
if isinstance(pattern_text, str) or isinstance(pattern_text, dict):
pattern_id = PatternId(f"{prefix}.{pattern_id_idx}")
self._pattern_spans[pattern_id] = pattern_span
yield BooleanRuleExpression(
operator=operator,
pattern_id=pattern_id,
children=None,
operand=pattern_text,
)
pattern_id_idx += 1
else:
raise Exception("Internal error: bad schema")
def test_build_exprs() -> None:
base_rule: Dict[str, Any] = {
"id": "test-id",
"message": "test message",
"languages": ["python"],
"severity": "ERROR",
}
rules: List[Dict[str, Any]] = [
{
**base_rule,
**{
"pattern": "test(...)"
}
},
{
**base_rule,
**{
"patterns": [{
"pattern": "test(...)"
}]
}
},
{
**base_rule,
**{
"pattern-either": [{
"pattern": "test(...)"
}]
}
},
]
results = [Rule.from_json(rule).expression for rule in rules]
base_expected = [
BooleanRuleExpression(OPERATORS.AND, PatternId(".0"), None,
"test(...)")
]
expected = [
BooleanRuleExpression(OPERATORS.AND, PatternId("test-id"), None,
"test(...)"),
BooleanRuleExpression(OPERATORS.AND_ALL, None, base_expected, None),
BooleanRuleExpression(OPERATORS.AND_EITHER, None, base_expected, None),
]
assert results == expected
def evaluate_expression(
exprs: List[BooleanRuleExpression],
pattern_ids_to_pattern_matches: Dict[PatternId, List[PatternMatch]],
flags: Optional[Dict[str, Any]] = None,
) -> Set[Range]:
# convert it to an implicit and
e = BooleanRuleExpression(OPERATORS.AND_ALL, None, exprs, None)
return raw_evaluate_expression(e, pattern_ids_to_pattern_matches, [],
flags)
def evaluate_expression(
exprs: List[BooleanRuleExpression],
pattern_ids_to_pattern_matches: Dict[PatternId, List[PatternMatch]],
allow_exec: bool = False,
) -> Set[Range]:
# convert it to an implicit and
e = BooleanRuleExpression(OPERATORS.AND_ALL, None, exprs, None)
result: Set[Range] = raw_evaluate_expression(
e, pattern_ids_to_pattern_matches, [], allow_exec=allow_exec)
return result
def _build_taint_expression(self, rule: YamlTree[YamlMap]) -> BooleanRuleExpression:
"""
Build an expression from the yml lines in the rule
"""
rule_raw = rule.value
_rule_id = cast(str, rule_raw["id"].unroll())
rule_id = PatternId(_rule_id)
for pattern_name in YAML_TAINT_MUST_HAVE_KEYS:
pattern = rule_raw[pattern_name]
self._pattern_spans[rule_id] = pattern.span
return BooleanRuleExpression(OPERATORS.AND, rule_id, None, None,)
def enumerate_patterns_in_boolean_expression(
expr: BooleanRuleExpression, ) -> Iterable[BooleanRuleExpression]:
"""
flatten a potentially nested expression
"""
if expr.children is not None:
# we need to preserve this parent of multiple children, but it has no corresponding pattern
yield BooleanRuleExpression(expr.operator, None, None, None)
# now yield all the children
for c in expr.children:
yield from enumerate_patterns_in_boolean_expression(c)
else:
yield expr
def _parse_boolean_expression(
self,
rule_patterns: YamlTree[List[YamlTree]],
pattern_id_idx: int = 0,
prefix: str = "",
) -> Iterator[BooleanRuleExpression]:
"""
Move through the expression from the YML, yielding tuples of (operator, unique-id-for-pattern, pattern)
"""
for rule_index, pattern_tree in enumerate(rule_patterns.value):
for boolean_operator_yaml, sub_pattern in pattern_tree.value.items(
):
operator = operator_for_pattern_name(boolean_operator_yaml)
if operator in OPERATORS_WITH_CHILDREN:
sub_expression = self._parse_boolean_expression(
sub_pattern, 0,
f"{prefix}.{rule_index}.{pattern_id_idx}")
yield BooleanRuleExpression(
operator=operator,
pattern_id=None,
children=list(sub_expression),
operand=None,
)
else:
if not isinstance(sub_pattern.value, (str, YamlMap)):
raise SemgrepError(
f"expected operator '{operator}' to have string or map value guaranteed by schema"
)
pattern_id = PatternId(f"{prefix}.{pattern_id_idx}")
self._pattern_spans[pattern_id] = sub_pattern.span
yield BooleanRuleExpression(
operator=operator,
pattern_id=pattern_id,
children=None,
operand=sub_pattern.value,
)
pattern_id_idx += 1
def _build_taint_expression(
self, rule: YamlTree[YamlMap]) -> BooleanRuleExpression:
"""
Build an expression from the yml lines in the rule
"""
rule_raw = rule.value
_rule_id = rule_raw["id"].unroll()
if not isinstance(_rule_id, str):
raise InvalidRuleSchemaError(
short_msg="invalid id",
long_msg=
f"rule id must be a string, but was {type(_rule_id).__name__}",
spans=[rule_raw["id"].span],
)
if rule_raw.get("metadata"):
raise InvalidRuleSchemaError(
short_msg="invalid key",
long_msg=f"metadata is not supported in {TAINT_MODE} mode",
spans=[rule_raw.key_tree("metadata").span],
)
rule_id = PatternId(_rule_id)
for pattern_name in YAML_TAINT_MUST_HAVE_KEYS:
pattern = rule_raw.get(pattern_name)
if not pattern:
raise InvalidRuleSchemaError(
short_msg=f"missing {pattern_name} key",
long_msg=
f"In {TAINT_MODE} mode, 'pattern-sources' and 'pattern-sinks' are both required",
spans=[rule.span.truncate(10)],
)
self._validate_list_operand(pattern_name, pattern)
self._pattern_spans[rule_id] = pattern.span
return BooleanRuleExpression(
OPERATORS.AND,
rule_id,
None,
None,
)
def _build_boolean_expression(
self, rule: YamlTree[YamlMap]) -> BooleanRuleExpression:
"""
Build a boolean expression from the yml lines in the rule
"""
rule_raw = rule.value
_rule_id = rule_raw["id"].unroll()
if not isinstance(_rule_id, str):
raise InvalidRuleSchemaError(
short_msg="invalid id",
long_msg=
f"rule id must be a string, but was {type(_rule_id).__name__}",
spans=[rule_raw["id"].span],
)
rule_id = PatternId(_rule_id)
for pattern_name in pattern_names_for_operator(OPERATORS.AND):
pattern = rule_raw.get(pattern_name)
if pattern:
self._pattern_spans[rule_id] = pattern.span
return BooleanRuleExpression(
OPERATORS.AND,
rule_id,
None,
self._validate_operand(pattern),
)
for pattern_name in pattern_names_for_operator(OPERATORS.REGEX):
pattern = rule_raw.get(pattern_name)
if pattern:
self._pattern_spans[rule_id] = pattern.span
return BooleanRuleExpression(
OPERATORS.REGEX,
rule_id,
None,
self._validate_operand(pattern),
)
for pattern_name in pattern_names_for_operator(OPERATORS.AND_ALL):
patterns = rule_raw.get(pattern_name)
if patterns:
return BooleanRuleExpression(
operator=OPERATORS.AND_ALL,
pattern_id=None,
children=list(self._parse_boolean_expression(patterns)),
operand=None,
)
for pattern_name in pattern_names_for_operator(OPERATORS.AND_EITHER):
patterns = rule_raw.get(pattern_name)
if patterns:
return BooleanRuleExpression(
operator=OPERATORS.AND_EITHER,
pattern_id=None,
children=list(self._parse_boolean_expression(patterns)),
operand=None,
)
required_operator = [
OPERATORS.AND_ALL,
OPERATORS.AND_EITHER,
OPERATORS.REGEX,
OPERATORS.AND,
]
raise InvalidRuleSchemaError(
short_msg="missing key",
long_msg=
f"missing a pattern type in rule, expected one of {pattern_names_for_operators(required_operator)}",
spans=[rule.span.truncate(10)],
)
def _parse_boolean_expression(
self,
rule_patterns: YamlTree[List[YamlTree]],
pattern_id_idx: int = 0,
prefix: str = "",
) -> Iterator[BooleanRuleExpression]:
"""
Move through the expression from the YML, yielding tuples of (operator, unique-id-for-pattern, pattern)
"""
if not isinstance(rule_patterns.value, list):
raise InvalidRuleSchemaError(
short_msg="invalid patterns",
long_msg=
f"invalid type for patterns; expected a list, but found {type(rule_patterns.unroll()).__name__}",
spans=[rule_patterns.span.with_context(before=1).truncate(5)],
help=
f"perhaps your YAML is missing a `-` on line {rule_patterns.span.start.line}?",
)
for rule_index, pattern_tree in enumerate(rule_patterns.value):
pattern = pattern_tree.value
if not isinstance(pattern, YamlMap):
raise InvalidRuleSchemaError(
short_msg="invalid pattern",
long_msg=
f"invalid type for pattern expected dict but found {type(pattern).__name__}",
spans=[pattern_tree.span],
help=f"Did you mean `pattern: {pattern}`?",
)
for boolean_operator_yaml, sub_pattern in pattern.items():
boolean_operator: str = boolean_operator_yaml.value
operator = operator_for_pattern_name(boolean_operator_yaml)
if operator in set(OPERATORS_WITH_CHILDREN):
if isinstance(sub_pattern.value, list):
sub_expression = self._parse_boolean_expression(
sub_pattern, 0,
f"{prefix}.{rule_index}.{pattern_id_idx}")
yield BooleanRuleExpression(
operator=operator,
pattern_id=None,
children=list(sub_expression),
operand=None,
)
else:
raise InvalidRuleSchemaError(
short_msg="missing children",
long_msg=
f"operator {boolean_operator} must have children",
spans=[
boolean_operator_yaml.span.extend_to(
sub_pattern.span)
],
)
else:
pattern_text, pattern_span = sub_pattern.value, sub_pattern.span
if isinstance(pattern_text, str):
pattern_id = PatternId(f"{prefix}.{pattern_id_idx}")
self._pattern_spans[pattern_id] = pattern_span
yield BooleanRuleExpression(
operator=operator,
pattern_id=pattern_id,
children=None,
operand=pattern_text,
)
pattern_id_idx += 1
else:
raise InvalidRuleSchemaError(
short_msg="invalid operand",
long_msg=
f"operand for {boolean_operator} must be a string, but instead was {type(sub_pattern.unroll()).__name__}",
spans=[
boolean_operator_yaml.span.extend_to(
pattern_span).truncate(5)
],
)
def test_exprs() -> None:
subexpression1 = [
BooleanRuleExpression(OPERATORS.AND_INSIDE, PatternId("pattern4"),
None, "p4"),
BooleanRuleExpression(OPERATORS.AND, PatternId("pattern2"), None,
"p2"),
]
subexpression2 = [
BooleanRuleExpression(OPERATORS.AND_NOT_INSIDE, PatternId("pattern4"),
None, "p4"),
BooleanRuleExpression(OPERATORS.AND, PatternId("pattern1"), None,
"p1"),
]
expression = BooleanRuleExpression(
OPERATORS.AND_ALL,
None,
[
BooleanRuleExpression(OPERATORS.AND_INSIDE, PatternId("pattern3"),
None, "p3"),
BooleanRuleExpression(
OPERATORS.AND_EITHER,
None,
[
BooleanRuleExpression(OPERATORS.AND_ALL,
PatternId("someid"), subexpression1),
BooleanRuleExpression(OPERATORS.AND_ALL,
PatternId("someid2"),
subexpression2),
],
),
],
)
flat = list(enumerate_patterns_in_boolean_expression(expression))
# print(flat)
expected = [
BooleanRuleExpression(OPERATORS.AND_ALL, None, None, None),
BooleanRuleExpression(OPERATORS.AND_INSIDE, PatternId("pattern3"),
None, "p3"),
BooleanRuleExpression(OPERATORS.AND_EITHER, None, None, None),
BooleanRuleExpression(OPERATORS.AND_ALL, None, None, None),
BooleanRuleExpression(OPERATORS.AND_INSIDE, PatternId("pattern4"),
None, "p4"),
BooleanRuleExpression(OPERATORS.AND, PatternId("pattern2"), None,
"p2"),
BooleanRuleExpression(OPERATORS.AND_ALL, None, None, None),
BooleanRuleExpression(OPERATORS.AND_NOT_INSIDE, PatternId("pattern4"),
None, "p4"),
BooleanRuleExpression(OPERATORS.AND, PatternId("pattern1"), None,
"p1"),
]
assert flat == expected, f"flat: {flat}"
def testF() -> None:
"""Nested boolean expressions
let pattern1 = bad(..., x=1)
let pattern2 = bad(..., y=2)
let pattern3 = def normal(): \n...
let pattern4 = def unusual(): \n...
We want to find (ONLY inside P3), either:
P2 inside P4
or
P1 not-inside P4
example:
000-100 def normal():
100-200 bad(x = 1) # P1 not-inside-P4
200-300 bad(y = 2) # no match
300-400 def unusual():
400-500 bad(x = 1) # no match
500-600 bad(y = 2) # P2 inside P4
600-700 def regular():
700-800 bad(x = 1) # P1 not-inside P4
800-900 bad(y = 2) # no-match
pattern-inside P3
and-either:
- patterns:
- pattern-inside P4
- and P2
- patterns:
- pattern-not-inside P4
- and P1
OUTPUT: [500-600], [700-800]
"""
results = {
PatternId("pattern1"): [
PatternMatchMock(100, 200),
PatternMatchMock(400, 500),
PatternMatchMock(700, 800),
],
PatternId("pattern2"): [
PatternMatchMock(200, 300),
PatternMatchMock(500, 600),
PatternMatchMock(800, 900),
],
PatternId("pattern3"): [PatternMatchMock(0, 900)],
PatternId("pattern4"): [PatternMatchMock(300, 600)],
}
subexpression1 = [
RuleExpr(OPERATORS.AND_INSIDE, "pattern4"),
RuleExpr(OPERATORS.AND, "pattern2"),
]
subexpression2 = [
RuleExpr(OPERATORS.AND_NOT_INSIDE, "pattern4"),
RuleExpr(OPERATORS.AND, "pattern1"),
]
expression = [
RuleExpr(OPERATORS.AND_INSIDE, "pattern3"),
BooleanRuleExpression(
OPERATORS.AND_EITHER,
None,
[
BooleanRuleExpression(OPERATORS.AND_ALL, None, subexpression1),
BooleanRuleExpression(OPERATORS.AND_ALL, None, subexpression2),
],
),
]
result = evaluate_expression(expression, results)
assert result == set(
[Range(100, 200, {}),
Range(500, 600, {}),
Range(700, 800, {})]), f"{result}"
