def test_comments(self):
"""Test that comments are valid syntax but stripped from AST."""
match = parse_query("process where pid=4 and ppid=0")
query = parse_query(
"""process where pid = 4 /* multi\nline\ncomment */ and ppid=0""")
self.assertEqual(match, query)
query = parse_query(
"""process where pid = 4 // something \n and ppid=0""")
self.assertEqual(match, query)
query = parse_query("""process where pid
= 4 and ppid=0
""")
self.assertEqual(match, query)
query = parse_query("""process where
// test
//
//line
//comments
pid = 4 and ppid = 0
""")
self.assertEqual(match, query)
match = parse_expression("true")
query = parse_expression(
"true // something else \r\n /* test\r\n something \r\n*/")
self.assertEqual(match, query)
commented = parse_definitions(
"macro test() pid = 4 and /* comment */ ppid = 0")
macro = parse_definitions("macro test() pid = 4 and ppid = 0")
self.assertEqual(commented, macro)
def test_mixed_definitions(self):
"""Test that macro and constant definitions can be loaded correctly."""
defn = parse_definitions("""
const magic = 100
macro OR(a, b) a or b
""")
pp = PreProcessor(defn)
# Confirm that copy and adding is working
pp2 = pp.copy()
pp.add_definition(
parse_definition("macro ABC(a, b, c) error_error_error"))
pp2.add_definition(
parse_definition("macro ABC(a, b, c) f(a, magic, c)"))
matches = [
("abc", "abc"),
("OR(x, y)", "x or y"),
("magic", "100"),
("ABC(0,1,2)", "f(0, 100, 2)"),
]
for before, after in matches:
before = parse_expression(before)
after = parse_expression(after)
self.assertEqual(pp2.expand(before), after)
def __init__(self, config=None):
"""Create the engine with an optional list of files."""
super(BaseEngine, self).__init__(config)
self.analytics = [] # type: list[EqlAnalytic]
self.preprocessor = PreProcessor()
with use_schema(self.schema):
definitions = self.get_config('definitions', [])
if is_string(definitions):
definitions = parse_definitions(definitions)
self.preprocessor.add_definitions(definitions)
for path in self.get_config('definitions_files', []):
with open(path, 'r') as f:
definitions = parse_definitions(f.read())
self.preprocessor.add_definitions(definitions)
def test_engine_schema(self):
"""Test loading the engine with a custom schema."""
queries = [
'movie where name == "*Breakfast*" and IN_80s(release)',
'person where name == "John Hughes"',
]
analytic_dicts = [{'query': q} for q in queries]
definitions = """
macro IN_80s(date) date == "*/*/1980"
"""
config = {
'schema': {
'event_types': {
'movie': 1,
'person': 2
}
},
'definitions': parse_definitions(definitions),
'analytics': analytic_dicts
}
pp = PreProcessor()
pp.add_definitions(config['definitions'])
with use_schema(config['schema']):
expected = [
parse_analytic(d, preprocessor=pp) for d in analytic_dicts
]
engine = BaseEngine(config)
with use_schema(engine.schema):
engine.add_analytics([parse_analytic(d) for d in analytic_dicts])
self.assertListEqual(
engine.analytics, expected,
"Analytics were not loaded and expanded properly.")
def test_macro_expansion(self):
"""Test EQL custom macros."""
expanded = {
"A_OR_B":
"a or b",
"XOR":
"(a and not b) or (b and not a)",
"IN_GRAYLIST":
'proc in ("msbuild.exe", "powershell.exe", "cmd.exe", "netsh.exe")',
"PROCESS_IN_GRAYLIST":
'process_name in ("msbuild.exe", "powershell.exe", "cmd.exe", "netsh.exe")',
"PARENT_XOR_CHILD_IN_GRAYLIST":
('( '
' process_name in ("msbuild.exe", "powershell.exe", "cmd.exe", "netsh.exe") and not '
' parent_process_name in ("msbuild.exe", "powershell.exe", "cmd.exe", "netsh.exe")'
') or ('
' parent_process_name in ("msbuild.exe", "powershell.exe", "cmd.exe", "netsh.exe") and not '
' process_name in ("msbuild.exe", "powershell.exe", "cmd.exe", "netsh.exe")'
')'),
'DESCENDANT_OF_PROC':
'descendant of [process where opcode == 1 and expr]'
}
macros = parse_definitions(self.macro_definitions)
lookup = OrderedDict()
for macro in macros:
lookup[macro.name] = macro
rendered = macro.render()
macro_copy = parse_definition(rendered)
self.assertEqual(macro, macro_copy)
self.assertEqual(rendered, macro_copy.render(),
"Macro doesn't render valid EQL.")
# Now load up each macro to the engine
engine = PreProcessor(macros)
# Confirm that nested macros are expanded appropriately
for name, macro in engine.macros.items():
if name == 'safePath':
continue
expected_expr = parse_expression(expanded[name])
self.assertEqual(macro.expression, expected_expr)
self.assertEqual(macro.expression.render(), expected_expr.render())
# Expand some EQL queries
queries = [
('process where DESCENDANT_OF_PROC(process_name="explorer.exe")',
'process where descendant of [process where opcode=1 and process_name == "explorer.exe"]'
),
('process where XOR(a=="b", c=="d")',
'process where ((a == "b") and not (c == "d")) or ((c == "d") and not (a == "b"))'
),
(
'file where true',
'file where true',
),
('process where opcode=1 and PROCESS_IN_GRAYLIST()',
'process where opcode==1 and process_name in ("msbuild.exe","powershell.exe","cmd.exe","netsh.exe")'
),
]
for query, expanded_query in queries:
before_node = parse_query(query)
actual = engine.expand(before_node)
expected = parse_query(expanded_query)
# Test that eval + repr works
actual_repr = repr(actual)
eval_actual = eval(actual_repr)
self.assertEqual(actual, expected)
self.assertEqual(eval_actual, actual)
self.assertTrue(actual == expected)
self.assertFalse(actual != expected)
error_msg = "'{}' expanded to '{}' instead of '{}'".format(
query, actual.render(), expected.render())
self.assertEqual(actual.render(), expected.render(), error_msg)
query = parse_expression("DESCENDANT_OF_PROC()")
self.assertRaisesRegexp(ValueError,
"Macro .+ expected \d+ arguments .*",
engine.expand, query)
query = parse_expression("DESCENDANT_OF_PROC(1,2,3)")
self.assertRaisesRegexp(ValueError,
"Macro .+ expected \d+ arguments .*",
engine.expand, query)