def testTuple(self):
query = q.Query("[1, 2, 3]")
self.assertEqual(solve.solve(query, {}).value, (1, 2, 3))
query = q.Query("[x + 5, 1 == 1, y['foo']]")
self.assertEqual(
solve.solve(query, {
"x": 2,
"y": {
"foo": "bar"
}
}).value, ([7], True, "bar"))
def testApply(self):
"""Make sure arguments to functions are normalized."""
original = query.Query(
("apply", ("var", "f"),
("|", ("var", "x"), ("|", ("var", "y"), ("var", "z"))), ("var",
"w")))
expected = query.Query(
("apply", ("var", "f"), ("|", ("var", "x"), ("var", "y"),
("var", "z")), ("var", "w")))
self.assertEqual(normalize.normalize(original), expected)
def testLispDottySQLRoundtrip(self):
lisp = syntax.Syntax.get_formatter("lisp")
dsql = syntax.Syntax.get_formatter("dottysql")
q = query.Query("(SELECT proc.pid FROM pslist(pid: 1)"
" WHERE proc.command in ['foo', 'bar'])"
"OR (SELECT proc.pid FROM psxview"
" WHERE proc.alive ORDER BY proc.command DESC)")
self.assertEqual(q.root, query.Query(lisp(q)).root)
self.assertEqual(dsql(q.root), dsql(query.Query(lisp(q)).root))
self.assertEqual(lisp(q),
lisp(query.Query(dsql(query.Query(lisp(q))))))
def testEquivalence(self):
self.assertTrue(
solve.solve(q.Query("pid == 1"), mocks.Process(1, None,
None)).value)
# repeated elements are expanded in the usual way.
self.assertTrue(
solve.solve(q.Query("[1] == 1"), mocks.Process(1, None,
None)).value)
# Same as [1,2] == [1, None]
self.assertFalse(
solve.solve(q.Query("[1, 2] == [1]"), mocks.Process(1, None,
None)).value)
def testIsInstance(self):
with self.assertRaises(
errors.EfilterTypeError,
error_f=lambda e: "Cannot find type named 'FooBar'" in str(e)):
solve.solve(q.Query("proc isa FooBar"), mocks.MockRootType())
self.assertTrue(
solve.solve(q.Query("proc isa Process"),
mocks.MockRootType()).value)
# Builtin types should work, too.
self.assertTrue(solve.solve(q.Query("5 isa int"), {}).value)
# Always never forget to test for negatives.
self.assertFalse(solve.solve(q.Query("5 isa str"), {}).value)
def testDottySQLRoundtrip(self):
f = syntax.Syntax.get_formatter("dottysql")
# Simple case:
q = query.Query("(SELECT * FROM pslist WHERE pid == 1)",
syntax="dottysql")
q2 = query.Query(f(q))
self.assertEqual(q.root, q2.root)
# ANY disambiguation:
q = query.Query(
"(SELECT ANY pslist WHERE pid == 1) "
"AND (SELECT ANY netstat WHERE socket.last_pid == 1)",
syntax="dottysql")
q2 = query.Query(f(q))
self.assertEqual(q.root, q2.root)
def search(query, data, replacements=None):
"""Yield objects from 'data' that match the 'query'."""
query = q.Query(query, params=replacements)
for entry in data:
if solve.solve(query, entry).value:
yield entry
def testMap(self):
self.assertEqual(
solve.solve(q.Query("foo.bar"), {
"foo": {
"bar": "baz"
}
}).value, "baz")
def _ParseRule(self, rule):
"""Parses a single tagging rule.
This method attempts to detect whether the rule is written with objectfilter
or dottysql syntax - either is acceptable.
Example:
_ParseRule('5 + 5')
# Returns Sum(Literal(5), Literal(5))
Args:
rule (str): rule in either objectfilter or dottysql syntax.
Returns:
efilter.query.Query: efilter query of the rule or None.
"""
if self._OBJECTFILTER_WORDS.search(rule):
syntax = u'objectfilter'
else:
syntax = u'dottysql'
try:
return efilter_query.Query(rule, syntax=syntax)
except efilter_errors.EfilterParseError as exception:
stripped_rule = rule.rstrip()
logging.warning(
u'Unable to build query from rule: "{0:s}" with error: {1:s}'.
format(stripped_rule, exception.message))
def testSelect(self):
self.assertEqual(
solve.solve(q.Query("x['y']"), {
"x": {
"y": 5
}
}).value, 5)
def _ParseEventTaggingRule(self, event_tagging_expression):
"""Parses an event tagging expression.
This method attempts to detect whether the event tagging expression is valid
objectfilter or dottysql syntax.
Example:
_ParseEventTaggingRule('5 + 5')
# Returns Sum(Literal(5), Literal(5))
Args:
event_tagging_expression (str): event tagging experssion either in
objectfilter or dottysql syntax.
Returns:
efilter.query.Query: efilter query of the event tagging expression.
Raises:
TaggingFileError: when the tagging file cannot be correctly parsed.
"""
if self._OBJECTFILTER_WORDS.search(event_tagging_expression):
syntax = 'objectfilter'
else:
syntax = 'dottysql'
try:
return efilter_query.Query(event_tagging_expression, syntax=syntax)
except efilter_errors.EfilterParseError as exception:
stripped_expression = event_tagging_expression.rstrip()
raise errors.TaggingFileError((
'Unable to parse event tagging expressoin: "{0:s}" with error: '
'{1!s}').format(stripped_expression, exception))
def render_output_analysis(self, renderer):
"""Render analysis of the expression's return type and its members."""
output_type = infer_type.infer_type(self.query, self)
renderer.section("Type Analysis", width=140)
renderer.table_header([
dict(name="Name",
cname="name",
type="TreeNode",
max_depth=2,
width=60),
dict(name="Type", cname="type", width=40)
])
renderer.table_row(self.query.source, repr(output_type), depth=1)
try:
for member in structured.getmembers(output_type):
subq = "(%s)[%r]" % (self.query.source, member)
subtype = infer_type.infer_type(q.Query(subq), self)
if isinstance(subtype, type):
subtype = subtype.__name__
else:
subtype = repr(subtype)
renderer.table_row(subq, subtype, depth=2)
except (NotImplementedError, TypeError, AttributeError):
pass
def testFormatters(self):
"""Creating a query with raw AST should generate the source."""
q = query.Query(
ast.Complement(
ast.Equivalence(ast.Map(ast.Var("Process"), ast.Var("pid")),
ast.Literal(10))))
self.assertEqual(q.source, "Process.pid != 10")
def testDifference(self):
# Adding a constant to a list, adds it to each element.
query = q.Query(
"SELECT ages - 10 as diff FROM (bind('ages': [10, 20, 30]))")
value = list(solve.solve(query, {}).value)
diff = structured.resolve(value[0], "diff")
self.assertEqual(diff, [0, 10, 20])
# Subtracting numbers from non numbers just gives None.
query = q.Query('SELECT ages - 10 as diff FROM '
'(bind("ages": ["foo", "bar", "baz"]))')
value = list(solve.solve(query, {}).value)
diff = structured.resolve(value[0], "diff")
self.assertEqual(diff, [None, None, None])
def testGroup(self):
result = api.apply(
query=q.Query(
(
"group",
# The input:
("apply", ("var", "csv"), ("param", 0), True),
# The grouper expression:
("var", "country"),
# The output reducers:
("reducer", ("var", "singleton"), ("var", "country")),
("reducer", ("var", "mean"), ("cast", ("var", "age"),
("var", "int"))),
("reducer", ("var", "sum"), ("cast", ("var", "age"),
("var", "int")))),
params=[testlib.get_fixture_path("fake_users.csv")]),
allow_io=True)
# Round the output means for comparison.
actual = []
for row in result:
row[1] = int(row[1])
actual.append(row)
expected = repeated.meld(['El Salvador', 55, 1287],
['Ethiopia', 55, 1210],
['French Guiana', 47, 381],
['Germany', 42, 299], ['Haiti', 46, 610],
['Mayotte', 50, 865], ['Portugal', 48, 485])
self.assertItemsEqual(expected, actual)
def testCrossComponentHints(self):
hint = self.hintQuery(
"MemoryDescriptor/process matches (Process/command == 'Adium')"
" and 'execute' in MemoryDescriptor/permissions"
" and 'write' in MemoryDescriptor/permissions",
selector="MemoryDescriptor/process")
self.assertEqual(hint, query.Query("Process/command == 'Adium'",
syntax="slashy"))
def testFilter(self):
self.assertValuesEqual(
solve.solve(
q.Query("select * from Process where (pid == 1)"), {
"Process":
repeated.meld(mocks.Process(2, None, None),
mocks.Process(1, None, None))
}).value, mocks.Process(1, None, None))
def testSourceCache(self):
"""Creating the query with valid source should preserve it."""
q = query.Query("Process.pid != 10") # Extra whitespace.
self.assertEqual(q.source, "Process.pid != 10")
# Normalization shouldn't mess up the code (because it's 1:1).
q = normalize.normalize(q)
self.assertEqual(q.source, "Process.pid != 10")
def testIfElse(self):
# Missing else:
q = query.Query(
ast.IfElse(
ast.Pair(ast.Literal(True), ast.Literal("foo")),
ast.Pair(ast.Literal(True), ast.Literal("bar"))))
with self.assertRaises(errors.EfilterLogicError):
validate.validate(q)
def testEach(self):
self.assertFalse(
solve.solve(
q.Query("each(Process.parent, (pid == 1))"), {
"Process": {
"parent":
repeated.meld(mocks.Process(1, None, None),
mocks.Process(2, None, None))
}
}).value)
def testAny(self):
self.assertTrue(
solve.solve(
q.Query("any Process.parent where (pid == 1)"), {
"Process": {
"parent":
repeated.meld(mocks.Process(1, None, None),
mocks.Process(2, None, None))
}
}).value)
# Test that unary ANY works as expected.
query = q.Query(ast.Any(ast.Var("x")))
self.assertFalse(solve.solve(query, {"x": None}).value)
self.assertTrue(solve.solve(query, {"x": 1}).value)
self.assertTrue(
solve.solve(query, {
"x": repeated.meld(1, 2, 3)
}).value)
def find_by_component(self, component, complete=True):
"""Finds all entities that have the component.
Arguments:
complete: If True, will attempt to collect the component.
"""
query = entity_query.Query(expression.ComponentLiteral(component))
if complete:
self.collect_for(query)
return list(self.lookup_tables["components"].lookup(component))
def infer(query,
replacements=None,
root_type=None,
libs=("stdcore", "stdmath")):
"""Determine the type of the query's output without actually running it.
Arguments:
query: A query object or string with the query.
replacements: Built-time parameters to the query, either as dict or as
an array (for positional interpolation).
root_type: The types of variables to be supplied to the query inference.
libs: What standard libraries should be taken into account for the
inference.
Returns:
The type of the query's output, if it can be determined. If undecidable,
returns efilter.protocol.AnyType.
NOTE: The inference returns the type of a row in the results, not of the
actual Python object returned by 'apply'. For example, if a query
returns multiple rows, each one of which is an integer, the type of the
output is considered to be int, not a collection of rows.
Examples:
infer("5 + 5") # -> INumber
infer("SELECT * FROM people WHERE age > 10") # -> AnyType
# If root_type implements the IStructured reflection API:
infer("SELECT * FROM people WHERE age > 10", root_type=...) # -> dict
"""
# Always make the scope stack start with stdcore.
if root_type:
type_scope = scope.ScopeStack(std_core.MODULE, root_type)
else:
type_scope = scope.ScopeStack(std_core.MODULE)
stdcore_included = False
for lib in libs:
if lib == "stdcore":
stdcore_included = True
continue
module = std_core.LibraryModule.ALL_MODULES.get(lib)
if not module:
raise TypeError("No standard library module %r." % lib)
type_scope = scope.ScopeStack(module, type_scope)
if not stdcore_included:
raise TypeError("'stdcore' must always be included.")
query = q.Query(query, params=replacements)
return infer_type.infer_type(query, type_scope)
def __init__(self, query, query_parameters=None, **kwargs):
super(EfilterPlugin, self).__init__(**kwargs)
self.query_source = query
try:
self.query = q.Query(query, params=query_parameters)
except errors.EfilterError as error:
self.query_error = error
self.query = None
self.query_parameters = query_parameters
def testDestructuring(self):
result = solve.solve(q.Query("Process.pid == 1"),
{"Process": {
"pid": 1
}})
self.assertTrue(result.value)
# Using a let-any form should succeed even if there is only one linked
# object.
result = solve.solve(
q.Query("any Process.parent where (Process.pid == 1 or "
"Process.command == 'foo')"),
{"Process": {
"parent": {
"Process": {
"pid": 1
}
}
}})
self.assertTrue(result.value)
def testRepeat(self):
query = q.Query("(1, 2, 3, 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, 2, 3, 4))
# Repeated values do not flatten automatically.
query = q.Query("(1, (2, 3), 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, [2, 3], 4))
# Expressions work.
query = q.Query("(1, (2 + 2), 3, 4)")
self.assertEqual(
solve.solve(query, {}).value,
# Operators always return a list.
repeated.meld(1, [4], 3, 4))
# None should be skipped.
query = q.Query(
ast.Repeat(ast.Literal(None), ast.Literal(2), ast.Literal(None),
ast.Literal(4)))
self.assertEqual(solve.solve(query, {}).value, repeated.meld(2, 4))
def __init__(self,
query=None,
explain=None,
columns=None,
sort=None,
width=None,
filter=None,
stream_results=False,
complete_results=True,
**kwargs):
super(EntityFind, self).__init__(**kwargs)
if query:
self.query = entity_query.Query(query)
else:
self.query = entity_query.Query(self.search)
if columns is not None:
self.columns = columns
if sort is not None:
self.sort = sort
if width is not None:
self.width = width
if filter is not None:
self.display_filter = entity_query.Query(filter)
if stream_results is not None:
if sort:
raise ValueError(
"Can't stream results and sort at the same time.")
self.stream_results = stream_results
if complete_results is not None:
self.complete_results = complete_results
self.explain = explain
def stream(self, query, handler, query_params=None):
query = entity_query.Query(query, params=query_params)
seen = set()
def _deduplicator(entity):
if entity in seen:
return
seen.add(entity)
handler(entity)
self.collect_for(query, result_stream_handler=_deduplicator)
for entity in self.find(query, complete=False, keep_cache=True):
_deduplicator(entity)
def __init__(self, *args, **kwargs):
super(EfilterPlugin, self).__init__(*args, **kwargs)
try:
self.query = q.Query(self.plugin_args.query,
params=self.plugin_args.query_parameters)
except errors.EfilterError as error:
self.query_error = error
self.query = None
except Exception:
# I am using a broad except here to make sure we always display a
# friendly error message. EFILTER will usually raise a friendly
# error, but we might get a non-EfilterError exception if the user
# gets creative (e.g. passing a custom object as query, instead of a
# string).
raise plugin.PluginError("Could not parse your query %r." %
(self.plugin_args.query, ))
def testQuery(self):
query = q.Query(
"SELECT proc.pid, proc.parent.pid FROM pslist() "
"WHERE proc.command == 'init'",
syntax="dottysql")
expected = \
("map",
("filter",
("apply", ("var", "pslist")),
("==",
(".", ("var", "proc"), "command"), "init")),
("bind",
(":", "pid", (".", ("var", "proc"), "pid")),
(":", "column_1",
(".", (".", ("var", "proc"), "parent"), "pid"))))
self.assertEqual(aslisp.aslisp(query), expected)
