def testApplication(self):
self.assertQueryMatches(
"f(x, y)", ast.Apply(ast.Var("f"), ast.Var("x"), ast.Var("y")))
self.assertQueryRaises("f(x, ,)")
self.assertQueryRaises("f(x, y")
self.assertQueryRaises("f (x, y)")
def testFormatters(self):
"""Creating a query with raw AST should generate the source."""
q = query.Query(
ast.Complement(
ast.Equivalence(ast.Map(ast.Var("Process"), ast.Var("pid")),
ast.Literal(10))))
self.assertEqual(q.source, "Process.pid != 10")
def testComplexSelect(self):
query = ("(SELECT proc.parent.pid AS ppid, proc.pid FROM pslist(10) "
"WHERE COUNT(proc.open_files) > 10) and True")
expected = ast.Intersection(
ast.Map(
ast.Filter(
ast.Apply(ast.Var("pslist"), ast.Literal(10)),
ast.StrictOrderedSet(
ast.Apply(
ast.Var("COUNT"),
ast.Resolve(ast.Var("proc"),
ast.Literal("open_files"))),
ast.Literal(10))),
ast.Bind(
ast.Pair(
ast.Literal("ppid"),
ast.Resolve(
ast.Resolve(ast.Var("proc"),
ast.Literal("parent")),
ast.Literal("pid"))),
ast.Pair(ast.Literal("pid"),
ast.Resolve(ast.Var("proc"),
ast.Literal("pid"))))),
ast.Literal(True))
self.assertQueryMatches(query, expected)
def testInfix(self):
self.assertQueryMatches(
"x == 'foo' and y.z contains 'bar'",
ast.Intersection(
ast.Equivalence(ast.Var("x"), ast.Literal("foo")),
ast.Membership(ast.Literal("bar"),
ast.Resolve(ast.Var("y"), ast.Literal("z")))))
def select_limit(self, source_expression):
"""Match LIMIT take [OFFSET drop]."""
start = self.tokens.matched.start
# The expression right after LIMIT is the count to take.
limit_count_expression = self.expression()
# Optional OFFSET follows.
if self.tokens.accept(grammar.select_offset):
offset_start = self.tokens.matched.start
offset_end = self.tokens.matched.end
# Next thing is the count to drop.
offset_count_expression = self.expression()
# We have a new source expression, which is drop(count, original).
offset_source_expression = ast.Apply(
ast.Var("drop", start=offset_start, end=offset_end,
source=self.original),
offset_count_expression,
source_expression,
start=offset_start, end=offset_count_expression.end,
source=self.original)
# Drop before taking, because obviously.
source_expression = offset_source_expression
limit_expression = ast.Apply(
ast.Var("take", start=start, end=limit_count_expression.end,
source=self.original),
limit_count_expression,
source_expression,
start=start, end=self.tokens.matched.end, source=self.original)
return limit_expression
def testSelectOrder(self):
# Order expressions.
self.assertQueryMatches(
"SELECT * FROM pslist() ORDER BY pid",
ast.Sort(ast.Apply(ast.Var("pslist")), ast.Var("pid")))
self.assertQueryMatches(
"SELECT * FROM pslist() ORDER BY pid DESC",
ast.Apply(ast.Var("reverse"),
ast.Sort(ast.Apply(ast.Var("pslist")), ast.Var("pid"))))
def testSelectWhereOrder(self):
self.assertQueryMatches(
"SELECT * FROM pslist() WHERE pid == 1 ORDER BY command DESC",
ast.Apply(
ast.Var("reverse"),
ast.Sort(
ast.Filter(ast.Apply(ast.Var("pslist")),
ast.Equivalence(ast.Var("pid"),
ast.Literal(1))),
ast.Var("command"))))
def testSelectLimit(self):
self.assertQueryMatches(
"SELECT * FROM pslist LIMIT 10",
ast.Apply(ast.Var("take"), ast.Literal(10), ast.Var("pslist")))
self.assertQueryMatches(
"SELECT * FROM pslist LIMIT 10 OFFSET 5",
ast.Apply(
ast.Var("take"), ast.Literal(10),
ast.Apply(ast.Var("drop"), ast.Literal(5), ast.Var("pslist"))))
def testListLiterals(self):
self.assertQueryMatches(
"[1, 2, 3]",
ast.Tuple(ast.Literal(1), ast.Literal(2), ast.Literal(3)))
# Empty list literals should work.
self.assertQueryMatches("[]", ast.Tuple())
# Arbitrary AST should now be allowed in lists.
self.assertQueryMatches(
"[x, f(x)]",
ast.Tuple(ast.Var("x"), ast.Apply(ast.Var("f"), ast.Var("x"))))
def testSelectAny(self):
self.assertQueryMatches("SELECT ANY FROM pslist()",
ast.Any(ast.Apply(ast.Var("pslist"))))
# Shorthands for any should work.
self.assertQueryMatches("SELECT ANY FROM pslist()",
ast.Any(ast.Apply(ast.Var("pslist"))))
self.assertQueryMatches("ANY pslist()",
ast.Any(ast.Apply(ast.Var("pslist"))))
# Any doesn't allow ORDER BY.
self.assertQueryRaises("SELECT ANY FROM pslist() ORDER BY pid")
def testLet(self):
self.assertEqual(
solve.solve(
ast.Let(ast.Bind(ast.Pair(ast.Literal("x"), ast.Literal(5))),
ast.Sum(ast.Var("x"), ast.Var("x"))), {}).value, [10])
# Previous binding should be made available to subsequent bindings.
self.assertEqual(
solve.solve(
ast.Let(
ast.Bind(
ast.Pair(ast.Literal("x"), ast.Literal(5)),
ast.Pair(ast.Literal("y"),
ast.Sum(ast.Var("x"), ast.Literal(5)))),
ast.Var("y")), {}).value, [10])
def testKVPairs(self):
self.assertQueryMatches("x: y", ast.Pair(ast.Var("x"), ast.Var("y")))
# KV pairs are used in named function arguments:
self.assertQueryMatches(
"f(10, 'strings': ['foo', 'bar'])",
ast.Apply(
ast.Var("f"), ast.Literal(10),
ast.Pair(ast.Literal("strings"),
ast.Tuple(ast.Literal("foo"), ast.Literal("bar")))))
# They can also appear in repeated values, forming a logical dictionary:
self.assertQueryMatches(
"('foo': foo, 'bar': bar)",
ast.Repeat(ast.Pair(ast.Literal("foo"), ast.Var("foo")),
ast.Pair(ast.Literal("bar"), ast.Var("bar"))))
def select_order(self, source_expression):
start = self.tokens.matched.start
sort_expression = ast.Sort(source_expression, self.expression(),
start=start, end=self.tokens.matched.end,
source=self.original)
if self.tokens.accept(grammar.select_asc):
sort_expression.end = self.tokens.matched.end
return sort_expression
if self.tokens.accept(grammar.select_desc):
# Descending sort uses the stdlib function 'reverse' on the sorted
# results. Standard library's core functions should ALWAYS be
# available.
sort_expression = ast.Apply(
ast.Var("reverse",
start=sort_expression.start,
end=self.tokens.matched.end,
source=self.original),
sort_expression,
start=sort_expression.start,
end=self.tokens.matched.end,
source=self.original)
if self.tokens.accept(grammar.select_limit):
return self.select_limit(sort_expression)
if self.tokens.accept(grammar.select_limit):
return self.select_limit(sort_expression)
return sort_expression
def testBasicSelect(self):
self.assertQueryMatches("SELECT * FROM pslist()",
ast.Apply(ast.Var("pslist")))
# The dotty-like where doesn't exist. SQL keywords are not permitted
# outside of a SELECT expression.
self.assertQueryRaises("pslist where pid == 1")
def testSubscript(self):
self.assertQueryMatches("d['foo']",
ast.Select(ast.Var("d"), ast.Literal("foo")))
self.assertQueryMatches(
"d['foo'] + 10",
ast.Sum(ast.Select(ast.Var("d"), ast.Literal("foo")),
ast.Literal(10)))
self.assertQueryMatches(
"obj.props[0]",
ast.Select(ast.Resolve(ast.Var("obj"), ast.Literal("props")),
ast.Literal(0)))
self.assertQueryMatches(
"obj.props[0].foo",
ast.Resolve(
ast.Select(ast.Resolve(ast.Var("obj"), ast.Literal("props")),
ast.Literal(0)), ast.Literal("foo")))
self.assertQueryMatches(
"obj.props[10 + 10].foo",
ast.Resolve(
ast.Select(ast.Resolve(ast.Var("obj"), ast.Literal("props")),
ast.Sum(ast.Literal(10), ast.Literal(10))),
ast.Literal("foo")))
self.assertQueryMatches(
"w['x'][y[5] + 5] * 10",
ast.Product(
ast.Select(
ast.Select(ast.Var("w"), ast.Literal("x")),
ast.Sum(ast.Select(ast.Var("y"), ast.Literal(5)),
ast.Literal(5))), ast.Literal(10)))
def asdottysql(expr):
if not isinstance(expr.rhs, ast.Literal):
return "<expression cannot be formatted as DottySQL>"
return _format_binary(expr.lhs,
ast.Var(expr.rhs.value),
grammar.OPERATORS.by_handler[ast.Resolve],
lspace="",
rspace="")
def testOperatorPrecedence(self):
# Prefix operator, like the unary minus sign, should respect operator
# precedence order.
self.assertQueryMatches(
"-x + y",
ast.Sum(ast.Product(ast.Literal(-1), ast.Var("x")), ast.Var("y")))
self.assertQueryMatches(
"not x and y",
ast.Intersection(ast.Complement(ast.Var("x")), ast.Var("y")))
self.assertQueryMatches(
"x / -f(y) or not z(a, b)",
ast.Union(
ast.Quotient(
ast.Var("x"),
ast.Product(ast.Literal(-1),
ast.Apply(ast.Var("f"), ast.Var("y")))),
ast.Complement(
ast.Apply(ast.Var("z"), ast.Var("a"), ast.Var("b")))))
def testInfix(self):
self.assertQueryMatches("x + y", ast.Sum(ast.Var("x"), ast.Var("y")))
self.assertQueryMatches(
"w.x.y.z",
ast.Resolve(
ast.Resolve(ast.Resolve(ast.Var("w"), ast.Literal("x")),
ast.Literal("y")), ast.Literal("z")))
# Operator precedence should work correctly.
self.assertQueryMatches(
"x + y * z",
ast.Sum(ast.Var("x"), ast.Product(ast.Var("y"), ast.Var("z"))))
self.assertQueryMatches(
"x * y + z",
ast.Sum(ast.Product(ast.Var("x"), ast.Var("y")), ast.Var("z")))
def testSelectWhat(self):
self.assertQueryMatches(
"SELECT proc.parent.pid AS ppid,"
"proc.pid,"
"'foo',"
"asdate(proc.starttime),"
"proc.fd[5]"
"FROM pslist()",
ast.Map(
ast.Apply(ast.Var("pslist")),
ast.Bind(
ast.Pair(
ast.Literal("ppid"),
ast.Resolve(
ast.Resolve(ast.Var("proc"),
ast.Literal("parent")),
ast.Literal("pid"))),
ast.Pair(ast.Literal("pid"),
ast.Resolve(ast.Var("proc"), ast.Literal("pid"))),
ast.Pair(ast.Literal("column_2"), ast.Literal("foo")),
ast.Pair(
ast.Literal("asdate"),
ast.Apply(
ast.Var("asdate"),
ast.Resolve(ast.Var("proc"),
ast.Literal("starttime")))),
ast.Pair(
ast.Literal("fd_5"),
ast.Select(
ast.Resolve(ast.Var("proc"), ast.Literal("fd")),
ast.Literal(5))))))
def _parse_s_expression(self, atom):
car = atom[0]
cdr = atom[1:]
# Vars are a little special. Don't make the value a Literal.
if car == "var":
return ast.Var(cdr[0])
# Params are interpolated right away.
if car == "param":
return ast.Literal(self.params[cdr[0]])
return EXPRESSIONS[car](*[self._parse_atom(a) for a in cdr])
def testLiterals(self):
# Numbers:
self.assertQueryMatches("5", ast.Literal(5))
self.assertQueryRaises("5)")
# Strings:
self.assertQueryMatches("'foo'", ast.Literal("foo"))
# Booleans:
self.assertQueryMatches("true", ast.Literal(True))
self.assertQueryMatches("false", ast.Literal(False))
self.assertQueryMatches("TRUE", ast.Literal(True))
self.assertQueryMatches("TRU", ast.Var("TRU"))
def testLet(self):
self.assertQueryMatches(
"let x = 5, y = 10 x + y",
ast.Let(
ast.Bind(ast.Pair(ast.Literal("x"), ast.Literal(5)),
ast.Pair(ast.Literal("y"), ast.Literal(10))),
ast.Sum(ast.Var("x"), ast.Var("y"))))
self.assertQueryMatches(
"let( (x = 5 - 3,y=(10+(10)) ) )x + y",
ast.Let(
ast.Bind(
ast.Pair(ast.Literal("x"),
ast.Difference(ast.Literal(5), ast.Literal(3))),
ast.Pair(ast.Literal("y"),
ast.Sum(ast.Literal(10), ast.Literal(10)))),
ast.Sum(ast.Var("x"), ast.Var("y"))))
self.assertQueryRaises("let x = 5) x + 5")
self.assertQueryRaises("let ((x = 5) x + 5")
self.assertQueryRaises("let (x = 5)) x + 5")
self.assertQueryRaises("let (x = 5 x + 5")
self.assertQueryRaises("let (x = 5)")
def atom(self):
# Unary operator.
if self.tokens.accept(grammar.prefix, self.operators):
operator = self.tokens.matched.operator
start = self.tokens.matched.start
children = [self.expression(operator.precedence)]
# Allow infix to be repeated in circumfix operators.
if operator.infix:
while self.tokens.accept(grammar.match_tokens(operator.infix)):
children.append(self.expression())
# If we have a suffix expect it now.
if operator.suffix:
self.tokens.expect(grammar.match_tokens(operator.suffix))
return operator.handler(*children,
start=start,
end=self.tokens.matched.end,
source=self.original)
if self.tokens.accept(grammar.literal):
return ast.Literal(self.tokens.matched.value,
source=self.original,
start=self.tokens.matched.start,
end=self.tokens.matched.end)
if self.tokens.accept(grammar.symbol):
return ast.Var(self.tokens.matched.value,
source=self.original,
start=self.tokens.matched.start,
end=self.tokens.matched.end)
if self.tokens.accept(grammar.lparen):
expr = self.expression()
self.tokens.expect(grammar.rparen)
return expr
if self.tokens.peek(0):
raise errors.EfilterParseError(
message="Was not expecting %r here." %
self.tokens.peek(0).name,
token=self.tokens.peek(0))
else:
raise errors.EfilterParseError("Unexpected end of input.")
def testAny(self):
self.assertTrue(
solve.solve(
q.Query("any Process.parent where (pid == 1)"), {
"Process": {
"parent":
repeated.meld(mocks.Process(1, None, None),
mocks.Process(2, None, None))
}
}).value)
# Test that unary ANY works as expected.
query = q.Query(ast.Any(ast.Var("x")))
self.assertFalse(solve.solve(query, {"x": None}).value)
self.assertTrue(solve.solve(query, {"x": 1}).value)
self.assertTrue(
solve.solve(query, {
"x": repeated.meld(1, 2, 3)
}).value)
def testParens(self):
# Base case.
self.assertQueryMatches(
"x + y * z",
ast.Sum(ast.Var("x"), ast.Product(ast.Var("y"), ast.Var("z"))))
# With parens.
self.assertQueryMatches(
"(x + y) * z",
ast.Product(ast.Sum(ast.Var("x"), ast.Var("y")), ast.Var("z")))
# Missing rparen.
self.assertQueryRaises("(x + y")
# Empty expressions make no sense.
self.assertQueryRaises("()")
def testBuiltins(self):
self.assertQueryMatches(
"filter(pslist(), proc.pid == 1)",
ast.Filter(
ast.Apply(ast.Var("pslist")),
ast.Equivalence(
ast.Resolve(ast.Var("proc"), ast.Literal("pid")),
ast.Literal(1))))
self.assertQueryMatches(
"map(pslist(), [proc.pid, proc['command']])",
ast.Map(
ast.Apply(ast.Var("pslist")),
ast.Tuple(ast.Resolve(ast.Var("proc"), ast.Literal("pid")),
ast.Select(ast.Var("proc"),
ast.Literal("command")))))
self.assertQueryMatches(
"bind(x: 1, y: 2)",
ast.Bind(ast.Pair(ast.Var("x"), ast.Literal(1)),
ast.Pair(ast.Var("y"), ast.Literal(2))))
self.assertQueryRaises("bind (x: 1, y: 2)")
def atom(self):
"""Parse an atom, which is most things.
Grammar:
atom =
[ prefix ]
( select_expression
| any_expression
| func_application
| let_expr
| var
| literal
| list
| "(" expression ")" ) .
"""
# Parameter replacement with literals.
if self.tokens.accept(grammar.param):
return self.param()
# Let expressions (let(x = 5, y = 10) x + y)
if self.tokens.accept(grammar.let):
return self.let()
# At the top level, we try to see if we are recursing into an SQL query.
if self.tokens.accept(grammar.select):
return self.select()
# A SELECT query can also start with 'ANY'.
if self.tokens.accept(grammar.select_any):
return self.select_any()
# Explicitly reject any keywords from SQL other than SELECT and ANY.
# If we don't do this they will match as valid symbols (variables)
# and that might be confusing to the user.
self.tokens.reject(grammar.sql_keyword)
# Match if-else before other things that consume symbols.
if self.tokens.accept(grammar.if_if):
return self.if_if()
# Operators must be matched first because the same symbols could also
# be vars or applications.
if self.tokens.accept(grammar.prefix):
operator = self.tokens.matched.operator
start = self.tokens.matched.start
expr = self.expression(operator.precedence)
return operator.handler(expr, start=start, end=expr.end,
source=self.original)
if self.tokens.accept(grammar.literal):
return ast.Literal(self.tokens.matched.value, source=self.original,
start=self.tokens.matched.start,
end=self.tokens.matched.end)
# Match builtin pseudo-functions before functions and vars to prevent
# overrides.
if self.tokens.accept(grammar.builtin):
return self.builtin(self.tokens.matched.value)
# Match applications before vars, because obviously.
if self.tokens.accept(grammar.application):
return self.application(
ast.Var(self.tokens.matched.value, source=self.original,
start=self.tokens.matched.start,
end=self.tokens.matched.first.end))
if self.tokens.accept(common_grammar.symbol):
name = self.tokens.matched.value
if self.aliases and name in self.aliases[-1]:
return self.aliases[-1][name]
return ast.Var(self.tokens.matched.value, source=self.original,
start=self.tokens.matched.start,
end=self.tokens.matched.end)
if self.tokens.accept(common_grammar.lparen):
# Parens will contain one or more expressions. If there are several
# expressions, separated by commas, then they are a repeated value.
#
# Unlike lists, repeated values must all be of the same type,
# otherwise evaluation of the query will fail at runtime (or
# type-check time, for simple cases.)
start = self.tokens.matched.start
expressions = [self.expression()]
while self.tokens.accept(common_grammar.comma):
expressions.append(self.expression())
self.tokens.expect(common_grammar.rparen)
if len(expressions) == 1:
return expressions[0]
return ast.Repeat(*expressions, source=self.original,
start=start, end=self.tokens.matched.end)
if self.tokens.accept(common_grammar.lbracket):
return self.list()
# We've run out of things we know the next atom could be. If there is
# still input left then it's illegal syntax. If there is nothing then
# the input cuts off when we still need an atom. Either is an error.
if self.tokens.peek(0):
return self.error(
"Was not expecting %r here." % self.tokens.peek(0).name,
start_token=self.tokens.peek(0))
return self.error("Unexpected end of input.")
def testSelectAnyWhere(self):
self.assertQueryMatches(
"SELECT ANY FROM pslist() WHERE pid == 1",
ast.Any(ast.Apply(ast.Var("pslist")),
ast.Equivalence(ast.Var("pid"), ast.Literal(1))))
def testFullSelect(self):
query = ("SELECT proc.parent.pid AS ppid, proc.pid"
" FROM pslist(pid: 10, ppid: 20)"
" WHERE count(proc.open_files) > 10"
" ORDER BY proc.command DESC"
" LIMIT 10 - 9 OFFSET add(5, 10)")
expected = ast.Map(
ast.Apply(
ast.Var("take"), ast.Difference(ast.Literal(10),
ast.Literal(9)),
ast.Apply(
ast.Var("drop"),
ast.Apply(ast.Var("add"), ast.Literal(5), ast.Literal(10)),
ast.Apply(
ast.Var("reverse"),
ast.Sort(
ast.Filter(
ast.Apply(
ast.Var("pslist"),
ast.Pair(ast.Var("pid"), ast.Literal(10)),
ast.Pair(ast.Var("ppid"),
ast.Literal(20))),
ast.StrictOrderedSet(
ast.Apply(
ast.Var("count"),
ast.Resolve(
ast.Var("proc"),
ast.Literal("open_files"))),
ast.Literal(10))),
ast.Resolve(ast.Var("proc"),
ast.Literal("command")))))),
ast.Bind(
ast.Pair(
ast.Literal("ppid"),
ast.Resolve(
ast.Resolve(ast.Var("proc"), ast.Literal("parent")),
ast.Literal("pid"))),
ast.Pair(ast.Literal("pid"),
ast.Resolve(ast.Var("proc"), ast.Literal("pid")))))
self.assertQueryMatches(query, expected)
def testReverse(self):
query = q.Query(
ast.Apply(
ast.Var("reverse"),
ast.Repeat(ast.Literal(1), ast.Literal(2), ast.Literal(3))))
self.assertEqual(solve.solve(query, {}).value, repeated.meld(3, 2, 1))