def testRepeat(self):
query = q.Query("(1, 2, 3, 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, 2, 3, 4))
# Repeated values flatten automatically.
query = q.Query("(1, (2, 3), 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, 2, 3, 4))
# Expressions work.
query = q.Query("(1, (2 + 2), 3, 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, 4, 3, 4))
# Repeated values are mono-types.
with self.assertRaises(errors.EfilterTypeError):
query = q.Query("(1, 'foo', 3, 4)")
solve.solve(query, {})
# None should be skipped.
query = q.Query(
ast.Repeat(ast.Literal(None), ast.Literal(2), ast.Literal(None),
ast.Literal(4)))
self.assertEqual(solve.solve(query, {}).value, repeated.meld(2, 4))
def testPrefix(self):
self.assertQueryParses(
"-5 + 5 eq - (10)",
ast.Equivalence(
ast.Sum(ast.Product(ast.Literal(-1), ast.Literal(5)),
ast.Literal(5)),
ast.Product(ast.Literal(-1), ast.Literal(10))))
def testInfix(self):
self.assertQueryMatches(
"x == 'foo' and y.z contains 'bar'",
ast.Intersection(
ast.Equivalence(ast.Var("x"), ast.Literal("foo")),
ast.Membership(ast.Literal("bar"),
ast.Resolve(ast.Var("y"), ast.Literal("z")))))
def _ParseTaggingFile(self, tag_file_path):
"""Parses tag definitions from the source.
Args:
tag_file_path (str): path to the tag file.
Returns:
efilter.ast.Expression: efilter abstract syntax tree (AST), containing the
tagging rules.
"""
tags = []
for label_name, rules in self._ParseDefinitions(tag_file_path):
if not rules:
logging.warning(
u'All rules for label "{0:s}" are invalid.'.format(
label_name))
continue
tag = efilter_ast.IfElse(
# Union will be true if any of the 'rules' match.
efilter_ast.Union(*[rule.root for rule in rules]),
# If so then evaluate to a string with the name of the tag.
efilter_ast.Literal(label_name),
# Otherwise don't return anything.
efilter_ast.Literal(None))
tags.append(tag)
# Generate a repeated value with all the tags (None will be skipped).
return efilter_ast.Repeat(*tags)
def select_what(self):
# Each value we select is in form EXPRESSION [AS SYMBOL]. Values are
# separated by commas.
start = self.tokens.matched.start
used_names = set() # Keeps track of named values to prevent duplicates.
vars = []
for idx in itertools.count():
value_expression = self.expression()
if self.tokens.accept(grammar.select_as):
# If there's an AS then we have an explicit name for this value.
self.tokens.expect(common_grammar.symbol)
if self.tokens.matched.value in used_names:
return self.error(
"Duplicate 'AS' name %r." % self.tokens.matched.value)
key_expression = ast.Literal(self.tokens.matched.value,
start=self.tokens.matched.start,
end=self.tokens.matched.end,
source=self.original)
# Record the value expression of the alias.
self.aliases[-1][self.tokens.matched.value] = value_expression
used_names.add(self.tokens.matched.value)
else:
# Try to guess the appropriate name of the column based on what
# the expression is.
name = self._guess_name_of(value_expression)
if (not name or
name in used_names or
(self.scope and name in self.scope)):
# Give up and just use the current idx for key.
name = "column_%d" % (idx,)
else:
used_names.add(name)
key_expression = ast.Literal(name)
end = key_expression.end or value_expression.end
vars.append(ast.Pair(key_expression, value_expression,
start=value_expression.start, end=end,
source=self.original))
if self.tokens.accept(grammar.select_from):
# Make ast.Bind here.
source_expression = self.select_from()
return ast.Map(
source_expression,
ast.Bind(*vars, start=start, end=vars[-1].end,
source=self.original),
start=start,
end=self.tokens.matched.end,
source=self.original)
self.tokens.expect(common_grammar.comma)
def testIfElse(self):
# Missing else:
q = query.Query(
ast.IfElse(
ast.Pair(ast.Literal(True), ast.Literal("foo")),
ast.Pair(ast.Literal(True), ast.Literal("bar"))))
with self.assertRaises(errors.EfilterLogicError):
validate.validate(q)
def testSelectLimit(self):
self.assertQueryMatches(
"SELECT * FROM pslist LIMIT 10",
ast.Apply(ast.Var("take"), ast.Literal(10), ast.Var("pslist")))
self.assertQueryMatches(
"SELECT * FROM pslist LIMIT 10 OFFSET 5",
ast.Apply(
ast.Var("take"), ast.Literal(10),
ast.Apply(ast.Var("drop"), ast.Literal(5), ast.Var("pslist"))))
def testComplexSelect(self):
query = ("(SELECT proc.parent.pid AS ppid, proc.pid FROM pslist(10) "
"WHERE COUNT(proc.open_files) > 10) and True")
expected = ast.Intersection(
ast.Map(
ast.Filter(
ast.Apply(ast.Var("pslist"), ast.Literal(10)),
ast.StrictOrderedSet(
ast.Apply(
ast.Var("COUNT"),
ast.Resolve(ast.Var("proc"),
ast.Literal("open_files"))),
ast.Literal(10))),
ast.Bind(
ast.Pair(
ast.Literal("ppid"),
ast.Resolve(
ast.Resolve(ast.Var("proc"),
ast.Literal("parent")),
ast.Literal("pid"))),
ast.Pair(ast.Literal("pid"),
ast.Resolve(ast.Var("proc"),
ast.Literal("pid"))))),
ast.Literal(True))
self.assertQueryMatches(query, expected)
def testListLiterals(self):
self.assertQueryMatches(
"[1, 2, 3]",
ast.Tuple(ast.Literal(1), ast.Literal(2), ast.Literal(3)))
# Empty list literals should work.
self.assertQueryMatches("[]", ast.Tuple())
# Arbitrary AST should now be allowed in lists.
self.assertQueryMatches(
"[x, f(x)]",
ast.Tuple(ast.Var("x"), ast.Apply(ast.Var("f"), ast.Var("x"))))
def testLiterals(self):
# Numbers:
self.assertQueryMatches("5", ast.Literal(5))
self.assertQueryRaises("5)")
# Strings:
self.assertQueryMatches("'foo'", ast.Literal("foo"))
# Booleans:
self.assertQueryMatches("true", ast.Literal(True))
self.assertQueryMatches("false", ast.Literal(False))
self.assertQueryMatches("TRUE", ast.Literal(True))
self.assertQueryMatches("TRU", ast.Var("TRU"))
def parse(self):
tags = []
for tag_name, rules in self._parse_tagfile():
tag = ast.IfElse(
# Union will be true if any of the 'rules' match.
ast.Union(*[rule.root for rule in rules]),
# If so then evaluate to a string with the name of the tag.
ast.Literal(tag_name),
# Otherwise don't return anything.
ast.Literal(None))
tags.append(tag)
self.original.close()
# Generate a repeated value with all the tags (None will be skipped).
return ast.Repeat(*tags)
def let(self):
saved_start = self.tokens.matched.start
expect_rparens = 0
while self.tokens.accept(common_grammar.lparen):
expect_rparens += 1
bindings = []
while True:
symbol = self.tokens.expect(common_grammar.symbol)
binding = ast.Literal(symbol.value, start=symbol.start,
end=symbol.end, source=self.original)
self.tokens.expect(grammar.let_assign)
value = self.expression()
bindings.append(ast.Pair(binding, value, start=binding.start,
end=value.end, source=self.original))
if not self.tokens.accept(common_grammar.comma):
break
bind = ast.Bind(*bindings, start=bindings[0].start,
end=bindings[-1].end, source=self.original)
while expect_rparens:
self.tokens.expect(common_grammar.rparen)
expect_rparens -= 1
nested_expression = self.expression()
return ast.Let(bind, nested_expression, start=saved_start,
end=nested_expression.end, source=self.original)
def testFormatters(self):
"""Creating a query with raw AST should generate the source."""
q = query.Query(
ast.Complement(
ast.Equivalence(ast.Map(ast.Var("Process"), ast.Var("pid")),
ast.Literal(10))))
self.assertEqual(q.source, "Process.pid != 10")
def NormalizeResolve(x, y, **kwargs):
if isinstance(y, ast.Var):
literal_y = ast.Literal(y.value, start=y.start, end=y.end,
source=y.source)
else:
raise TypeError("Type of RHS must be Var. Got %r." % y)
return ast.Resolve(x, literal_y, **kwargs)
def testInfix(self):
self.assertQueryMatches("x + y", ast.Sum(ast.Var("x"), ast.Var("y")))
self.assertQueryMatches(
"w.x.y.z",
ast.Resolve(
ast.Resolve(ast.Resolve(ast.Var("w"), ast.Literal("x")),
ast.Literal("y")), ast.Literal("z")))
# Operator precedence should work correctly.
self.assertQueryMatches(
"x + y * z",
ast.Sum(ast.Var("x"), ast.Product(ast.Var("y"), ast.Var("z"))))
self.assertQueryMatches(
"x * y + z",
ast.Sum(ast.Product(ast.Var("x"), ast.Var("y")), ast.Var("z")))
def dot_rhs(self):
"""Match the right-hand side of a dot (.) operator.
The RHS must be a symbol token, but it is interpreted as a literal
string (because that's what goes in the AST of Resolve.)
"""
self.tokens.expect(common_grammar.symbol)
return ast.Literal(self.tokens.matched.value,
start=self.tokens.matched.start,
end=self.tokens.matched.end, source=self.original)
def testSelectWhereOrder(self):
self.assertQueryMatches(
"SELECT * FROM pslist() WHERE pid == 1 ORDER BY command DESC",
ast.Apply(
ast.Var("reverse"),
ast.Sort(
ast.Filter(ast.Apply(ast.Var("pslist")),
ast.Equivalence(ast.Var("pid"),
ast.Literal(1))),
ast.Var("command"))))
def testMixfix(self):
self.assertQueryParses("'foo'[0 ]",
ast.Select(ast.Literal("foo"), ast.Literal(0)))
self.assertQueryParses(
# I refer you to my previous statement about making sense.
" (5 +5) [ 'foo']",
ast.Select(ast.Sum(ast.Literal(5), ast.Literal(5)),
ast.Literal("foo")))
self.assertQueryParses(
"5 + 5['foo' + 10]",
ast.Sum(
ast.Literal(5),
ast.Select(ast.Literal(5),
ast.Sum(ast.Literal("foo"), ast.Literal(10)))))
def testOperatorPrecedence(self):
# Prefix operator, like the unary minus sign, should respect operator
# precedence order.
self.assertQueryMatches(
"-x + y",
ast.Sum(ast.Product(ast.Literal(-1), ast.Var("x")), ast.Var("y")))
self.assertQueryMatches(
"not x and y",
ast.Intersection(ast.Complement(ast.Var("x")), ast.Var("y")))
self.assertQueryMatches(
"x / -f(y) or not z(a, b)",
ast.Union(
ast.Quotient(
ast.Var("x"),
ast.Product(ast.Literal(-1),
ast.Apply(ast.Var("f"), ast.Var("y")))),
ast.Complement(
ast.Apply(ast.Var("z"), ast.Var("a"), ast.Var("b")))))
def testRepeat(self):
query = q.Query("(1, 2, 3, 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, 2, 3, 4))
# Repeated values do not flatten automatically.
query = q.Query("(1, (2, 3), 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, [2, 3], 4))
# Expressions work.
query = q.Query("(1, (2 + 2), 3, 4)")
self.assertEqual(
solve.solve(query, {}).value,
# Operators always return a list.
repeated.meld(1, [4], 3, 4))
# None should be skipped.
query = q.Query(
ast.Repeat(ast.Literal(None), ast.Literal(2), ast.Literal(None),
ast.Literal(4)))
self.assertEqual(solve.solve(query, {}).value, repeated.meld(2, 4))
def _parse_s_expression(self, atom):
car = atom[0]
cdr = atom[1:]
# Vars are a little special. Don't make the value a Literal.
if car == "var":
return ast.Var(cdr[0])
# Params are interpolated right away.
if car == "param":
return ast.Literal(self.params[cdr[0]])
return EXPRESSIONS[car](*[self._parse_atom(a) for a in cdr])
def asdottysql(expr):
branches = [
"if %s then %s" % (asdottysql(c), asdottysql(v))
for c, v in expr.conditions()
]
if_ = " else ".join(branches)
else_ = expr.default()
if not else_ or else_ == ast.Literal(None):
return if_
return "%s else %s" % (if_, asdottysql(else_))
def testCircumfix(self):
self.assertQueryParses(
"[1, 2, 3]",
ast.Tuple(ast.Literal(1), ast.Literal(2), ast.Literal(3)))
self.assertQueryParses(
# Lists and selection are non-ambiguous.
"10 + ['foo', 'bar'][1]",
ast.Sum(
ast.Literal(10),
ast.Select(ast.Tuple(ast.Literal("foo"), ast.Literal("bar")),
ast.Literal(1))))
def testLet(self):
self.assertQueryMatches(
"let x = 5, y = 10 x + y",
ast.Let(
ast.Bind(ast.Pair(ast.Literal("x"), ast.Literal(5)),
ast.Pair(ast.Literal("y"), ast.Literal(10))),
ast.Sum(ast.Var("x"), ast.Var("y"))))
self.assertQueryMatches(
"let( (x = 5 - 3,y=(10+(10)) ) )x + y",
ast.Let(
ast.Bind(
ast.Pair(ast.Literal("x"),
ast.Difference(ast.Literal(5), ast.Literal(3))),
ast.Pair(ast.Literal("y"),
ast.Sum(ast.Literal(10), ast.Literal(10)))),
ast.Sum(ast.Var("x"), ast.Var("y"))))
self.assertQueryRaises("let x = 5) x + 5")
self.assertQueryRaises("let ((x = 5) x + 5")
self.assertQueryRaises("let (x = 5)) x + 5")
self.assertQueryRaises("let (x = 5 x + 5")
self.assertQueryRaises("let (x = 5)")
def GetEventTaggingRules(self):
"""Retrieves the event tagging rules from the tagging file.
Returns:
efilter.ast.Expression: efilter abstract syntax tree (AST), containing the
tagging rules.
"""
tags = []
for label_name, rules in self._ParseDefinitions(self._path):
if not rules:
continue
tag = efilter_ast.IfElse(
# Union will be true if any of the 'rules' match.
efilter_ast.Union(*[rule.root for rule in rules]),
# If so then evaluate to a string with the name of the tag.
efilter_ast.Literal(label_name),
# Otherwise don't return anything.
efilter_ast.Literal(None))
tags.append(tag)
# Generate a repeated value with all the tags (None will be skipped).
return efilter_ast.Repeat(*tags)
def atom(self):
# Unary operator.
if self.tokens.accept(grammar.prefix, self.operators):
operator = self.tokens.matched.operator
start = self.tokens.matched.start
children = [self.expression(operator.precedence)]
# Allow infix to be repeated in circumfix operators.
if operator.infix:
while self.tokens.accept(grammar.match_tokens(operator.infix)):
children.append(self.expression())
# If we have a suffix expect it now.
if operator.suffix:
self.tokens.expect(grammar.match_tokens(operator.suffix))
return operator.handler(*children,
start=start,
end=self.tokens.matched.end,
source=self.original)
if self.tokens.accept(grammar.literal):
return ast.Literal(self.tokens.matched.value,
source=self.original,
start=self.tokens.matched.start,
end=self.tokens.matched.end)
if self.tokens.accept(grammar.symbol):
return ast.Var(self.tokens.matched.value,
source=self.original,
start=self.tokens.matched.start,
end=self.tokens.matched.end)
if self.tokens.accept(grammar.lparen):
expr = self.expression()
self.tokens.expect(grammar.rparen)
return expr
if self.tokens.peek(0):
raise errors.EfilterParseError(
message="Was not expecting %r here." %
self.tokens.peek(0).name,
token=self.tokens.peek(0))
else:
raise errors.EfilterParseError("Unexpected end of input.")
def param(self):
if self.tokens.matched.value is None:
param = self.last_param
self.last_param += 1
elif isinstance(self.tokens.matched.value, int):
param = self.last_param = self.tokens.matched.value
elif isinstance(self.tokens.matched.value, six.string_types):
param = self.tokens.matched.value
else:
return self.error(
"Invalid param %r." % self.tokens.matched.value,
start_token=self.tokens.matched.first)
if param not in self.params:
return self.error(
"Param %r unavailable. (Available: %r)" % (param, self.params),
start_token=self.tokens.matched.first)
return ast.Literal(self.params[param], start=self.tokens.matched.start,
end=self.tokens.matched.end, source=self.original)
def testLet(self):
self.assertEqual(
solve.solve(
ast.Let(ast.Bind(ast.Pair(ast.Literal("x"), ast.Literal(5))),
ast.Sum(ast.Var("x"), ast.Var("x"))), {}).value, [10])
# Previous binding should be made available to subsequent bindings.
self.assertEqual(
solve.solve(
ast.Let(
ast.Bind(
ast.Pair(ast.Literal("x"), ast.Literal(5)),
ast.Pair(ast.Literal("y"),
ast.Sum(ast.Var("x"), ast.Literal(5)))),
ast.Var("y")), {}).value, [10])
def testKVPairs(self):
self.assertQueryMatches("x: y", ast.Pair(ast.Var("x"), ast.Var("y")))
# KV pairs are used in named function arguments:
self.assertQueryMatches(
"f(10, 'strings': ['foo', 'bar'])",
ast.Apply(
ast.Var("f"), ast.Literal(10),
ast.Pair(ast.Literal("strings"),
ast.Tuple(ast.Literal("foo"), ast.Literal("bar")))))
# They can also appear in repeated values, forming a logical dictionary:
self.assertQueryMatches(
"('foo': foo, 'bar': bar)",
ast.Repeat(ast.Pair(ast.Literal("foo"), ast.Var("foo")),
ast.Pair(ast.Literal("bar"), ast.Var("bar"))))
def if_if(self):
start = self.tokens.matched.start
# Even-numbered children are conditions; odd-numbered are results.
# Last child is the else expression.
children = [self.expression()]
self.tokens.expect(grammar.if_then)
children.append(self.expression())
while self.tokens.accept(grammar.if_else_if):
children.append(self.expression())
self.tokens.expect(grammar.if_then)
children.append(self.expression())
if self.tokens.accept(grammar.if_else):
children.append(self.expression())
else:
children.append(ast.Literal(None))
return ast.IfElse(*children, start=start, end=self.tokens.matched.end,
source=self.original)