def testRepeat(self):
query = q.Query("(1, 2, 3, 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, 2, 3, 4))
# Repeated values flatten automatically.
query = q.Query("(1, (2, 3), 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, 2, 3, 4))
# Expressions work.
query = q.Query("(1, (2 + 2), 3, 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, 4, 3, 4))
# Repeated values are mono-types.
with self.assertRaises(errors.EfilterTypeError):
query = q.Query("(1, 'foo', 3, 4)")
solve.solve(query, {})
# None should be skipped.
query = q.Query(
ast.Repeat(ast.Literal(None), ast.Literal(2), ast.Literal(None),
ast.Literal(4)))
self.assertEqual(solve.solve(query, {}).value, repeated.meld(2, 4))
def _ParseTaggingFile(self, tag_file_path):
"""Parses tag definitions from the source.
Args:
tag_file_path (str): path to the tag file.
Returns:
efilter.ast.Expression: efilter abstract syntax tree (AST), containing the
tagging rules.
"""
tags = []
for label_name, rules in self._ParseDefinitions(tag_file_path):
if not rules:
logging.warning(
u'All rules for label "{0:s}" are invalid.'.format(
label_name))
continue
tag = efilter_ast.IfElse(
# Union will be true if any of the 'rules' match.
efilter_ast.Union(*[rule.root for rule in rules]),
# If so then evaluate to a string with the name of the tag.
efilter_ast.Literal(label_name),
# Otherwise don't return anything.
efilter_ast.Literal(None))
tags.append(tag)
# Generate a repeated value with all the tags (None will be skipped).
return efilter_ast.Repeat(*tags)
def parse(self):
tags = []
for tag_name, rules in self._parse_tagfile():
tag = ast.IfElse(
# Union will be true if any of the 'rules' match.
ast.Union(*[rule.root for rule in rules]),
# If so then evaluate to a string with the name of the tag.
ast.Literal(tag_name),
# Otherwise don't return anything.
ast.Literal(None))
tags.append(tag)
self.original.close()
# Generate a repeated value with all the tags (None will be skipped).
return ast.Repeat(*tags)
def testKVPairs(self):
self.assertQueryMatches("x: y", ast.Pair(ast.Var("x"), ast.Var("y")))
# KV pairs are used in named function arguments:
self.assertQueryMatches(
"f(10, 'strings': ['foo', 'bar'])",
ast.Apply(
ast.Var("f"), ast.Literal(10),
ast.Pair(ast.Literal("strings"),
ast.Tuple(ast.Literal("foo"), ast.Literal("bar")))))
# They can also appear in repeated values, forming a logical dictionary:
self.assertQueryMatches(
"('foo': foo, 'bar': bar)",
ast.Repeat(ast.Pair(ast.Literal("foo"), ast.Var("foo")),
ast.Pair(ast.Literal("bar"), ast.Var("bar"))))
def testRepeat(self):
query = q.Query("(1, 2, 3, 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, 2, 3, 4))
# Repeated values do not flatten automatically.
query = q.Query("(1, (2, 3), 4)")
self.assertEqual(
solve.solve(query, {}).value, repeated.meld(1, [2, 3], 4))
# Expressions work.
query = q.Query("(1, (2 + 2), 3, 4)")
self.assertEqual(
solve.solve(query, {}).value,
# Operators always return a list.
repeated.meld(1, [4], 3, 4))
# None should be skipped.
query = q.Query(
ast.Repeat(ast.Literal(None), ast.Literal(2), ast.Literal(None),
ast.Literal(4)))
self.assertEqual(solve.solve(query, {}).value, repeated.meld(2, 4))
def GetEventTaggingRules(self):
"""Retrieves the event tagging rules from the tagging file.
Returns:
efilter.ast.Expression: efilter abstract syntax tree (AST), containing the
tagging rules.
"""
tags = []
for label_name, rules in self._ParseDefinitions(self._path):
if not rules:
continue
tag = efilter_ast.IfElse(
# Union will be true if any of the 'rules' match.
efilter_ast.Union(*[rule.root for rule in rules]),
# If so then evaluate to a string with the name of the tag.
efilter_ast.Literal(label_name),
# Otherwise don't return anything.
efilter_ast.Literal(None))
tags.append(tag)
# Generate a repeated value with all the tags (None will be skipped).
return efilter_ast.Repeat(*tags)
def atom(self):
"""Parse an atom, which is most things.
Grammar:
atom =
[ prefix ]
( select_expression
| any_expression
| func_application
| let_expr
| var
| literal
| list
| "(" expression ")" ) .
"""
# Parameter replacement with literals.
if self.tokens.accept(grammar.param):
return self.param()
# Let expressions (let(x = 5, y = 10) x + y)
if self.tokens.accept(grammar.let):
return self.let()
# At the top level, we try to see if we are recursing into an SQL query.
if self.tokens.accept(grammar.select):
return self.select()
# A SELECT query can also start with 'ANY'.
if self.tokens.accept(grammar.select_any):
return self.select_any()
# Explicitly reject any keywords from SQL other than SELECT and ANY.
# If we don't do this they will match as valid symbols (variables)
# and that might be confusing to the user.
self.tokens.reject(grammar.sql_keyword)
# Match if-else before other things that consume symbols.
if self.tokens.accept(grammar.if_if):
return self.if_if()
# Operators must be matched first because the same symbols could also
# be vars or applications.
if self.tokens.accept(grammar.prefix):
operator = self.tokens.matched.operator
start = self.tokens.matched.start
expr = self.expression(operator.precedence)
return operator.handler(expr, start=start, end=expr.end,
source=self.original)
if self.tokens.accept(grammar.literal):
return ast.Literal(self.tokens.matched.value, source=self.original,
start=self.tokens.matched.start,
end=self.tokens.matched.end)
# Match builtin pseudo-functions before functions and vars to prevent
# overrides.
if self.tokens.accept(grammar.builtin):
return self.builtin(self.tokens.matched.value)
# Match applications before vars, because obviously.
if self.tokens.accept(grammar.application):
return self.application(
ast.Var(self.tokens.matched.value, source=self.original,
start=self.tokens.matched.start,
end=self.tokens.matched.first.end))
if self.tokens.accept(common_grammar.symbol):
name = self.tokens.matched.value
if self.aliases and name in self.aliases[-1]:
return self.aliases[-1][name]
return ast.Var(self.tokens.matched.value, source=self.original,
start=self.tokens.matched.start,
end=self.tokens.matched.end)
if self.tokens.accept(common_grammar.lparen):
# Parens will contain one or more expressions. If there are several
# expressions, separated by commas, then they are a repeated value.
#
# Unlike lists, repeated values must all be of the same type,
# otherwise evaluation of the query will fail at runtime (or
# type-check time, for simple cases.)
start = self.tokens.matched.start
expressions = [self.expression()]
while self.tokens.accept(common_grammar.comma):
expressions.append(self.expression())
self.tokens.expect(common_grammar.rparen)
if len(expressions) == 1:
return expressions[0]
return ast.Repeat(*expressions, source=self.original,
start=start, end=self.tokens.matched.end)
if self.tokens.accept(common_grammar.lbracket):
return self.list()
# We've run out of things we know the next atom could be. If there is
# still input left then it's illegal syntax. If there is nothing then
# the input cuts off when we still need an atom. Either is an error.
if self.tokens.peek(0):
return self.error(
"Was not expecting %r here." % self.tokens.peek(0).name,
start_token=self.tokens.peek(0))
return self.error("Unexpected end of input.")
def testReverse(self):
query = q.Query(
ast.Apply(
ast.Var("reverse"),
ast.Repeat(ast.Literal(1), ast.Literal(2), ast.Literal(3))))
self.assertEqual(solve.solve(query, {}).value, repeated.meld(3, 2, 1))
def testRepeatedExpressions(self):
self.assertQueryMatches(
"(1, 2, 3)",
ast.Repeat(ast.Literal(1), ast.Literal(2), ast.Literal(3)))
