def testRepeat(self): query = q.Query("(1, 2, 3, 4)") self.assertEqual( solve.solve(query, {}).value, repeated.meld(1, 2, 3, 4)) # Repeated values flatten automatically. query = q.Query("(1, (2, 3), 4)") self.assertEqual( solve.solve(query, {}).value, repeated.meld(1, 2, 3, 4)) # Expressions work. query = q.Query("(1, (2 + 2), 3, 4)") self.assertEqual( solve.solve(query, {}).value, repeated.meld(1, 4, 3, 4)) # Repeated values are mono-types. with self.assertRaises(errors.EfilterTypeError): query = q.Query("(1, 'foo', 3, 4)") solve.solve(query, {}) # None should be skipped. query = q.Query( ast.Repeat(ast.Literal(None), ast.Literal(2), ast.Literal(None), ast.Literal(4))) self.assertEqual(solve.solve(query, {}).value, repeated.meld(2, 4))
def _ParseTaggingFile(self, tag_file_path): """Parses tag definitions from the source. Args: tag_file_path (str): path to the tag file. Returns: efilter.ast.Expression: efilter abstract syntax tree (AST), containing the tagging rules. """ tags = [] for label_name, rules in self._ParseDefinitions(tag_file_path): if not rules: logging.warning( u'All rules for label "{0:s}" are invalid.'.format( label_name)) continue tag = efilter_ast.IfElse( # Union will be true if any of the 'rules' match. efilter_ast.Union(*[rule.root for rule in rules]), # If so then evaluate to a string with the name of the tag. efilter_ast.Literal(label_name), # Otherwise don't return anything. efilter_ast.Literal(None)) tags.append(tag) # Generate a repeated value with all the tags (None will be skipped). return efilter_ast.Repeat(*tags)
def parse(self): tags = [] for tag_name, rules in self._parse_tagfile(): tag = ast.IfElse( # Union will be true if any of the 'rules' match. ast.Union(*[rule.root for rule in rules]), # If so then evaluate to a string with the name of the tag. ast.Literal(tag_name), # Otherwise don't return anything. ast.Literal(None)) tags.append(tag) self.original.close() # Generate a repeated value with all the tags (None will be skipped). return ast.Repeat(*tags)
def testKVPairs(self): self.assertQueryMatches("x: y", ast.Pair(ast.Var("x"), ast.Var("y"))) # KV pairs are used in named function arguments: self.assertQueryMatches( "f(10, 'strings': ['foo', 'bar'])", ast.Apply( ast.Var("f"), ast.Literal(10), ast.Pair(ast.Literal("strings"), ast.Tuple(ast.Literal("foo"), ast.Literal("bar"))))) # They can also appear in repeated values, forming a logical dictionary: self.assertQueryMatches( "('foo': foo, 'bar': bar)", ast.Repeat(ast.Pair(ast.Literal("foo"), ast.Var("foo")), ast.Pair(ast.Literal("bar"), ast.Var("bar"))))
def testRepeat(self): query = q.Query("(1, 2, 3, 4)") self.assertEqual( solve.solve(query, {}).value, repeated.meld(1, 2, 3, 4)) # Repeated values do not flatten automatically. query = q.Query("(1, (2, 3), 4)") self.assertEqual( solve.solve(query, {}).value, repeated.meld(1, [2, 3], 4)) # Expressions work. query = q.Query("(1, (2 + 2), 3, 4)") self.assertEqual( solve.solve(query, {}).value, # Operators always return a list. repeated.meld(1, [4], 3, 4)) # None should be skipped. query = q.Query( ast.Repeat(ast.Literal(None), ast.Literal(2), ast.Literal(None), ast.Literal(4))) self.assertEqual(solve.solve(query, {}).value, repeated.meld(2, 4))
def GetEventTaggingRules(self): """Retrieves the event tagging rules from the tagging file. Returns: efilter.ast.Expression: efilter abstract syntax tree (AST), containing the tagging rules. """ tags = [] for label_name, rules in self._ParseDefinitions(self._path): if not rules: continue tag = efilter_ast.IfElse( # Union will be true if any of the 'rules' match. efilter_ast.Union(*[rule.root for rule in rules]), # If so then evaluate to a string with the name of the tag. efilter_ast.Literal(label_name), # Otherwise don't return anything. efilter_ast.Literal(None)) tags.append(tag) # Generate a repeated value with all the tags (None will be skipped). return efilter_ast.Repeat(*tags)
def atom(self): """Parse an atom, which is most things. Grammar: atom = [ prefix ] ( select_expression | any_expression | func_application | let_expr | var | literal | list | "(" expression ")" ) . """ # Parameter replacement with literals. if self.tokens.accept(grammar.param): return self.param() # Let expressions (let(x = 5, y = 10) x + y) if self.tokens.accept(grammar.let): return self.let() # At the top level, we try to see if we are recursing into an SQL query. if self.tokens.accept(grammar.select): return self.select() # A SELECT query can also start with 'ANY'. if self.tokens.accept(grammar.select_any): return self.select_any() # Explicitly reject any keywords from SQL other than SELECT and ANY. # If we don't do this they will match as valid symbols (variables) # and that might be confusing to the user. self.tokens.reject(grammar.sql_keyword) # Match if-else before other things that consume symbols. if self.tokens.accept(grammar.if_if): return self.if_if() # Operators must be matched first because the same symbols could also # be vars or applications. if self.tokens.accept(grammar.prefix): operator = self.tokens.matched.operator start = self.tokens.matched.start expr = self.expression(operator.precedence) return operator.handler(expr, start=start, end=expr.end, source=self.original) if self.tokens.accept(grammar.literal): return ast.Literal(self.tokens.matched.value, source=self.original, start=self.tokens.matched.start, end=self.tokens.matched.end) # Match builtin pseudo-functions before functions and vars to prevent # overrides. if self.tokens.accept(grammar.builtin): return self.builtin(self.tokens.matched.value) # Match applications before vars, because obviously. if self.tokens.accept(grammar.application): return self.application( ast.Var(self.tokens.matched.value, source=self.original, start=self.tokens.matched.start, end=self.tokens.matched.first.end)) if self.tokens.accept(common_grammar.symbol): name = self.tokens.matched.value if self.aliases and name in self.aliases[-1]: return self.aliases[-1][name] return ast.Var(self.tokens.matched.value, source=self.original, start=self.tokens.matched.start, end=self.tokens.matched.end) if self.tokens.accept(common_grammar.lparen): # Parens will contain one or more expressions. If there are several # expressions, separated by commas, then they are a repeated value. # # Unlike lists, repeated values must all be of the same type, # otherwise evaluation of the query will fail at runtime (or # type-check time, for simple cases.) start = self.tokens.matched.start expressions = [self.expression()] while self.tokens.accept(common_grammar.comma): expressions.append(self.expression()) self.tokens.expect(common_grammar.rparen) if len(expressions) == 1: return expressions[0] return ast.Repeat(*expressions, source=self.original, start=start, end=self.tokens.matched.end) if self.tokens.accept(common_grammar.lbracket): return self.list() # We've run out of things we know the next atom could be. If there is # still input left then it's illegal syntax. If there is nothing then # the input cuts off when we still need an atom. Either is an error. if self.tokens.peek(0): return self.error( "Was not expecting %r here." % self.tokens.peek(0).name, start_token=self.tokens.peek(0)) return self.error("Unexpected end of input.")
def testReverse(self): query = q.Query( ast.Apply( ast.Var("reverse"), ast.Repeat(ast.Literal(1), ast.Literal(2), ast.Literal(3)))) self.assertEqual(solve.solve(query, {}).value, repeated.meld(3, 2, 1))
def testRepeatedExpressions(self): self.assertQueryMatches( "(1, 2, 3)", ast.Repeat(ast.Literal(1), ast.Literal(2), ast.Literal(3)))