def testComparingTokens(self): t1 = grammar.Token("lparen", "(", 1, 2) t2 = grammar.Token("lparen", "(", 20, 21) self.assertEqual(t1, t2) t3 = grammar.Token("lparen", "(", 0, 10) t4 = grammar.Token("lparen", ")", 0, 10) self.assertNotEqual(t3, t4)
def emit_param(self, match, pattern, **_): param_name = match.group(1) if not param_name or param_name == "?": param_name = self._param_idx self._param_idx += 1 elif param_name and re.match(r"^\d+$", param_name): param_name = int(param_name) return grammar.Token(name=pattern.name, value=param_name, start=match.start(), end=match.end())
# DottySQL's operator table. The parser only supports pure prefix and infix # operators, as well as infix operators that have a suffix (like x[y]). # # Circumfix and pure suffix operators can be declared, but won't do anything. OPERATORS = common.OperatorTable( # Infix operators: common.Operator(name="or", precedence=0, assoc="left", handler=ast.Union, docstring="Logical OR.", prefix=None, suffix=None, infix=common.Token("symbol", "or")), common.Operator(name="and", precedence=1, assoc="left", handler=ast.Intersection, docstring="Logical AND.", prefix=None, suffix=None, infix=common.Token("symbol", "and")), common.Operator(name="==", precedence=3, assoc="left", handler=ast.Equivalence, docstring="Equivalence.", prefix=None, suffix=None,
class ExpressionParserTest(testlib.EfilterTestCase): OPERATORS = [ # Infix with precedence grammar.Operator(name="+", precedence=4, assoc="left", handler=ast.Sum, docstring=None, prefix=None, suffix=None, infix=grammar.Token("symbol", "+")), grammar.Operator(name="eq", precedence=3, assoc="left", handler=ast.Equivalence, docstring=None, prefix=None, infix=grammar.Token("symbol", "eq"), suffix=None), # Prefix grammar.Operator(name="unary -", precedence=5, assoc="right", handler=ast_transforms.NegateValue, docstring=None, infix=None, suffix=None, prefix=grammar.Token("symbol", "-")), # Mixfix (infix + suffix) grammar.Operator(name="[]", precedence=12, assoc="left", handler=ast.Select, docstring=None, prefix=None, infix=grammar.Token("lbracket", "["), suffix=grammar.Token("rbracket", "]")), # Circumfix with separator grammar.Operator(name="list builder", precedence=14, assoc="left", handler=ast.Tuple, docstring=None, prefix=grammar.Token("lbracket", "["), infix=grammar.Token("comma", ","), suffix=grammar.Token("rbracket", "]")) ] def parseQuery(self, query): t = tokenizer.LazyTokenizer(query) p = parser.ExpressionParser(self.OPERATORS, t) return p.parse() def assertQueryParses(self, query, expected): self.assertEqual(q.Query(expected), q.Query(self.parseQuery(query))) def testFailures(self): with self.assertRaises(errors.EfilterParseError): self.parseQuery("+ 5") with self.assertRaises(errors.EfilterParseError): self.parseQuery("5 +") with self.assertRaises(errors.EfilterParseError): self.parseQuery("") with self.assertRaises(errors.EfilterParseError): self.parseQuery("5 * 10") def testInfix(self): self.assertQueryParses( "5 + 5 eq 10", ast.Equivalence(ast.Sum(ast.Literal(5), ast.Literal(5)), ast.Literal(10))) def testParens(self): self.assertQueryParses( "5 + (5 eq 10)", # It doesn't have to make sense. ast.Sum(ast.Literal(5), ast.Equivalence(ast.Literal(5), ast.Literal(10)))) def testPrefix(self): self.assertQueryParses( "-5 + 5 eq - (10)", ast.Equivalence( ast.Sum(ast.Product(ast.Literal(-1), ast.Literal(5)), ast.Literal(5)), ast.Product(ast.Literal(-1), ast.Literal(10)))) def testMixfix(self): self.assertQueryParses("'foo'[0 ]", ast.Select(ast.Literal("foo"), ast.Literal(0))) self.assertQueryParses( # I refer you to my previous statement about making sense. " (5 +5) [ 'foo']", ast.Select(ast.Sum(ast.Literal(5), ast.Literal(5)), ast.Literal("foo"))) self.assertQueryParses( "5 + 5['foo' + 10]", ast.Sum( ast.Literal(5), ast.Select(ast.Literal(5), ast.Sum(ast.Literal("foo"), ast.Literal(10))))) def testCircumfix(self): self.assertQueryParses( "[1, 2, 3]", ast.Tuple(ast.Literal(1), ast.Literal(2), ast.Literal(3))) self.assertQueryParses( # Lists and selection are non-ambiguous. "10 + ['foo', 'bar'][1]", ast.Sum( ast.Literal(10), ast.Select(ast.Tuple(ast.Literal("foo"), ast.Literal("bar")), ast.Literal(1))))
def testOperatorLookups(self): tl = grammar.TokenLookupTable() tl.set(grammar.Token("symbol", "func"), "function") tl.set((grammar.Token("symbol", "end"), ), "end_st") self.assertEqual(tl.match([grammar.Token("symbol", "func", 1, 20)]), ("function", (grammar.Token("symbol", "func", 1, 20), ))) self.assertEqual( tl.match([ grammar.Token("symbol", "func_not", 0, None), grammar.Token("symbol", "func", 1, 20) ]), (None, None)) # Multi-token matches tl.set((grammar.Token("symbol", "not"), grammar.Token("symbol", "in")), "not in") self.assertEqual( tl.match([ grammar.Token("symbol", "not"), grammar.Token("symbol", "in"), grammar.Token("blah", "blah") ]), ("not in", (grammar.Token("symbol", "not"), grammar.Token("symbol", "in")))) # Default match is the longest tl.set(grammar.Token("symbol", "not"), "not") self.assertEqual( tl.match([ grammar.Token("symbol", "not"), grammar.Token("symbol", "in"), grammar.Token("blah", "blah") ]), ("not in", (grammar.Token("symbol", "not"), grammar.Token("symbol", "in")))) self.assertEqual( tl.match([ grammar.Token("symbol", "not"), grammar.Token("blah", "blah") ]), ("not", (grammar.Token("symbol", "not"), )))
class ObjectFilterSyntax(syntax.Syntax): OPERATORS = [ # Aliases for equivalence: grammar.Operator(name="equals", precedence=3, assoc="left", handler=ast.Equivalence, docstring=None, prefix=None, infix=grammar.Token("symbol", "equals"), suffix=None), grammar.Operator(name="is", precedence=3, assoc="left", handler=ast.Equivalence, docstring=None, prefix=None, infix=grammar.Token("symbol", "is"), suffix=None), grammar.Operator(name="==", precedence=3, assoc="left", handler=ast.Equivalence, docstring=None, prefix=None, infix=grammar.Token("symbol", "=="), suffix=None), grammar.Operator(name="notequals", precedence=3, assoc="left", handler=ast_transforms.ComplementEquivalence, docstring=None, prefix=None, infix=grammar.Token("symbol", "notequals"), suffix=None), grammar.Operator(name="isnot", precedence=3, assoc="left", handler=ast_transforms.ComplementEquivalence, docstring=None, prefix=None, infix=grammar.Token("symbol", "isnot"), suffix=None), grammar.Operator(name="!=", precedence=3, assoc="left", handler=ast_transforms.ComplementEquivalence, docstring=None, prefix=None, infix=grammar.Token("symbol", "!="), suffix=None), # Logical: grammar.Operator(name="or", precedence=0, assoc="left", handler=ast.Union, docstring="Logical OR.", prefix=None, suffix=None, infix=grammar.Token("symbol", "or")), grammar.Operator(name="and", precedence=1, assoc="left", handler=ast.Intersection, docstring="Logical AND.", prefix=None, suffix=None, infix=grammar.Token("symbol", "and")), grammar.Operator(name="||", precedence=0, assoc="left", handler=ast.Union, docstring="Logical OR.", prefix=None, suffix=None, infix=grammar.Token("symbol", "||")), grammar.Operator(name="&&", precedence=1, assoc="left", handler=ast.Intersection, docstring="Logical AND.", prefix=None, suffix=None, infix=grammar.Token("symbol", "&&")), # Comparisons: grammar.Operator(name=">=", precedence=3, assoc="left", handler=ast.PartialOrderedSet, docstring="Equal-or-greater-than.", prefix=None, suffix=None, infix=grammar.Token("symbol", ">=")), grammar.Operator(name="<=", precedence=3, assoc="left", handler=ast_transforms.ReversePartialOrderedSet, docstring="Equal-or-less-than.", prefix=None, suffix=None, infix=grammar.Token("symbol", "<=")), grammar.Operator(name=">", precedence=3, assoc="left", handler=ast.StrictOrderedSet, docstring="Greater-than.", prefix=None, suffix=None, infix=grammar.Token("symbol", ">")), grammar.Operator(name="<", precedence=3, assoc="left", handler=ast_transforms.ReverseStrictOrderedSet, docstring="Less-than.", prefix=None, suffix=None, infix=grammar.Token("symbol", "<")), # Set ops: grammar.Operator(name="notinset", precedence=3, assoc="left", handler=ast_transforms.ComplementMembership, docstring="Left-hand operand is not in list.", prefix=None, suffix=None, infix=(grammar.Token("symbol", "notinset"))), grammar.Operator(name="inset", precedence=3, assoc="left", handler=ast.Membership, docstring="Left-hand operand is in list.", prefix=None, suffix=None, infix=grammar.Token("symbol", "inset")), grammar.Operator(name="notcontains", precedence=3, assoc="left", handler=ast_transforms.ReverseComplementMembership, docstring="Right-hand operand is not in list.", prefix=None, suffix=None, infix=(grammar.Token("symbol", "notcontains"))), grammar.Operator(name="contains", precedence=3, assoc="left", handler=ast_transforms.ReverseMembership, docstring="Right-hand operand is in list.", prefix=None, suffix=None, infix=grammar.Token("symbol", "contains")), # Miscellaneous: grammar.Operator(name="unary -", precedence=5, assoc="right", handler=ast_transforms.NegateValue, docstring=None, infix=None, suffix=None, prefix=grammar.Token("symbol", "-")), grammar.Operator(name="list builder", precedence=14, assoc="left", handler=ast.Tuple, docstring=None, prefix=grammar.Token("lbracket", "["), infix=grammar.Token("comma", ","), suffix=grammar.Token("rbracket", "]")), grammar.Operator(name="regexp", precedence=3, assoc="left", handler=ast.RegexFilter, docstring="Match LHS against regex on RHS.", prefix=None, suffix=None, infix=grammar.Token("symbol", "regexp")), grammar.Operator(name=".", precedence=12, assoc="left", handler=ast_transforms.NormalizeResolve, docstring="OBJ.MEMBER -> return MEMBER of OBJ.", prefix=None, suffix=None, infix=grammar.Token("symbol", ".")), ] def __init__(self, original, params=None): super(ObjectFilterSyntax, self).__init__(original) if params is not None: raise ValueError("ObjectFilterSyntax doesn't support parameters.") t = tokenizer.LazyTokenizer(original) self.parser = parser.ExpressionParser(operators=self.OPERATORS, tokenizer=t) @property def root(self): return self.parser.parse()
def string_end(self, pattern, match, **_): self._pop_state() return grammar.Token(name=pattern.name, value=self.string, start=self.string_position, end=match.end())
def emit_float(self, string, match, pattern, **_): return grammar.Token(name=pattern.name, value=float(string), start=match.start(), end=match.end())
def emit(self, string, match, pattern, **_): """Emits a token using the current pattern match and pattern label.""" return grammar.Token(name=pattern.name, value=string, start=match.start(), end=match.end())