def terminal(self, text): """terminal = '"' . (printable - '"') + . '"' | "'" . (printable - "'") + . "'" ;""" self._attempting(text) return alternation([ concatenation([ '"', one_or_more( exclusion( self.printable, '"' ), ignore_whitespace=False ), '"', ], ignore_whitespace=False), concatenation([ "'", one_or_more( exclusion( self.printable, "'" ), ignore_whitespace=False ), "'", ], ignore_whitespace=False), ])(text).compressed(TokenType.terminal)
def number(self, text): """number = ["-"] . ("0" | digit - "0" . {digit}) . ["." . digit +] ;""" self._attempting(text) return concatenation([ option( "-" ), alternation([ "0", concatenation([ exclusion( self.digit, "0" ), zero_or_more( self.digit, ignore_whitespace=False ), ], ignore_whitespace=False), ]), option( concatenation([ ".", one_or_more( self.digit, ignore_whitespace=False ), ], ignore_whitespace=False) ), ], ignore_whitespace=False)(text).compressed(TokenType.number)
def test_optional_repetition(self): # Makes sure nested repetitions will terminate. c0 = P.zero_or_more(P.option(P.alternation(["a", "b"]))) t0 = "ababaabbaaab" n0 = c0(t0) self.assertEqual(n0.consumed, len(t0))
def identifier(self, text): """identifier = simple_identifier | complex_identifier ;""" self._attempting(text) return alternation([ self.simple_identifier, self.complex_identifier, ])(text).retyped(TokenType.identifier)
def identifier(self, text): """identifier = (alpha_character | "_") . {alpha_character | "_" | digit} ;""" self._attempting(text) return concatenation([ alternation([ self.alpha_character, "_", ]), zero_or_more( alternation([ self.alpha_character, "_", self.digit, ]), ignore_whitespace=False ), ], ignore_whitespace=False)(text).compressed(TokenType.identifier)
def operator(self, text): """operator = "|" | "." | "," | "-" ;""" self._attempting(text) return alternation([ "|", ".", ",", "-", ])(text).retyped(TokenType.operator)
def simple_identifier(self, text): """simple_identifier = (alpha_character | "_") . {alpha_character | digit | "_" | "."} ;""" self._attempting(text) return concatenation([ alternation([ self.alpha_character, "_", ]), zero_or_more( alternation([ self.alpha_character, self.digit, "_", ".", ]), ignore_whitespace=False ), ], ignore_whitespace=False)(text).retyped(TokenType.simple_identifier)
def test_alternation(self): c0 = P.alternation(["a", "b", "c", "a("]) self.assertEqual(c0("agh").value, "a") self.assertEqual(c0("b(").value, "b") self.assertEqual(c0("c_-").value, "c") self.assertEqual(c0("a(b, c)").value, "a(") with self.assertRaises(P.DeadEnd): c0("d")
def statement(self, text): """statement = comment | assignment , ";" ;""" self._attempting(text) return alternation([ self.comment, concatenation([ self.assignment, ";", ], ignore_whitespace=True), ])(text)
def reserved_words(self, text): """reserved_words = "return" | "if" | "elif" | "else" | "switch" | "when";""" self._attempting(text) return alternation([ "return", "if", "elif", "else", "switch", "when", ])(text).retyped(TokenType.reserved_words)
def assignment(self, text): """assignment = variable , (":=" | "<-") , expression ;""" self._attempting(text) return concatenation([ self.variable, alternation([ ":=", "<-", ]), self.expression, ], ignore_whitespace=True)(text).retyped(TokenType.assignment)
def test_exclusion(self): c0 = P.alternation(["a", "b"]) c1 = P.exclusion(c0, "b") n0 = c1("a") self.assertEqual(n0.node_type, P.ParseNodeType.terminal) self.assertEqual(n0.value, "a") self.assertEqual(c0("b").value, "b") with self.assertRaises(P.DeadEnd): c1("ba")
def expression_terminal(self, text): """expression_terminal = identifier | terminal | option_group | repetition_group | grouping_group | special_handling ;""" self._attempting(text) return alternation([ self.identifier, self.terminal, self.option_group, self.repetition_group, self.grouping_group, self.special_handling, ])(text)
def grammar(self, text): """grammar = {comment} , rule , {comment | rule} ;""" self._attempting(text) return concatenation([ zero_or_more( self.comment, ignore_whitespace=True ), self.rule, zero_or_more( alternation([ self.comment, self.rule, ]), ignore_whitespace=True ), ], ignore_whitespace=True)(text).retyped(TokenType.grammar)
def expression_terminal(self, text): """expression_terminal = number | string | variable | subexpression | function_call | branch | switch ;""" self._attempting(text) return alternation([ self.number, self.string, self.variable, self.subexpression, self.function_call, self.branch, self.switch, ])(text)
def expression(self, text): """expression = number , op_mult , expression | expression_terminal , op_mult , number , [operator , expression] | expression_terminal , op_add , [operator , expression] | expression_terminal , [operator , expression] ;""" self._attempting(text) return alternation([ concatenation([ self.number, self.op_mult, self.expression, ], ignore_whitespace=True), concatenation([ self.expression_terminal, self.op_mult, self.number, option( concatenation([ self.operator, self.expression, ], ignore_whitespace=True) ), ], ignore_whitespace=True), concatenation([ self.expression_terminal, self.op_add, option( concatenation([ self.operator, self.expression, ], ignore_whitespace=True) ), ], ignore_whitespace=True), concatenation([ self.expression_terminal, option( concatenation([ self.operator, self.expression, ], ignore_whitespace=True) ), ], ignore_whitespace=True), ])(text).retyped(TokenType.expression)
def single_string_char(self, text): """single_string_char = "\\" . all_characters - "'" | "\" . "'" | all_characters - "'" ;""" self._attempting(text) return alternation([ concatenation([ "\\\\", exclusion( self.all_characters, "'" ), ], ignore_whitespace=False), concatenation([ "\\", "'", ], ignore_whitespace=False), exclusion( self.all_characters, "'" ), ])(text).retyped(TokenType.single_string_char)
def double_string_char(self, text): """double_string_char = "\\" . all_characters - '"' | "\" . '"' | all_characters - '"' ;""" self._attempting(text) return alternation([ concatenation([ "\\\\", exclusion( self.all_characters, '"' ), ], ignore_whitespace=False), concatenation([ "\\", '"', ], ignore_whitespace=False), exclusion( self.all_characters, '"' ), ])(text).retyped(TokenType.double_string_char)
def string(self, text): """string = '"' . {double_string_char} . '"' | "'" . {single_string_char} . "'" ;""" self._attempting(text) return alternation([ concatenation([ '"', zero_or_more( self.double_string_char, ignore_whitespace=False ), '"', ], ignore_whitespace=False), concatenation([ "'", zero_or_more( self.single_string_char, ignore_whitespace=False ), "'", ], ignore_whitespace=False), ])(text).compressed(TokenType.string)
def comment(self, text): """comment = "/*" . {all_characters - "*" | "*" . all_characters - "/"} . "*/" ;""" self._attempting(text) return concatenation([ "/*", zero_or_more( alternation([ exclusion( self.all_characters, "*" ), concatenation([ "*", exclusion( self.all_characters, "/" ), ], ignore_whitespace=False), ]), ignore_whitespace=False ), "*/", ], ignore_whitespace=False)(text).compressed(TokenType.comment)
def operator(self, text): """operator = "+" | "-" | "*" | "/" | "%" | "^" | "=" | "!=" | ">=" | "<=" | ">" | "<" | "!" | "|" | "&" | "?" ;""" self._attempting(text) return alternation([ "+", "-", "*", "/", "%", "^", "=", "!=", ">=", "<=", ">", "<", "!", "|", "&", "?", ])(text).retyped(TokenType.operator)
def comment(self, text): """comment = "(*" . {printable - "*" | "*" . printable - ")"} . "*)" ;""" self._attempting(text) return concatenation([ "(*", zero_or_more( alternation([ exclusion( self.printable, "*" ), concatenation([ "*", exclusion( self.printable, ")" ), ], ignore_whitespace=False), ]), ignore_whitespace=False ), "*)", ], ignore_whitespace=False)(text).compressed(TokenType.comment)