def define_number(self): """ Return the syntax definition for a number in Arabic Numerals. Override this method to support numeral systems other than Arabic Numerals (0-9). Do not override this method just to change the character used to separate thousands and decimals: Use :attr:`T_THOUSANDS_SEPARATOR` and :attr:`T_DECIMAL_SEPARATOR`, respectively. """ # Defining the basic tokens: to_dot = lambda t: "." to_plus = lambda t: "+" to_minus = lambda t: "-" positive_sign = Literal(self._grammar.get_token("positive_sign")) positive_sign.setParseAction(to_plus) negative_sign = Literal(self._grammar.get_token("negative_sign")) negative_sign.setParseAction(to_minus) decimal_sep = Literal(self._grammar.get_token("decimal_separator")) decimal_sep.setParseAction(to_dot) thousands_sep = Suppress( self._grammar.get_token("thousands_separator")) digits = Word(nums) # Building the integers and decimals: sign = positive_sign | negative_sign thousands = Word(nums, max=3) + \ OneOrMore(thousands_sep + Word(nums, exact=3)) integers = thousands | digits decimals = decimal_sep + digits number = Combine(Optional(sign) + integers + Optional(decimals)) number.setParseAction(self.make_number) number.setName("number") return number
def parser(): global _parser if _parser is None: ParserElement.setDefaultWhitespaceChars("") lbrack, rbrack, lbrace, rbrace, lparen, rparen = map(Literal, "[]{}()") reMacro = Combine("\\" + oneOf(list("dws"))) escapedChar = ~ reMacro + Combine("\\" + oneOf(list(printables))) reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t" reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack) reLiteral = (escapedChar | oneOf(list(reLiteralChar))) reDot = Literal(".") repetition = ( (lbrace + Word(nums).setResultsName("count") + rbrace) | (lbrace + Word(nums).setResultsName("minCount") + "," + Word(nums).setResultsName("maxCount") + rbrace) | oneOf(list("*+?")) ) reRange.setParseAction(handle_range) reLiteral.setParseAction(handle_literal) reMacro.setParseAction(handle_macro) reDot.setParseAction(handle_dot) reTerm = (reLiteral | reRange | reMacro | reDot) reExpr = operatorPrecedence(reTerm, [ (repetition, 1, opAssoc.LEFT, handle_repetition), (None, 2, opAssoc.LEFT, handle_sequence), (Suppress('|'), 2, opAssoc.LEFT, handle_alternative), ]) _parser = reExpr return _parser
def integer_value(): ''' Grammar for integer numbers ''' integer_t = Combine(Optional(MINUS_OP) + Word(nums)) integer_t.setParseAction(lambda t: int(t[0])) return integer_t
def define_number(self): """ Return the syntax definition for a number in Arabic Numerals. Override this method to support numeral systems other than Arabic Numerals (0-9). Do not override this method just to change the character used to separate thousands and decimals: Use :attr:`T_THOUSANDS_SEPARATOR` and :attr:`T_DECIMAL_SEPARATOR`, respectively. """ # Defining the basic tokens: to_dot = lambda t: "." to_plus = lambda t: "+" to_minus = lambda t: "-" positive_sign = Literal(self._grammar.get_token("positive_sign")) positive_sign.setParseAction(to_plus) negative_sign = Literal(self._grammar.get_token("negative_sign")) negative_sign.setParseAction(to_minus) decimal_sep = Literal(self._grammar.get_token("decimal_separator")) decimal_sep.setParseAction(to_dot) thousands_sep = Suppress(self._grammar.get_token("thousands_separator")) digits = Word(nums) # Building the integers and decimals: sign = positive_sign | negative_sign thousands = Word(nums, max=3) + \ OneOrMore(thousands_sep + Word(nums, exact=3)) integers = thousands | digits decimals = decimal_sep + digits number = Combine(Optional(sign) + integers + Optional(decimals)) number.setParseAction(self.make_number) number.setName("number") return number
def _grammar(): from pyparsing import alphas, alphanums, nums from pyparsing import oneOf, Suppress, Optional, Group, ZeroOrMore, NotAny from pyparsing import Forward, operatorPrecedence, opAssoc, Word, White from pyparsing import delimitedList, Combine, Literal, OneOrMore expression = Forward() LPAR, RPAR, DOT, LBRAC, RBRAC = map(Suppress, "().{}") nw = NotAny(White()) identifier = Word(alphas + "_", alphanums + "_") integer = Word(nums) integer.setParseAction(IntegerNode) fractional = Combine(Word('+' + '-' + nums, nums) + '.' + Word(nums)) fractional.setParseAction(FloatNode) literal = fractional | integer arglist = delimitedList(expression) seqrange = LBRAC + expression + Suppress('..') + expression + RBRAC seqrange.setParseAction(lambda t: SequenceNode(start=t[0], stop=t[1])) seqexplicit = LBRAC + Optional(arglist) + RBRAC seqexplicit.setParseAction(lambda t: SequenceNode(lst=t)) sequence = seqrange | seqexplicit rollmod = nw + Group(oneOf("d k r e x") + Optional(integer)) numdice = Optional(integer, default=1) roll = numdice + nw + Suppress("d") + nw + (integer | sequence) roll += Group(ZeroOrMore(rollmod)) roll.setParseAction(DieRollNode) call = LPAR + Group(Optional(arglist)) + RPAR function = identifier + call function.setParseAction(FunctionNode) seqexpr = ((roll | sequence | function) + Group(OneOrMore(DOT + identifier + call))) seqexpr.setParseAction(SeqMethodNode) variable = Word(alphas + "_", alphanums + "_ ") variable.setParseAction(VariableNode) atom = seqexpr | roll | literal | sequence | function | variable expoop = Literal('^') signop = oneOf("+ -") multop = oneOf("* /") plusop = oneOf("+ -") # noinspection PyUnresolvedReferences expression << operatorPrecedence(atom, [ (expoop, 2, opAssoc.LEFT, BinaryOpNode), (signop, 1, opAssoc.RIGHT, UnaryOpNode), (multop, 2, opAssoc.LEFT, BinaryOpNode), (plusop, 2, opAssoc.LEFT, BinaryOpNode), ]) return expression
def func_tokens(dictionary, parse_action): func_name = Word(alphas+'_', alphanums+'_') func_ident = Combine('$' + func_name.copy()('funcname')) func_tok = func_ident + originalTextFor(nestedExpr())('args') func_tok.leaveWhitespace() func_tok.setParseAction(parse_action) func_tok.enablePackrat() rx_tok = Combine(Literal('$').suppress() + Word(nums)('num')) def replace_token(tokens): index = int(tokens.num) return dictionary.get(index, u'') rx_tok.setParseAction(replace_token) strip = lambda s, l, tok: tok[0].strip() text_tok = CharsNotIn(u',').setParseAction(strip) quote_tok = QuotedString('"') if dictionary: arglist = Optional(delimitedList(quote_tok | rx_tok | text_tok)) else: arglist = Optional(delimitedList(quote_tok | text_tok)) return func_tok, arglist, rx_tok
def func_tokens(dictionary, parse_action): func_name = Word(alphas + '_', alphanums + '_') func_ident = Combine('$' + func_name.copy()('funcname')) func_tok = func_ident + originalTextFor(nestedExpr())('args') func_tok.leaveWhitespace() func_tok.setParseAction(parse_action) func_tok.enablePackrat() rx_tok = Combine(Literal('$').suppress() + Word(nums)('num')) def replace_token(tokens): index = int(tokens.num) return dictionary.get(index, '') rx_tok.setParseAction(replace_token) strip = lambda s, l, tok: tok[0].strip() text_tok = CharsNotIn(',').setParseAction(strip) quote_tok = QuotedString('"') if dictionary: arglist = Optional(delimitedList(quote_tok | rx_tok | text_tok)) else: arglist = Optional(delimitedList(quote_tok | text_tok)) return func_tok, arglist, rx_tok
def parser(): global _parser if _parser is None: ParserElement.setDefaultWhitespaceChars("") lbrack, rbrack, lbrace, rbrace, lparen, rparen, colon, qmark = map( Literal, "[]{}():?") reMacro = Combine("\\" + oneOf(list("dws"))) escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables))) reLiteralChar = "".join( c for c in printables if c not in r"\[]{}().*?+|") + " \t" reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack) reLiteral = (escapedChar | oneOf(list(reLiteralChar))) reNonCaptureGroup = Suppress("?:") reDot = Literal(".") repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) | (lbrace + Word(nums).setResultsName("minCount") + "," + Word(nums).setResultsName("maxCount") + rbrace) | oneOf(list("*+?"))) reRange.setParseAction(handleRange) reLiteral.setParseAction(handleLiteral) reMacro.setParseAction(handleMacro) reDot.setParseAction(handleDot) reTerm = (reLiteral | reRange | reMacro | reDot | reNonCaptureGroup) reExpr = infixNotation(reTerm, [ (repetition, 1, opAssoc.LEFT, handleRepetition), (None, 2, opAssoc.LEFT, handleSequence), (Suppress('|'), 2, opAssoc.LEFT, handleAlternative), ]) _parser = reExpr return _parser
def __init__(self): # speed up infixNotation considerably at the price of some cache memory ParserElement.enablePackrat() boolean = Keyword('True') | Keyword('False') none = Keyword('None') integer = Word(nums) real = Combine(Word(nums) + "." + Word(nums)) string = (QuotedString('"', escChar='\\') | QuotedString("'", escChar='\\')) regex = QuotedString('/', escChar='\\') identifier = Word(alphas, alphanums + '_') dereference = infixNotation(identifier, [ (Literal('.'), 2, opAssoc.LEFT, EvalArith), ]) result = (Keyword('bad') | Keyword('fail') | Keyword('good') | Keyword('ignore') | Keyword('unknown')) rval = boolean | none | real | integer | string | regex | result | dereference rvallist = Group( Suppress('[') + Optional(delimitedList(rval)) + Suppress(']')) rvalset = Group( Suppress('{') + Optional(delimitedList(rval)) + Suppress('}')) operand = rval | rvallist | rvalset # parse actions replace the parsed tokens with an instantiated object # which we can later call into for evaluation of its content boolean.setParseAction(EvalBoolean) none.setParseAction(EvalNone) integer.setParseAction(EvalInteger) real.setParseAction(EvalReal) string.setParseAction(EvalString) regex.setParseAction(EvalRegex) identifier.setParseAction(EvalIdentifier) result.setParseAction(EvalResult) rvallist.setParseAction(EvalList) rvalset.setParseAction(EvalSet) identity_test = Keyword('is') + ~Keyword('not') | Combine( Keyword('is') + Keyword('not'), adjacent=False, joinString=' ') membership_test = Keyword('in') | Combine( Keyword('not') + Keyword('in'), adjacent=False, joinString=' ') comparison_op = oneOf('< <= > >= != == isdisjoint') comparison = identity_test | membership_test | comparison_op self.parser = infixNotation(operand, [ (Literal('**'), 2, opAssoc.LEFT, EvalPower), (oneOf('+ - ~'), 1, opAssoc.RIGHT, EvalModifier), (oneOf('* / // %'), 2, opAssoc.LEFT, EvalArith), (oneOf('+ -'), 2, opAssoc.LEFT, EvalArith), (oneOf('<< >>'), 2, opAssoc.LEFT, EvalArith), (Literal('&'), 2, opAssoc.LEFT, EvalArith), (Literal('^'), 2, opAssoc.LEFT, EvalArith), (Literal('|'), 2, opAssoc.LEFT, EvalArith), (comparison, 2, opAssoc.LEFT, EvalLogic), (Keyword('not'), 1, opAssoc.RIGHT, EvalModifier), (Keyword('and'), 2, opAssoc.LEFT, EvalLogic), (Keyword('or'), 2, opAssoc.LEFT, EvalLogic), (Keyword('->'), 2, opAssoc.LEFT, EvalArith), ])
def float_value(): ''' Grammar for float values ''' from grammar.symbols import DOT float_t = Combine(Optional(MINUS_OP) + Word(nums) + DOT + Word(nums)) float_t.setParseAction(lambda t: float(t[0])) return float_t
def _build_filter_parser(field_names): field = _build_field_expr(field_names) negation = CaselessKeyword('not') negation.setParseAction(lambda x: Negation(x[0])) comparison_operator = Group( Keyword('=') ^ Keyword('!=') ^ Keyword('>=') ^ Keyword('<=') ^ Keyword('<') ^ Keyword('>') ^ Keyword('lte') # match before lt ^ Keyword('lt') ^ Keyword('gte') # match before gt ^ Keyword('gt') ^ (Optional(negation) + (CaselessKeyword('contains') ^ CaselessKeyword('icontains') ^ CaselessKeyword('startswith') ^ CaselessKeyword('istartswith') ^ CaselessKeyword('endswith') ^ CaselessKeyword('iendswith') ^ CaselessKeyword('eq')))) comparison_operator.setParseAction(lambda x: Operator(x)) single_value_operator = Group( CaselessKeyword('isnull') ^ (Optional(negation) + CaselessKeyword('isnull'))) single_value_operator.setParseAction(lambda x: Operator(x)) plusorminus = Literal('+') | Literal('-') num_integer = Combine(Optional(plusorminus) + Word(nums)) num_integer.setParseAction(lambda x: Integer(x[0])) num_float = Combine(Optional(plusorminus) + Word(nums) + '.' + Word(nums)) num_float.setParseAction(lambda x: Float(x[0])) quoted_string = (QuotedString("'") ^ QuotedString('"')) quoted_string.setParseAction(lambda x: String(x[0])) boolean = Or([ CaselessKeyword(v) for v in BOOLEAN_TRUE_VALUES + BOOLEAN_FALSE_VALUES ]) boolean.setParseAction(lambda x: Boolean(x[0])) value = (quoted_string ^ num_integer ^ boolean) comparison = Group((field + comparison_operator + value) ^ (value + comparison_operator + field) ^ (field + comparison_operator + field) ^ (field + single_value_operator)) comparison.setParseAction(lambda x: Comparison(x)) invalid_comparison = Group( (value + comparison_operator + value).setParseAction(lambda x: fail( "Value may not be compared with values: {0}".format(' '.join(x))))) logical_op = Group(CaselessKeyword("and") | CaselessKeyword("or")) logical_op.setParseAction(lambda x: LogicalOp(x[0][0])) statement = Optional(comparison | invalid_comparison) + ZeroOrMore( logical_op + (comparison | invalid_comparison)) return statement
def _BNF(self): base16 = Literal("$") hex = Combine(base16 + Word(hexnums + "_")) base4 = Literal("%%") quaternary = Combine(base4 + Word("0123_")) base2 = Literal("%") binary = Combine(base2 + Word("01_")) plusminus = Literal("+") | Literal("-") integer = Combine(Optional(plusminus) + Word(nums+"_")) name_token = Combine(Optional(Literal(":") | Literal("@")) + Word("_" + alphas, "_" + alphanums)) name_token.setParseAction(self._mark_name_token) lparens = Literal("(").suppress() rparens = Literal(")").suppress() # op0 = Literal("@") op1 = (Literal("^^") | Literal("||") | Literal("|<") | Literal(">|") | Literal("!")).setParseAction(self._mark_unary) op2 = Literal("->") | Literal("<-") | Literal(">>") | Literal("<<") | Literal("~>") | Literal("><") op3 = Literal("&") op4 = Literal("|") | Literal("^") op5 = Literal("**") | Literal("*") | Literal("//") | Literal("/") op6 = Literal("+") | Literal("-") op7 = Literal("#>") | Literal("<#") op8 = Literal("<") | Literal(">") | Literal("<>") | Literal("==") | Literal("=<") | Literal("=>") op9 = Literal("NOT").setParseAction(self._mark_unary) op10 = Literal("AND") op11 = Literal("OR") op12 = Literal(",") expr = Forward() atom = name_token | hex | quaternary | binary | integer | quotedString atom.setParseAction(self._push) atom = atom | (lparens + expr.suppress() + rparens) # term0 = atom + ZeroOrMore((op0 + atom) .setParseAction(self._push)) # term1 = term0 + ZeroOrMore((op1 + term0) .setParseAction(self._push)) term1 = atom + ZeroOrMore((op1 + atom) .setParseAction(self._push)) term2 = term1 + ZeroOrMore((op2 + term1) .setParseAction(self._push)) term3 = term2 + ZeroOrMore((op3 + term2) .setParseAction(self._push)) term4 = term3 + ZeroOrMore((op4 + term3) .setParseAction(self._push)) term5 = term4 + ZeroOrMore((op5 + term4) .setParseAction(self._push)) term6 = term5 + ZeroOrMore((op6 + term5) .setParseAction(self._push)) term7 = term6 + ZeroOrMore((op7 + term6) .setParseAction(self._push)) term8 = term7 + ZeroOrMore((op8 + term7) .setParseAction(self._push)) term9 = term8 + ZeroOrMore((op9 + term8) .setParseAction(self._push)) term10 = term9 + ZeroOrMore((op10 + term9) .setParseAction(self._push)) term11 = term10 + ZeroOrMore((op11 + term10).setParseAction(self._push)) expr << term11 + ZeroOrMore((op12 + term11).setParseAction(self._push)) return expr
def parseString(self): """ Parse string. Initial attempt at grammar, needs restructuring. ANDop :: 'AND' ORop :: 'OR' lpar :: '(' rpar :: ')' keyword :: alphas result :: alphanums word :: alphanums hashtag :: COMBINE('#' word) cashtag :: COMBINE ('$' word) atom :: COMBINE('-' atom) | word | COMBINE(keyword ':' result) | lpar expr rpar | hashtag | cashtag term :: atom [ ORop atom ]* expr :: term [ ANDop term ]* """ if self.bnf is None: ANDop = Keyword('AND') ORop = Keyword('OR') colon=Literal(':') word = ~colon + ~ORop+ ~ANDop + Word(alphanums,min=1) keyword = ~colon + ~ORop + ~ANDop + Word(alphas,min=1) quoted = quotedString hashtag = Combine(Literal('#')+Word(alphanums)) cashtag = Combine(Literal('$')+Word(alphanums)) minus = Literal('-') lpar, rpar = map(Suppress, "()") expr = Forward() atom = Forward() atom <<= (minus[...].setParseAction(self.push_first)+ (Group(lpar + expr + rpar) | Group(keyword.setResultsName("keyword") + colon + word.setResultsName("value")).setParseAction(self.push_first) | hashtag.setParseAction(self.push_first) | cashtag.setParseAction(self.push_first) | Group(quoted.setResultsName('quote')).setParseAction(self.push_first) | word.setParseAction(self.push_first).setResultsName('word')[1] ) ).setParseAction(self.push_unary_minus) # AND is optional word1 word2 implicitly is word1 AND word2 term = Forward() term <<= atom + (ORop + atom).setParseAction(self.push_first)[...] expr <<= term + (Optional(ANDop) + term).setParseAction(self.push_and)[...] self.bnf = expr self.parsed_str=self.bnf.parseString(self.query_str, parseAll=True)
def getkw_bnf(self): sect_begin = Literal("{").suppress() sect_end = Literal("}").suppress() array_begin = Literal("[").suppress() array_end = Literal("]").suppress() tag_begin = Literal("<").suppress() tag_end = Literal(">").suppress() eql = Literal("=").suppress() dmark = Literal('$').suppress() end_data=Literal('$end').suppress() prtable = alphanums+r'!$%&*+-./<>?@^_|~' ival=Regex('[-]?\d+') dval=Regex('-?\d+\.\d*([eE]?[+-]?\d+)?') lval=Regex('([Yy]es|[Nn]o|[Tt]rue|[Ff]alse|[Oo]n|[Oo]ff)') # Helper definitions kstr= quotedString.setParseAction(removeQuotes) ^ \ dval ^ ival ^ lval ^ Word(prtable) name = Word(alphas+"_",alphanums+"_") vec=array_begin+delimitedList(dval ^ ival ^ lval ^ Word(prtable) ^ \ Literal("\n").suppress() ^ \ quotedString.setParseAction(removeQuotes))+array_end sect=name+sect_begin tag_sect=name+Group(tag_begin+name+tag_end)+sect_begin # Grammar keyword = name + eql + kstr vector = name + eql + vec data=Combine(dmark+name)+SkipTo(end_data)+end_data section=Forward() sect_def=(sect | tag_sect ) #| vec_sect) input=section | data | vector | keyword section << sect_def+ZeroOrMore(input) + sect_end # Parsing actions ival.setParseAction(self.conv_ival) dval.setParseAction(self.conv_dval) lval.setParseAction(self.conv_lval) keyword.setParseAction(self.store_key) vector.setParseAction(self.store_vector) data.setParseAction(self.store_data) sect.setParseAction(self.add_sect) tag_sect.setParseAction(self.add_sect) sect_end.setParseAction(self.pop_sect) bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error) bnf.ignore(pythonStyleComment) return bnf
def rhs_value_p(): Ipv4Address = Combine(Word(nums) + ('.' + Word(nums)) * 3).setResultsName('ipv4') Ipv4Address = Ipv4Address.setParseAction(lambda s, l, toks: toks[0]) Int = Word(nums) Int = Int.setParseAction(lambda s, l, toks: int(toks[0])) Float = Combine(Word(nums) + '.' + Word(nums)).setResultsName('float') Float = Float.setParseAction(lambda s, l, toks: float(toks[0])) String = quotedString.copy().addParseAction(pyparsing.removeQuotes) rhs = pyparsing.Or([String, Int, Float, Ipv4Address]) return rhs
def grammar(self): """ Define the parser grammar. """ # Ignore TeX commands between delimiters $$, \(, \) tex_eq = (Literal(r'\(') | Literal(r'$$') | Literal(r'\[')) + ... + ( Literal(r'\)') | Literal(r'$$') | Literal(r'\]')) # Define elemtary stuff leftAc = Literal('{').suppress() rightAc = Literal('}').suppress() lpar = Literal('(') rpar = Literal(')') integer = Word(nums) # simple unsigned integer real = Regex(r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?") real.setParseAction(self.real_hook) number = real | integer # Define function fnname = Word(alphas, alphanums + "_")('name') # Require expr to finalize the def function = Forward() function.setParseAction(self.function_hook) # What are the namming rule for the jocker? Need to start by a letter, # may contain almost everything variable = Combine(leftAc + Word(alphas, alphanums + "_") + rightAc)('name') variable.setParseAction(self.variable_hook) variable.ignore(tex_eq) # arithmetic operators minus = Literal('-') arithOp = oneOf("+ * /") | minus equal = Literal('{=').suppress() # Require atom to finalize the def expr = Forward() # Define atom atom = number | (0, None) * minus + (Group(lpar + expr + rpar) | variable | function) atom.setParseAction(self.atom_hook) # Finalize postponed elements... expr << atom + ZeroOrMore(arithOp + atom) # Need to group arguments for swapping them function << fnname + Group(lpar + Group(ZeroOrMore(expr)) + Optional(Literal(',') + Group(expr)) + rpar) # Define equation equation = equal + expr + rightAc equation.setParseAction(self.equation_hook) return equation, variable
def parser(): global _parser if _parser is None: ParserElement.setDefaultWhitespaceChars("") lbrack = Literal("[") rbrack = Literal("]") lbrace = Literal("{") rbrace = Literal("}") lparen = Literal("(") rparen = Literal(")") reMacro = Suppress("\\") + oneOf(list("dwsZ")) escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables))) reLiteralChar = "".join(c for c in string.printable if c not in r"\[]{}().*?+|") reRange = Combine(lbrack.suppress() + SkipTo(rbrack, ignore=escapedChar) + rbrack.suppress()) reLiteral = (escapedChar | oneOf(list(reLiteralChar))) reDot = Literal(".") repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) | (lbrace + Word(nums).setResultsName("minCount") + "," + Word(nums).setResultsName("maxCount") + rbrace) | oneOf(list("*+?"))) reExpr = Forward() reGroup = (lparen.suppress() + Optional(Literal("?").suppress() + oneOf(list(":P"))).setResultsName("option") + reExpr.setResultsName("expr") + rparen.suppress()) reTerm = (reLiteral | reRange | reMacro | reDot | reGroup) reExpr << operatorPrecedence(reTerm, [ (repetition, 1, opAssoc.LEFT, create(Repetition)), (None, 2, opAssoc.LEFT, create(Sequence)), (Suppress('|'), 2, opAssoc.LEFT, create(Alternation)), ]) reGroup.setParseAction(create(Group)) reRange.setParseAction(create(Range)) reLiteral.setParseAction(create(Character)) reMacro.setParseAction(create(Macro)) reDot.setParseAction(create(Dot)) _parser = reExpr return _parser
def parser(): global _parser if _parser is None: ParserElement.setDefaultWhitespaceChars("") lbrack = Literal("[") rbrack = Literal("]") lbrace = Literal("{") rbrace = Literal("}") lparen = Literal("(") rparen = Literal(")") reMacro = Suppress("\\") + oneOf(list("dwsZ")) escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables))) reLiteralChar = "".join(c for c in string.printable if c not in r"\[]{}().*?+|") reRange = Combine(lbrack.suppress() + SkipTo(rbrack,ignore=escapedChar) + rbrack.suppress()) reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) ) reDot = Literal(".") repetition = ( ( lbrace + Word(nums).setResultsName("count") + rbrace ) | ( lbrace + Word(nums).setResultsName("minCount")+","+ Word(nums).setResultsName("maxCount") + rbrace ) | oneOf(list("*+?")) ) reExpr = Forward() reGroup = (lparen.suppress() + Optional(Literal("?").suppress() + oneOf(list(":P"))).setResultsName("option") + reExpr.setResultsName("expr") + rparen.suppress()) reTerm = ( reLiteral | reRange | reMacro | reDot | reGroup ) reExpr << operatorPrecedence( reTerm, [ (repetition, 1, opAssoc.LEFT, create(Repetition)), (None, 2, opAssoc.LEFT, create(Sequence)), (Suppress('|'), 2, opAssoc.LEFT, create(Alternation)), ] ) reGroup.setParseAction(create(Group)) reRange.setParseAction(create(Range)) reLiteral.setParseAction(create(Character)) reMacro.setParseAction(create(Macro)) reDot.setParseAction(create(Dot)) _parser = reExpr return _parser
def getkw_bnf(self): lcb = Literal("{").suppress() rcb = Literal("}").suppress() lsb = Literal("[").suppress() rsb = Literal("]").suppress() lps = Literal("(").suppress() rps = Literal(")").suppress() eql = Literal("=").suppress() dmark = Literal('$').suppress() end_sect = rcb end_data = Literal('$end').suppress() prtable = srange("[0-9a-zA-Z]") + '!$%&*+-./<>?@^_|~:' kstr = Word(prtable) ^ quotedString.setParseAction(removeQuotes) name = Word(alphas + "_", alphanums + "_") vec=lsb+delimitedList(Word(prtable) ^ Literal("\n").suppress() ^\ quotedString.setParseAction(removeQuotes))+rsb key = kstr ^ vec keyword = name + eql + kstr vector = name + eql + vec data = Combine(dmark + name) + SkipTo(end_data) + end_data data.setParseAction(self.store_data) sect = name + lcb sect.setParseAction(self.add_sect) key_sect = name + Group(lps + kstr + rps) + lcb key_sect.setParseAction(self.add_sect) vec_sect = name + Group(lps + vec + rps) + lcb vec_sect.setParseAction(self.add_vecsect) end_sect.setParseAction(self.pop_sect) keyword.setParseAction(self.store_key) vector.setParseAction(self.store_vector) section = Forward() input = section ^ data ^ keyword ^ vector sectdef = sect ^ key_sect ^ vec_sect section << sectdef + ZeroOrMore(input) + rcb bnf = ZeroOrMore(input) bnf.ignore(pythonStyleComment) return bnf
def grammar(self, tabilify=True): number = Combine(Word(nums) + Optional("." + OneOrMore(Word(nums)))) table = Combine(Word(alphas) + OneOrMore(Word("_"+alphanums))) if tabilify: number.setParseAction(self._number_parse_action) table.setParseAction(self._table_parse_action) signop = oneOf('+ -') multop = oneOf('* /') plusop = oneOf('+ -') operand = number | table return operatorPrecedence(operand, [ (signop, 1, opAssoc.RIGHT), (multop, 2, opAssoc.LEFT), (plusop, 2, opAssoc.LEFT) ])
def grammar(self): number = Combine(Word(nums) + Optional("." + OneOrMore(Word(nums)))) table = Combine(Word(alphas) + OneOrMore(Word("_"+alphanums))) number.setParseAction(self._number_parse_action) table.setParseAction(self._table_parse_action) signop = oneOf('+ -') multop = oneOf('* /') plusop = oneOf('+ -') operand = number | table return operatorPrecedence(operand, [ (signop, 1, opAssoc.RIGHT), (multop, 2, opAssoc.LEFT), (plusop, 2, opAssoc.LEFT) ])
def getkw_bnf(self): sect_begin = Literal("{").suppress() sect_end = Literal("}").suppress() array_begin = Literal("[").suppress() array_end = Literal("]").suppress() arg_begin = Literal("(").suppress() arg_end = Literal(")").suppress() eql = Literal("=").suppress() dmark = Literal('$').suppress() end_data=Literal('$end').suppress() prtable = alphanums+r'!$%&*+-./<>?@^_|~' # Helper definitions kstr=Word(prtable) ^ quotedString.setParseAction(removeQuotes) name = Word(alphas+"_",alphanums+"_") vec=array_begin+delimitedList(Word(prtable) ^ \ Literal("\n").suppress() ^ \ quotedString.setParseAction(removeQuotes))+array_end sect=name+sect_begin key_sect=name+Group(arg_begin+kstr+arg_end)+sect_begin vec_sect=name+Group(arg_begin+vec+ arg_end)+sect_begin # Grammar keyword = name + eql + kstr vector = name + eql + vec data=Combine(dmark+name)+SkipTo(end_data)+end_data section=Forward() sect_def=(sect | key_sect | vec_sect) input=section | data | vector | keyword section << sect_def+ZeroOrMore(input) + sect_end # Parsing actions keyword.setParseAction(self.store_key) vector.setParseAction(self.store_vector) data.setParseAction(self.store_data) sect.setParseAction(self.add_sect) key_sect.setParseAction(self.add_sect) vec_sect.setParseAction(self.add_vecsect) sect_end.setParseAction(self.pop_sect) bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error) bnf.ignore(pythonStyleComment) return bnf
def _build_grammar(self): expr = Forward() float_lit = Combine(Word(nums) + '.' + Word(nums)) float_lit.setName('float') float_lit.setParseAction(lambda x: \ self.to_literal(float(x[0]))) int_lit = Word(nums) int_lit.setName('int') int_lit.setParseAction(lambda x: \ self.to_literal(int(x[0]))) num = (float_lit | int_lit) num.setParseAction(lambda x: x[0]) tag_name = Word(alphas + "_", alphanums + "_") tag_name.setName('tag_name') tag_name.setParseAction(lambda t: tag_reference.TagReference(t[0])) quoted_string = QuotedString("'") quoted_string.setParseAction(lambda s: self.to_literal(s[0])) oper = oneOf('+ * / -') oper.setParseAction(lambda o: o[0]) lpar = Literal("(").suppress() rpar = Literal(")").suppress() arith = Group(lpar + expr + oper + expr + rpar) arith.setParseAction(lambda t: \ self.to_arith(t[0][0], t[0][1], t[0][2])) assign = tag_name + '=' + expr assign.setName('assign') assign.setParseAction(lambda x: self.to_assign(x[0],x[2])) print_tags = Literal('?') print_tags.setParseAction(lambda x: self.to_print_tags()) expr <<(arith|assign|tag_name|num|quoted_string|print_tags) expr.setParseAction(lambda x: x[0]) return expr
def create_bnf(): cvt_int = lambda toks: int(toks[0]) cvt_real = lambda toks: float(toks[0]) cvt_tuple = lambda toks : tuple(toks.asList()) cvt_dict = lambda toks: dict(toks.asList()) # define punctuation as suppressed literals (lparen, rparen, lbrack, rbrack, lbrace, rbrace, colon) = map(Suppress,"()[]{}:") integer = Combine(Optional(oneOf("+ -")) + Word(nums)).setName("integer") integer.setParseAction(cvt_int) real = Combine(Optional(oneOf("+ -"))+ Word(nums) + "." + Optional(Word(nums)) + Optional("e" + Optional(oneOf("+ -")) + Word(nums))).setName("real") real.setParseAction(cvt_real) tuple_str = Forward() list_str = Forward() dict_str = Forward() list_item = (real | integer | Group(list_str) | tuple_str | dict_str | quotedString.setParseAction(removeQuotes) | Word(alphas8bit + alphas, alphas8bit + alphanums + "_")) list_item2 = list_item | Empty().setParseAction(lambda: [None]) tuple_str << (Suppress("(") + Optional(delimitedList(list_item)) + Optional(Suppress(",")) + Suppress(")")) tuple_str.setParseAction(cvt_tuple) list_str << (lbrack + Optional(delimitedList(list_item) + Optional(Suppress(","))) + rbrack) dict_entry = Group(list_item + colon + list_item2) dict_inner = delimitedList(dict_entry) + Optional(Suppress(",")) dict_inner.setParseAction(cvt_dict) dict_str << (lbrace + Optional(dict_inner) + rbrace) return dict_inner
def numeric_literal() -> Token: numeral = Combine( Word(nums) + ZeroOrMore(Optional(Word("_")) + Word(nums))) numeral.setParseAction(lambda t: t[0].replace("_", "")) decimal_literal = Group(numeral) decimal_literal.setParseAction(lambda t: (int(t[0][0]), 0)) extended_digit = Word(nums + "ABCDEF") based_numeral = Combine(extended_digit + ZeroOrMore(Optional("_") + extended_digit)) based_numeral.setParseAction(lambda t: t[0].replace("_", "")) based_literal = numeral + Literal("#") - based_numeral - Literal("#") based_literal.setParseAction(lambda t: (int(t[2], int(t[0])), int(t[0]))) num_literal = based_literal | decimal_literal num_literal.setName("Number") return locatedExpr(num_literal).setParseAction(lambda s, l, t: Number( t[0][1][0], t[0][1][1], parser_location(t[0][0], t[0][2], s)))
def num_parser(): """A floating point number parser """ T_DOT = Literal(".") T_MIN = Literal("-") T_PLU = Literal("+") T_EXP = Literal("e") num = Combine( Optional(T_MIN) + Word(nums) + Optional(T_DOT + Word(nums)) + Optional(T_EXP + Optional(T_MIN ^ T_PLU) + Word(nums))) num = num.setParseAction(lambda t: float(t[0])) return num
def pattern(): """pyparsing pattern """ def attachLocation(s, loc, tocs): """pyparsing callback. Saves path position in the original string """ return [(loc, tocs[0])] path = CharsNotIn(" \t")("path") path.setParseAction(attachLocation) longPath = CharsNotIn(" \t", min=2)("path") longPath.setParseAction(attachLocation) slashPath = Combine(Literal('/') + Optional(CharsNotIn(" \t")))("path") slashPath.setParseAction(attachLocation) pat = ((Literal('f ') + Optional(White()) + Optional(path)) ^ longPath ^ slashPath) + \ Optional(White() + Word(nums)("line")) pat.leaveWhitespace() pat.setParseAction(CommandOpen.create) return pat
def pattern(): """pyparsing pattern """ def attachLocation(s, loc, tocs): """pyparsing callback. Saves path position in the original string """ return [(loc, tocs[0])] from pyparsing import CharsNotIn, Combine, Literal, Optional, White, Word, nums # delayed import, performance optimization path = CharsNotIn(" \t")("path") path.setParseAction(attachLocation) longPath = CharsNotIn(" \t", min=2)("path") longPath.setParseAction(attachLocation) slashPath = Combine(Literal('/') + Optional(CharsNotIn(" \t")))("path") slashPath.setParseAction(attachLocation) pat = ((Literal('f ') + Optional(White()) + Optional(path)) ^ longPath ^ slashPath) + \ Optional(White() + Word(nums)("line")) pat.leaveWhitespace() pat.setParseAction(CommandOpen.create) return pat
def expr(self) -> ParserElement: MENTION = Combine( "[" + Optional( SkipTo("|", failOn="]") + Suppress("|"), default="", ) + "~" + Optional(CaselessLiteral("accountid:")) + Word(alphanums + ":-").setResultsName("accountid") + "]", ) return ((StringStart() | Optional(PrecededBy(White(), retreat=1), default=" ")) + MENTION.setParseAction(self.action) + (StringEnd() | Optional(FollowedBy( White() | Char(punctuation, excludeChars="[") | MENTION), default=" ")))
def expr(self) -> ParserElement: NON_ALPHANUMS = Regex(r"\W", flags=re.UNICODE) TOKEN = Suppress(self.TOKEN) IGNORE = White() + TOKEN | self.get_ignore_expr() ELEMENT = Combine( TOKEN + (~White() & ~Char(self.TOKEN)) + SkipTo(TOKEN, ignore=IGNORE, failOn="\n") + TOKEN + FollowedBy(NON_ALPHANUMS | StringEnd()), ) return (StringStart() | PrecededBy(NON_ALPHANUMS, retreat=1)) + Combine( ELEMENT.setParseAction(self.action) + Optional(~ELEMENT, default=" "), )
def define_math(self): digits = Word(nums) variable = Word(alphas + self._grammar.get_token("namespace_separator")) to_dot = lambda t: "." decimal_sep = Literal(self._grammar.get_token("decimal_separator")) decimal_sep.setParseAction(to_dot) thousands_sep = Suppress( self._grammar.get_token("thousands_separator")) thousands = Word( nums, max=3) + OneOrMore(thousands_sep + Word(nums, exact=3)) integers = thousands | digits decimals = decimal_sep + digits expop = Literal('^') signop = oneOf('+ -') multop = oneOf('* /') plusop = oneOf('+ -') factop = Literal('!') modop = Literal('%') operand = Combine((integers + Optional(decimals)) | variable) expr = operatorPrecedence(operand, [ (Literal("["), 1, opAssoc.RIGHT), (Literal("]"), 1, opAssoc.LEFT), ("!", 1, opAssoc.LEFT), (expop, 2, opAssoc.RIGHT), (signop, 1, opAssoc.RIGHT), (multop, 2, opAssoc.LEFT), (modop, 2, opAssoc.LEFT), (plusop, 2, opAssoc.LEFT), ]) expr = Combine(expr) expr.setParseAction(self.make_arithmetic) expr.setName("arithmetic") return expr
def expr(self) -> ParserElement: INTENSITY = Word(nums) ALPHA = Word(nums + ".") SEP = "," + Optional(White()) RGBA = (CaselessLiteral("rgba(") + INTENSITY.setResultsName("red") + SEP + INTENSITY.setResultsName("green") + SEP + INTENSITY.setResultsName("blue") + SEP + ALPHA + ")") COLOR = Word("#", hexnums) ^ Word(alphas) ^ RGBA expr = Combine( "{color:" + COLOR.setResultsName("color") + "}" + SkipTo("{color}").setResultsName("text") + "{color}", ) return expr.setParseAction(self.action)
def parser(): global _parser if _parser is None: ParserElement.setDefaultWhitespaceChars("") lbrack,rbrack,lbrace,rbrace,lparen,rparen,colon,qmark = map(Literal,"[]{}():?") reMacro = Combine("\\" + oneOf(list("dws"))) escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables))) reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t" reRange = Combine(lbrack + SkipTo(rbrack,ignore=escapedChar) + rbrack) reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) ) reNonCaptureGroup = Suppress("?:") reDot = Literal(".") repetition = ( ( lbrace + Word(nums)("count") + rbrace ) | ( lbrace + Word(nums)("minCount")+","+ Word(nums)("maxCount") + rbrace ) | oneOf(list("*+?")) ) reRange.setParseAction(handleRange) reLiteral.setParseAction(handleLiteral) reMacro.setParseAction(handleMacro) reDot.setParseAction(handleDot) reTerm = ( reLiteral | reRange | reMacro | reDot | reNonCaptureGroup) reExpr = infixNotation( reTerm, [ (repetition, 1, opAssoc.LEFT, handleRepetition), (None, 2, opAssoc.LEFT, handleSequence), (Suppress('|'), 2, opAssoc.LEFT, handleAlternative), ] ) _parser = reExpr return _parser
def _simple_number(): s = Combine(optional_sign + usn) s.setParseAction(lambda s, l, tok: SimpleNumber(tok[0])) return s
raise ParseException(loc, "invalid float format %s"%toks[0]) exponent = CaselessLiteral("e")+Optional(sign)+Word(nums) #note that almost all these fields are optional, #and this can match almost anything. We rely on Pythons built-in #float() function to clear out invalid values - loosely matching like this #speeds up parsing quite a lot floatingPointConstant = Combine( Optional(sign) + Optional(Word(nums)) + Optional(Literal(".") + Optional(Word(nums)))+ Optional(exponent) ) floatingPointConstant.setParseAction(convertToFloat) number = floatingPointConstant #same as FP constant but don't allow a - sign nonnegativeNumber = Combine( Optional(Word(nums)) + Optional(Literal(".") + Optional(Word(nums)))+ Optional(exponent) ) nonnegativeNumber.setParseAction(convertToFloat) coordinate = number #comma or whitespace can seperate values all over the place in SVG maybeComma = Optional(Literal(',')).suppress()
#!/usr/bin/env python3 from pyparsing import Word, alphas, nums, Combine, oneOf # Identifier ident = Combine(oneOf("me you") + '.' + Word(alphas)) # ident.setParseAction(lambda s,l,t: [ eval(t[0]) ]) # literal values val = Combine(Word(nums) + '.' + Word(nums)) |\ Word(nums) val.setParseAction(lambda s, l, t: [float(t[0])]) val.setName("value") # Arithmetic expression expr = val + '+' + val |\ val + '-' + val |\ val + '*' + val |\ val + '/' + val |\ val # Combinations of arithmetic expressions exprs = expr + '+' + expr |\ expr + '-' + expr |\ expr + '*' + expr |\ expr + '/' + expr |\ expr # Conditional expressions c_expr = ident + '&' + ident |\ ident + '|' + ident |\
return query NO_BRTS = printables.replace('(', '').replace(')', '') SINGLE = Word(NO_BRTS.replace('*', '')) WILDCARDS = Optional('*') + SINGLE + Optional('*') + WordEnd(wordChars=NO_BRTS) QUOTED = quotedString.setParseAction(removeQuotes) OPER_AND = CaselessLiteral('and') OPER_OR = CaselessLiteral('or') OPER_NOT = '-' TERM = Combine(Optional(Word(alphas).setResultsName('meta') + ':') + (QUOTED.setResultsName('query') | WILDCARDS.setResultsName('query'))) TERM.setParseAction(createQ) EXPRESSION = operatorPrecedence(TERM, [ (OPER_NOT, 1, opAssoc.RIGHT), (OPER_OR, 2, opAssoc.LEFT), (Optional(OPER_AND, default='and'), 2, opAssoc.LEFT)]) EXPRESSION.setParseAction(unionQ) QUERY = OneOrMore(EXPRESSION) + StringEnd() QUERY.setParseAction(unionQ) def advanced_search(pattern): """Parse the grammar of a pattern and build a queryset with it""" query_parsed = QUERY.parseString(pattern)
def transform_human(text, variables=None): """Transform user input with given context. Args: text (str): User input. variables (dict): Variables for purposes of substitution. Returns: A 2-tuple of: (A human-readable script that Script can parse, A list of contextual information for tooltips, etc.) """ if variables is None: variables = {} # No mutable default value. # these are parseActions for pyparsing. def str_literal_to_hex(s, loc, toks): for i, t in enumerate(toks): toks[i] = ''.join(['0x', t.encode('hex')]) return toks def var_name_to_value(s, loc, toks): for i, t in enumerate(toks): val = variables.get(t[1:]) if val: toks[i] = val return toks def implicit_opcode_to_explicit(s, loc, toks): """Add "OP_" prefix to an opcode.""" for i, t in enumerate(toks): toks[i] = '_'.join(['OP', t]) return toks def hex_to_formatted_hex(s, loc, toks): """Add "0x" prefix and ensure even length.""" for i, t in enumerate(toks): new_tok = t # Add '0x' prefix if not t.startswith('0x'): if t.startswith('x'): new_tok = ''.join(['0', t]) else: new_tok = ''.join(['0x', t]) # Even-length string if len(new_tok) % 2 != 0: new_tok = ''.join([new_tok[0:2], '0', new_tok[2:]]) toks[i] = new_tok return toks # ^ parseActions for pyparsing end here. str_literal = QuotedString('"') str_literal.setParseAction(str_literal_to_hex) var_name = Combine(Word('$') + Word(pyparsing.alphas)) var_name.setParseAction(var_name_to_value) # Here we populate the list of contextual tips. # Explicit opcode names op_names = [str(i) for i in OPCODE_NAMES.keys()] op_names_explicit = ' '.join(op_names) def is_small_int(op): """True if op is one of OP_1, OP_2, ...OP_16""" try: i = int(op[3:]) return True except ValueError: return False op_names_implicit = ' '.join([i[3:] for i in op_names if not is_small_int(i)]) # Hex, implicit (e.g. 'a') and explicit (e.g. '0x0a') explicit_hex = Combine(Word('0x') + Word(pyparsing.hexnums) + pyparsing.WordEnd()) implicit_hex = Combine(pyparsing.WordStart() + OneOrMore(Word(pyparsing.hexnums)) + pyparsing.WordEnd()) explicit_hex.setParseAction(hex_to_formatted_hex) implicit_hex.setParseAction(hex_to_formatted_hex) # Opcodes, implicit (e.g. 'ADD') and explicit (e.g. 'OP_ADD') explicit_op = pyparsing.oneOf(op_names_explicit) implicit_op = Combine(pyparsing.WordStart() + pyparsing.oneOf(op_names_implicit)) implicit_op.setParseAction(implicit_opcode_to_explicit) contexts = pyparsing.Optional(var_name('Variable') | str_literal('String literal') | explicit_op('Opcode') | implicit_op('Opcode') | explicit_hex('Hex') | implicit_hex('Hex')) matches = [(i[0].asDict(), i[1], i[2]) for i in contexts.scanString(text)] context_tips = [] for i in matches: d = i[0] if len(d.items()) == 0: continue match_type, value = d.items()[0] start = i[1] end = i[2] context_tips.append( (start, end, value, match_type) ) # Now we do the actual transformation. s = text s = var_name.transformString(s) s = str_literal.transformString(s) s = implicit_op.transformString(s) s = implicit_hex.transformString(s) s = explicit_hex.transformString(s) return s, context_tips
def __init__(self): self.ae = False self.local_dict = None self.f = None self.user_functions = None self.expr_stack = [] self.texpr_stack = [] # Define constants self.constants = {} # Define Operators self.opn = {"+": operator.add, "-": operator.sub, "*": operator.mul, "/": operator.truediv, ">": operator.gt, ">=": operator.ge, "<": operator.lt, "<=": operator.le, "==": operator.eq, "!=": operator.ne, "|": operator.or_, "&": operator.and_, "!": operator.inv} # Define xarray DataArray operators with 1 input parameter self.xfn1 = {"angle": xr.ufuncs.angle, "arccos": xr.ufuncs.arccos, "arccosh": xr.ufuncs.arccosh, "arcsin": xr.ufuncs.arcsin, "arcsinh": xr.ufuncs.arcsinh, "arctan": xr.ufuncs.arctan, "arctanh": xr.ufuncs.arctanh, "ceil": xr.ufuncs.ceil, "conj": xr.ufuncs.conj, "cos": xr.ufuncs.cos, "cosh": xr.ufuncs.cosh, "deg2rad": xr.ufuncs.deg2rad, "degrees": xr.ufuncs.degrees, "exp": xr.ufuncs.exp, "expm1": xr.ufuncs.expm1, "fabs": xr.ufuncs.fabs, "fix": xr.ufuncs.fix, "floor": xr.ufuncs.floor, "frexp": xr.ufuncs.frexp, "imag": xr.ufuncs.imag, "iscomplex": xr.ufuncs.iscomplex, "isfinite": xr.ufuncs.isfinite, "isinf": xr.ufuncs.isinf, "isnan": xr.ufuncs.isnan, "isreal": xr.ufuncs.isreal, "log": xr.ufuncs.log, "log10": xr.ufuncs.log10, "log1p": xr.ufuncs.log1p, "log2": xr.ufuncs.log2, "rad2deg": xr.ufuncs.rad2deg, "radians": xr.ufuncs.radians, "real": xr.ufuncs.real, "rint": xr.ufuncs.rint, "sign": xr.ufuncs.sign, "signbit": xr.ufuncs.signbit, "sin": xr.ufuncs.sin, "sinh": xr.ufuncs.sinh, "sqrt": xr.ufuncs.sqrt, "square": xr.ufuncs.square, "tan": xr.ufuncs.tan, "tanh": xr.ufuncs.tanh, "trunc": xr.ufuncs.trunc} # Define xarray DataArray operators with 2 input parameter self.xfn2 = {"arctan2": xr.ufuncs.arctan2, "copysign": xr.ufuncs.copysign, "fmax": xr.ufuncs.fmax, "fmin": xr.ufuncs.fmin, "fmod": xr.ufuncs.fmod, "hypot": xr.ufuncs.hypot, "ldexp": xr.ufuncs.ldexp, "logaddexp": xr.ufuncs.logaddexp, "logaddexp2": xr.ufuncs.logaddexp2, "logicaland": xr.ufuncs.logical_and, "logicalnot": xr.ufuncs.logical_not, "logicalor": xr.ufuncs.logical_or, "logicalxor": xr.ufuncs.logical_xor, "maximum": xr.ufuncs.maximum, "minimum": xr.ufuncs.minimum, "nextafter": xr.ufuncs.nextafter} # Define non-xarray DataArray operators with 2 input parameter self.fn2 = {"percentile": np.percentile} # Define xarray DataArray reduction operators self.xrfn = {"all": xr.DataArray.all, "any": xr.DataArray.any, "argmax": xr.DataArray.argmax, "argmin": xr.DataArray.argmin, "max": xr.DataArray.max, "mean": xr.DataArray.mean, "median": xr.DataArray.median, "min": xr.DataArray.min, "prod": xr.DataArray.prod, "sum": xr.DataArray.sum, "std": xr.DataArray.std, "var": xr.DataArray.var} # Define non-xarray DataArray operators with 2 input parameter self.xcond = {"<": np.percentile} # Define Grammar point = Literal(".") e = CaselessLiteral("E") fnumber = Combine(Word("+-"+nums, nums) + Optional(point + Optional(Word(nums))) + Optional(e + Word("+-"+nums, nums))) variable = Word(alphas, alphas+nums+"_$") seq = Literal("=") b_not = Literal("~") plus = Literal("+") minus = Literal("-") mult = Literal("*") div = Literal("/") gt = Literal(">") gte = Literal(">=") lt = Literal("<") lte = Literal("<=") eq = Literal("==") neq = Literal("!=") b_or = Literal("|") b_and = Literal("&") l_not = Literal("!") lpar = Literal("(").suppress() rpar = Literal(")").suppress() comma = Literal(",") colon = Literal(":") lbrac = Literal("[") rbrac = Literal("]") lcurl = Literal("{") rcurl = Literal("}") qmark = Literal("?") scolon = Literal(";") addop = plus | minus multop = mult | div sliceop = colon compop = gte | lte | gt | lt eqop = eq | neq bitcompop = b_or | b_and bitnotop = b_not logicalnotop = l_not assignop = seq expop = Literal("^") expr = Forward() indexexpr = Forward() atom = (Optional("-") + (variable + seq + expr).setParseAction(self.push_assign) | indexexpr.setParseAction(self.push_index) | (lpar + expr + qmark.setParseAction(self.push_ternary1) + expr + scolon.setParseAction(self.push_ternary2) + expr + rpar).setParseAction(self.push_ternary) | (lpar + expr + qmark + expr + scolon + expr + rpar).setParseAction(self.push_ternary) | (logicalnotop + expr).setParseAction(self.push_ulnot) | (bitnotop + expr).setParseAction(self.push_unot) | (minus + expr).setParseAction(self.push_uminus) | (variable + lcurl + expr + rcurl).setParseAction(self.push_mask) | (variable + lpar + expr + (comma + expr)*3 + rpar).setParseAction(self.push_expr4) | (variable + lpar + expr + (comma + expr)*2 + rpar).setParseAction(self.push_expr3) | (variable + lpar + expr + comma + expr + rpar).setParseAction(self.push_expr2) | (variable + lpar + expr + rpar | variable).setParseAction(self.push_expr1) | fnumber.setParseAction(self.push_expr) | (lpar + expr + ZeroOrMore(comma + expr).setParseAction(self.get_tuple) + rpar).setParseAction(self.push_tuple) | (lpar + expr.suppress() + rpar).setParseAction(self.push_uminus)) # Define order of operations for operators factor = Forward() factor << atom + ZeroOrMore((expop + factor).setParseAction(self.push_op)) term = factor + ZeroOrMore((multop + factor).setParseAction(self.push_op)) term2 = term + ZeroOrMore((addop + term).setParseAction(self.push_op)) term3 = term2 + ZeroOrMore((sliceop + term2).setParseAction(self.push_op)) term4 = term3 + ZeroOrMore((compop + term3).setParseAction(self.push_op)) term5 = term4 + ZeroOrMore((eqop + term4).setParseAction(self.push_op)) term6 = term5 + ZeroOrMore((bitcompop + term5).setParseAction(self.push_op)) expr << term6 + ZeroOrMore((assignop + term6).setParseAction(self.push_op)) # Define index operators colon_expr = (colon + FollowedBy(comma) ^ colon + FollowedBy(rbrac)).setParseAction(self.push_colon) range_expr = colon_expr | expr | colon indexexpr << (variable + lbrac + delimitedList(range_expr, delim=',') + rbrac).setParseAction(self.push_expr) self.parser = expr
def _grammar(): from pyparsing import alphas, alphanums, nums from pyparsing import oneOf, Suppress, Optional, Group, ZeroOrMore, NotAny from pyparsing import Forward, operatorPrecedence, opAssoc, Word, White from pyparsing import delimitedList, Combine, Literal, OneOrMore expression = Forward() LPAR, RPAR, DOT, LBRAC, RBRAC = map(Suppress, "().{}") nw = NotAny(White()) identifier = Word(alphas + "_", alphanums + "_") integer = Word(nums) integer.setParseAction(IntegerNode) fractional = Combine(Word('+' + '-' + nums, nums) + '.' + Word(nums)) fractional.setParseAction(FloatNode) literal = fractional | integer arglist = delimitedList(expression) seqrange = LBRAC + expression + Suppress('..') + expression + RBRAC seqrange.setParseAction(lambda t: SequenceNode(start=t[0], stop=t[1])) seqexplicit = LBRAC + Optional(arglist) + RBRAC seqexplicit.setParseAction(lambda t: SequenceNode(lst=t)) sequence = seqrange | seqexplicit rollmod = nw + Group(oneOf("d k r e x") + Optional(integer)) numdice = Optional(integer, default=1) roll = numdice + nw + Suppress("d") + nw + (integer | sequence) roll += Group(ZeroOrMore(rollmod)) roll.setParseAction(DieRollNode) call = LPAR + Group(Optional(arglist)) + RPAR function = identifier + call function.setParseAction(FunctionNode) seqexpr = ((roll | sequence | function) + Group(OneOrMore(DOT + identifier + call))) seqexpr.setParseAction(SeqMethodNode) variable = Word(alphas + "_", alphanums + "_ ") variable.setParseAction(VariableNode) atom = seqexpr | roll | literal | sequence | function | variable expoop = Literal('^') signop = oneOf("+ -") multop = oneOf("* /") plusop = oneOf("+ -") # noinspection PyUnresolvedReferences expression << operatorPrecedence( atom, [ (expoop, 2, opAssoc.LEFT, BinaryOpNode), (signop, 1, opAssoc.RIGHT, UnaryOpNode), (multop, 2, opAssoc.LEFT, BinaryOpNode), (plusop, 2, opAssoc.LEFT, BinaryOpNode), ] ) return expression
sizing the data with the NUMBER similar to Dan Bernstein's netstrings setup. SPACE White space is basically ignored. This is interesting because since Stackish is serialized consistently this means you can use \n as the separation character and perform reasonable diffs on two structures. """ from pyparsing import Suppress,Word,nums,alphas,alphanums,Combine,oneOf,\ Optional,QuotedString,Forward,Group,ZeroOrMore,printables,srange MARK,UNMARK,AT,COLON,QUOTE = map(Suppress,"[]@:'") NUMBER = Word(nums) NUMBER.setParseAction(lambda t:int(t[0])) FLOAT = Combine(oneOf("+ -") + Word(nums) + "." + Optional(Word(nums))) FLOAT.setParseAction(lambda t:float(t[0])) STRING = QuotedString('"', multiline=True) WORD = Word(alphas,alphanums+"_:") ATTRIBUTE = Combine(AT + WORD) strBody = Forward() def setBodyLength(tokens): strBody << Word(srange(r'[\0x00-\0xffff]'), exact=int(tokens[0])) return "" BLOB = Combine(QUOTE + Word(nums).setParseAction(setBodyLength) + COLON + strBody + QUOTE) item = Forward() def assignUsing(s): def assignPA(tokens): if s in tokens:
def _unsigned_integer(): s = Combine(Optional("+") + Word(nums)) s.setParseAction(lambda s, l, tok: SimpleInteger(tok[0])) return s
model_reference = Regex(r'([\w\.]*#[\w\.]+)') variable = Regex(r'([a-zA-Z0-9\._]+)') string = QuotedString('"', escChar="\\") | QuotedString('\'', escChar="\\") operand = model_reference | real | integer | constant | string | variable plusop = oneOf('+ -') multop = oneOf('* / // %') groupop = Literal(',') expr = Forward() modifier = Combine(Word(alphas + nums) + ':') integer.setParseAction(EvalInteger) real.setParseAction(EvalReal) string.setParseAction(EvalString) constant.setParseAction(EvalConstant) variable.setParseAction(EvalVariable) model_reference.setParseAction(EvalModelReference) comparisonop = (oneOf("< <= > >= != == ~= ^= $=") | (Literal('not in') + WordEnd()) | (oneOf("in lt lte gt gte matches contains icontains like") + WordEnd())) logicopOR = Literal('or') + WordEnd() logicopAND = Literal('and') + WordEnd() expr << operatorPrecedence(operand, [ (modifier, 1, opAssoc.RIGHT, EvalModifierOp),
''' ParsingTmp.keywords.append(x) W = Where O = Optional S = Suppress number = Word(nums) point = Literal('.') e = CaselessLiteral('E') plusorminus = Literal('+') | Literal('-') integer = Combine(O(plusorminus) + number) floatnumber = Combine(integer + (point + O(number)) ^ (e + integer)) integer.setParseAction(lambda tokens: SimpleRValue(int(tokens[0]))) floatnumber.setParseAction(lambda tokens: SimpleRValue(float(tokens[0]))) pi = Keyword('pi').setParseAction(lambda tokens: SimpleRValue(math.pi, 'pi')) #@UnusedVariable isnumber = lambda x: isinstance(x, Number) rvalue = Forward() rvalue.setName('rvalue') contract_expression = Forward() contract_expression.setName('contract') simple_contract = Forward() simple_contract.setName('simple_contract') # Import all expressions -- they will call add_contract() from .library import (EqualTo, Unary, Binary, composite_contract, identifier_contract, misc_variables_contract, int_variables_contract, int_variables_ref,
def create_bnf(stack): point = Literal(".") comma = Literal(",") e = CaselessLiteral("E") inumber = Word(nums) fnumber = Combine(Word("+-"+nums, nums) + Optional(point + Optional(Word(nums))) + Optional(e + Word("+-"+nums, nums))) _of = Literal('of') _in = Literal('in') _by = Literal('by') _copy = Literal('copy') _mv = Literal('-v').setParseAction(replace('OA_SubV')) _me = Literal('-e').setParseAction(replace('OA_SubE')) _mf = Literal('-f').setParseAction(replace('OA_SubF')) _mc = Literal('-c').setParseAction(replace('OA_SubC')) _ms = Literal('-s').setParseAction(replace('OA_SubS')) _pv = Literal('+v').setParseAction(replace('OA_AddV')) _pe = Literal('+e').setParseAction(replace('OA_AddE')) _pf = Literal('+f').setParseAction(replace('OA_AddF')) _pc = Literal('+c').setParseAction(replace('OA_AddC')) _ps = Literal('+s').setParseAction(replace('OA_AddS')) _inv = Literal('*v').setParseAction(replace('OA_IntersectV')) _ine = Literal('*e').setParseAction(replace('OA_IntersectE')) _inf = Literal('*f').setParseAction(replace('OA_IntersectF')) _inc = Literal('*c').setParseAction(replace('OA_IntersectC')) _ins = Literal('*s').setParseAction(replace('OA_IntersectS')) regop = (_mv | _me | _mf | _mc | _ms | _pv | _pe | _pf | _pc | _ps | _inv | _ine | _inf | _inc | _ins) lpar = Literal("(").suppress() rpar = Literal(")").suppress() _all = Literal('all').setParseAction(replace('KW_All')) vertex = Literal('vertex') vertices = Literal('vertices') cell = Literal('cell') cells = Literal('cells') group = Literal('group') _set = Literal('set') surface = Literal('surface') ident = Word(alphas + '_.', alphanums + '_.') set_name = Word(nums) | ident function = Word(alphas + '_', alphanums + '_') function = Group(function).setParseAction(join_tokens) region = Combine(Literal('r.') + Word(alphas + '_', '_' + alphas + nums + '.')) region = Group(Optional(_copy, default='nocopy') + region) region.setParseAction(replace('KW_Region', keep=True)) coor = oneOf('x y z') boolop = oneOf('& |') relop = oneOf('< > <= >= != ==') bool_term = (ZeroOrMore('(') + (coor | fnumber) + relop + (coor | fnumber) + ZeroOrMore(')')) relation = Forward() relation << (ZeroOrMore('(') + bool_term + ZeroOrMore(boolop + relation) + ZeroOrMore(')')) relation = Group(relation).setParseAction(join_tokens) nos = Group(vertices + _of + surface).setParseAction(replace('E_VOS')) nir = Group(vertices + _in + relation).setParseAction( replace('E_VIR', keep=True)) nbf = Group(vertices + _by + function).setParseAction( replace('E_VBF', keep=True)) ebf = Group(cells + _by + function).setParseAction( replace('E_CBF', keep=True)) eog = Group(cells + _of + group + Word(nums)).setParseAction( replace('E_COG', keep=True)) nog = Group(vertices + _of + group + Word(nums)).setParseAction( replace('E_VOG', keep=True)) onir = Group(vertex + _in + region).setParseAction( replace_with_region('E_OVIR', 2)) ni = Group(vertex + delimitedList(inumber)).setParseAction( replace('E_VI', keep=True)) ei1 = Group(cell + delimitedList(inumber)).setParseAction( replace('E_CI1', keep=True)) etuple = (lpar.suppress() + inumber + comma.suppress() + inumber + rpar.suppress()) ei2 = Group(cell + delimitedList(etuple)).setParseAction( replace('E_CI2', keep=True)) noset = Group(vertices + _of + _set + set_name).setParseAction( replace('E_VOSET', keep=True)) eoset = Group(cells + _of + _set + set_name).setParseAction( replace('E_COSET', keep=True)) region_expression = Forward() atom1 = (_all | region | ni | onir | nos | nir | nbf | ei1 | ei2 | ebf | eog | nog | noset | eoset) atom1.setParseAction(to_stack(stack)) atom2 = (lpar + region_expression.suppress() + rpar) atom = (atom1 | atom2) aux = (regop + region_expression) aux.setParseAction(to_stack(stack)) region_expression << atom + ZeroOrMore(aux) region_expression = StringStart() + region_expression + StringEnd() return region_expression
return _handle_id(tokens[0]) elif tokens[1] == ".": return Dot(_handle_id(tokens[0]), _handle_ci(tokens[2:])) else: # tokens[1] == "[" return _handle_ci([ArrayAccess(_handle_id(tokens[0]), tokens[2])] + tokens[4:]) complex_identifier.setParseAction(lambda s, l, t: _handle_ci(t)) _define_identifier = complex_identifier _variable_identifier = complex_identifier # Integer numbers _integer_number = Combine(Optional("-") + PWord(nums)) _integer_number.setParseAction(lambda s, l, t: int(t[0])) # Constants _boolean_constant = oneOf("TRUE FALSE") _boolean_constant.setParseAction(lambda s, l, t: Falseexp() if t[0] == "FALSE" else Trueexp()) _integer_constant = _integer_number _symbolic_constant = identifier _range_constant = _integer_number + Suppress("..") + _integer_number _range_constant.setParseAction(lambda s, l, t: RangeConst(t[0], t[1])) _word_sign_specifier = oneOf("u s") _word_base = oneOf("b B o O d D h H") _word_width = PWord(nums) _word_value = PWord("0123456789abcdefABCDEF", "0123456789abcdefABCDEF_") _word_constant = Combine(
res += other if DEBUG: print "CollectionOut", res return [res] # SPARQL Grammar from http://www.w3.org/TR/sparql11-query/#grammar # ------ TERMINALS -------------- # [139] IRIREF ::= '<' ([^<>"{}|^`\]-[#x00-#x20])* '>' IRIREF = Combine( Suppress('<') + Regex(r'[^<>"{}|^`\\%s]*' % ''.join('\\x%02X' % i for i in range(33))) + Suppress('>')) IRIREF.setParseAction(lambda x: rdflib.URIRef(x[0])) # [164] P_CHARS_BASE ::= [A-Z] | [a-z] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x02FF] | [#x0370-#x037D] | [#x037F-#x1FFF] | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF] if sys.maxunicode == 0xffff: # this is narrow python build (default on windows/osx) # this means that unicode code points over 0xffff are stored # as several characters, which in turn means that regex character # ranges with these characters do not work. # See # * http://bugs.python.org/issue12729 # * http://bugs.python.org/issue12749 # * http://bugs.python.org/issue3665 # # Here we simple skip the [#x10000-#xEFFFF] part # this means that some SPARQL queries will not parse :(
def _BNF(self): base16 = Literal("$") hex = Combine(base16 + Word(hexnums + "_")) base4 = Literal("%%") quaternary = Combine(base4 + Word("0123_")) base2 = Literal("%") binary = Combine(base2 + Word("01_")) plusminus = Literal("+") | Literal("-") integer = Combine(Optional(plusminus) + Word(nums + "_")) name_token = Combine( Optional(Literal(":") | Literal("@")) + Word("_" + alphas, "_" + alphanums)) name_token.setParseAction(self._mark_name_token) lparens = Literal("(").suppress() rparens = Literal(")").suppress() # op0 = Literal("@") op1 = (Literal("^^") | Literal("||") | Literal("|<") | Literal(">|") | Literal("!")).setParseAction(self._mark_unary) op2 = Literal("->") | Literal("<-") | Literal(">>") | Literal( "<<") | Literal("~>") | Literal("><") op3 = Literal("&") op4 = Literal("|") | Literal("^") op5 = Literal("**") | Literal("*") | Literal("//") | Literal("/") op6 = Literal("+") | Literal("-") op7 = Literal("#>") | Literal("<#") op8 = Literal("<") | Literal(">") | Literal("<>") | Literal( "==") | Literal("=<") | Literal("=>") op9 = Literal("NOT").setParseAction(self._mark_unary) op10 = Literal("AND") op11 = Literal("OR") op12 = Literal(",") expr = Forward() atom = name_token | hex | quaternary | binary | integer | quotedString atom.setParseAction(self._push) atom = atom | (lparens + expr.suppress() + rparens) # term0 = atom + ZeroOrMore((op0 + atom) .setParseAction(self._push)) # term1 = term0 + ZeroOrMore((op1 + term0) .setParseAction(self._push)) term1 = atom + ZeroOrMore((op1 + atom).setParseAction(self._push)) term2 = term1 + ZeroOrMore((op2 + term1).setParseAction(self._push)) term3 = term2 + ZeroOrMore((op3 + term2).setParseAction(self._push)) term4 = term3 + ZeroOrMore((op4 + term3).setParseAction(self._push)) term5 = term4 + ZeroOrMore((op5 + term4).setParseAction(self._push)) term6 = term5 + ZeroOrMore((op6 + term5).setParseAction(self._push)) term7 = term6 + ZeroOrMore((op7 + term6).setParseAction(self._push)) term8 = term7 + ZeroOrMore((op8 + term7).setParseAction(self._push)) term9 = term8 + ZeroOrMore((op9 + term8).setParseAction(self._push)) term10 = term9 + ZeroOrMore((op10 + term9).setParseAction(self._push)) term11 = term10 + ZeroOrMore( (op11 + term10).setParseAction(self._push)) expr << term11 + ZeroOrMore((op12 + term11).setParseAction(self._push)) return expr
if res: res += [res[-3], rdflib.RDF.rest, b, b, rdflib.RDF.first, x] else: res += [b, rdflib.RDF.first, x] res += [b, rdflib.RDF.rest, rdflib.RDF.nil] if DEBUG: print "CollectionOut", res return [res] # SPARQL Grammar from http://www.w3.org/TR/sparql11-query/#grammar # ------ TERMINALS -------------- # [139] IRIREF ::= '<' ([^<>"{}|^`\]-[#x00-#x20])* '>' IRIREF = Combine(Suppress("<") + Regex(r'[^<>"{}|^`\\%s]*' % "".join("\\x%02X" % i for i in range(33))) + Suppress(">")) IRIREF.setParseAction(lambda x: rdflib.URIRef(x[0])) # [164] P_CHARS_BASE ::= [A-Z] | [a-z] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x02FF] | [#x0370-#x037D] | [#x037F-#x1FFF] | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF] if sys.maxunicode == 0xFFFF: # this is narrow python build (default on windows/osx) # this means that unicode code points over 0xffff are stored # as several characters, which in turn means that regex character # ranges with these characters do not work. # See # * http://bugs.python.org/issue12729 # * http://bugs.python.org/issue12749 # * http://bugs.python.org/issue3665 # # Here we simple skip the [#x10000-#xEFFFF] part # this means that some SPARQL queries will not parse :(
Create pyparsing grammar for problem configuration and options. """ from pyparsing import (Word, Group, Suppress, Combine, Optional, Forward, Empty, quotedString, oneOf, removeQuotes, delimitedList, nums, alphas, alphanums, Keyword, CaselessLiteral) word_free = Word(alphas + '][@_-/.+**' + alphanums) word_strict = Word(alphas, alphas + alphanums + '_' ) (lparen, rparen, lbrack, rbrack, lbrace, rbrace, colon, equal_sign) = map(Suppress, '()[]{}:=') integer = Combine(Optional(oneOf('+ -')) + Word(nums)).setName('integer') cvt_int = lambda toks: int(toks[0]) integer.setParseAction(cvt_int) boolean_true = Keyword('True', caseless=True) boolean_true.setParseAction(lambda x: True) boolean_false = Keyword('False', caseless=True) boolean_false.setParseAction(lambda x: False) boolean = boolean_true | boolean_false none = Keyword('None', caseless=True) cvt_none = lambda toks: [None] none.setParseAction(cvt_none) e = CaselessLiteral("e") real = (Combine(Optional(oneOf('+ -')) + Word(nums)
sizing the data with the NUMBER similar to Dan Bernstein's netstrings setup. SPACE White space is basically ignored. This is interesting because since Stackish is serialized consistently this means you can use \n as the separation character and perform reasonable diffs on two structures. """ from pyparsing import Suppress,Word,nums,alphas,alphanums,Combine,oneOf,\ Optional,QuotedString,Forward,Group,ZeroOrMore,printables,srange MARK, UNMARK, AT, COLON, QUOTE = map(Suppress, "[]@:'") NUMBER = Word(nums) NUMBER.setParseAction(lambda t: int(t[0])) FLOAT = Combine(oneOf("+ -") + Word(nums) + "." + Optional(Word(nums))) FLOAT.setParseAction(lambda t: float(t[0])) STRING = QuotedString('"', multiline=True) WORD = Word(alphas, alphanums + "_:") ATTRIBUTE = Combine(AT + WORD) strBody = Forward() def setBodyLength(tokens): strBody << Word(srange(r'[\0x00-\0xffff]'), exact=int(tokens[0])) return "" BLOB = Combine(QUOTE + Word(nums).setParseAction(setBodyLength) + COLON + strBody + QUOTE)
def parsebinop(opexpr): "parse action for binary operators" left, opstr, right = opexpr for opclass in binopclasses: if opstr in opclass.literals: return opclass(left, right) binop = oneOf(binopstr) arithSign = Word("+-", exact=1) realNum = Combine(Optional(arithSign) + (Word(nums) + "." + Optional(Word(nums)) | ("." + Word(nums))) # noqa + Optional(E + Optional(arithSign) + Word(nums))) realNum.setParseAction(lambda x: expression.NumericLiteral(float(x[0]))) intNum = Combine(Optional(arithSign) + Word(nums) + Optional(E + Optional("+") + Word(nums))) intNum.setParseAction(lambda x: expression.NumericLiteral(int(x[0]))) number = realNum | intNum variable = ident.copy() variable.setParseAction(lambda x: model.Var(x[0])) quotedString.setParseAction(lambda x: expression.StringLiteral(x[0][1:-1])) literal = quotedString | number valueref = variable | literal
Optional( '.' + Word(nums)) + Optional( Word('eE',exact=1) + Word(nums+'+-',nums))) str_sexp = QuotedString('"', escChar='\\', unquoteResults=False) bool_sexp = (TRUE | FALSE) atom = (num_sexp | str_sexp | bool_sexp | NONE) extra_allowed_ident_chars = '+-=*/!?%' var_sexp = Word(initChars=alphas + extra_allowed_ident_chars, bodyChars=alphanums + extra_allowed_ident_chars) sexp << (atom | var_sexp | list_sexp).ignore(comment) multi_sexp = OneOrMore(sexp) def convert_num(s, l, toks): n = toks[0] try: return int(n) except ValueError: return float(n) num_sexp.setParseAction(convert_num) def get_syntax_tree(fname): infile = open(fname, 'r') try: file_text = infile.read() return get_syntax_tree_from_str(file_text).asList() finally: infile.close() def get_syntax_tree_from_str(str): return multi_sexp.parseString(str, parseAll=True)
def create_bnf( stack ): point = Literal( "." ) comma = Literal( "," ) e = CaselessLiteral( "E" ) inumber = Word( nums ) fnumber = Combine( Word( "+-"+nums, nums ) + Optional( point + Optional( Word( nums ) ) ) + Optional( e + Word( "+-"+nums, nums ) ) ) _of = Literal( 'of' ) _in = Literal( 'in' ) _by = Literal( 'by' ) _copy = Literal( 'copy' ) _mn = Literal( '-n' ).setParseAction( replace( 'OA_SubN' ) ) _me = Literal( '-e' ).setParseAction( replace( 'OA_SubE' ) ) _pn = Literal( '+n' ).setParseAction( replace( 'OA_AddN' ) ) _pe = Literal( '+e' ).setParseAction( replace( 'OA_AddE' ) ) _inn = Literal( '*n' ).setParseAction( replace( 'OA_IntersectN' ) ) _ine = Literal( '*e' ).setParseAction( replace( 'OA_IntersectE' ) ) regop = (_mn | _me | _pn | _pe | _inn | _ine) lpar = Literal( "(" ).suppress() rpar = Literal( ")" ).suppress() _all = Literal( 'all' ).setParseAction( replace( 'KW_All' ) ) node = Literal( 'node' ) nodes = Literal( 'nodes' ) element = Literal( 'element' ) elements = Literal( 'elements' ) group = Literal( 'group' ) surface = Literal( 'surface' ) variable = Word( 'xyz', max = 1 ) | Literal( 'domain' ) any_var = Word( alphas + '_', alphanums + '_' ) | fnumber function = Word( alphas, alphanums + '_' ) function = Group( function ).setParseAction( join_tokens ) region = Combine( Literal( 'r.' ) + Word( alphas, '_' + alphas + nums ) ) region = Group( Optional( _copy, default = 'nocopy' ) + region ) region.setParseAction( replace( 'KW_Region', keep = True ) ) coor = oneOf( 'x y z' ) boolop = oneOf( '& |' ) relop = oneOf( '< > <= >= != ==' ) bool_term = ZeroOrMore( '(' ) + (coor | fnumber ) + relop + (coor | fnumber)\ + ZeroOrMore( ')' ) relation = Forward() relation << ZeroOrMore( '(' )\ + bool_term + ZeroOrMore( boolop + relation )\ + ZeroOrMore( ')' ) relation = Group( relation ).setParseAction( join_tokens ) nos = Group( nodes + _of + surface ).setParseAction( replace( 'E_NOS' ) ) nir = Group( nodes + _in + relation ).setParseAction( \ replace( 'E_NIR', keep = True ) ) nbf = Group( nodes + _by + function ).setParseAction( \ replace( 'E_NBF', keep = True ) ) ebf = Group( elements + _by + function ).setParseAction( \ replace( 'E_EBF', keep = True ) ) eog = Group( elements + _of + group + Word( nums ) ).setParseAction( \ replace( 'E_EOG', keep = True ) ) nog = Group( nodes + _of + group + Word( nums ) ).setParseAction( \ replace( 'E_NOG', keep = True ) ) onir = Group( node + _in + region ).setParseAction( \ replace_with_region( 'E_ONIR', 2 ) ) ni = Group( node + delimitedList( inumber ) ).setParseAction( \ replace( 'E_NI', keep = True ) ) ei1 = Group( element + delimitedList( inumber ) ).setParseAction( \ replace( 'E_EI1', keep = True ) ) etuple = lpar.suppress() + inumber + comma.suppress() \ + inumber + rpar.suppress() ei2 = Group( element + delimitedList( etuple ) ).setParseAction( \ replace( 'E_EI2', keep = True ) ) region_expression = Forward() atom1 = (_all | region | ni | onir | nos | nir | nbf | ei1 | ei2 | ebf | eog | nog) atom1.setParseAction( to_stack( stack ) ) atom2 = (lpar + region_expression.suppress() + rpar) atom = (atom1 | atom2) aux = (regop + region_expression) aux.setParseAction( to_stack( stack ) ) region_expression << atom + ZeroOrMore( aux ) region_expression = StringStart() + region_expression + StringEnd() # region.set_debug() # relation.set_debug() # region_expression.set_debug() return region_expression
and the discussion of the "-" operator in the docs. """ ParsingTmp.keywords.append(x) W = Where O = Optional S = Suppress basenumber = Word(nums) point = Literal('.') e = CaselessLiteral('E') plusorminus = Literal('+') | Literal('-') integer = Combine(O(plusorminus) + basenumber) integer.setParseAction(lambda tokens: SimpleRValue(int(tokens[0]))) floatnumber = Combine( O(plusorminus) + integer + (point + O(basenumber)) ^ (e + integer)) floatnumber.setParseAction(lambda tokens: SimpleRValue(float(tokens[0]))) pi = Keyword('pi').setParseAction( lambda tokens: SimpleRValue(math.pi, 'pi')) # @UnusedVariable try: import numpy except ImportError: numpy = None def isnumber(x): # These are scalar quantities that we can compare (=,>,>=, etc.) if isinstance(x, Number):
equal_op = Literal( "=" ) not_equal_op = Literal( "!=" ) greater_op = Combine( Literal( ">" ) + ~Literal( "=" ) ) greater_equal_op = Combine( Literal( ">" ) + Literal( "=" ) ) less_op = Combine( Literal( "<" ) + ~Literal( "=" ) ) less_equal_op = Combine( Literal( "<" ) + Literal( "=" ) ) addop = plus | minus multop = mult | div compop = less_op | greater_op | less_equal_op | greater_equal_op | not_equal_op | equal_op expop = Literal( "^" ) #assign = Literal( "=" ) band = Literal( "@" ) expr = Forward() atom = ( ( e | floatnumber | integer | ident.setParseAction( assignVar ) | fn + lpar + expr + rpar | fn + lpar + expr + colon + expr + rpar | fn + lpar + expr + colon + expr + colon + expr + rpar ).setParseAction(pushFirst) | ( lpar + expr.suppress() + rpar ) ) factor = Forward() factor << atom + ( ( band + factor ).setParseAction( pushFirst ) | ZeroOrMore( ( expop + factor ).setParseAction( pushFirst ) ) ) term = factor + ZeroOrMore( ( multop + factor ).setParseAction( pushFirst ) ) addterm = term + ZeroOrMore( ( addop + term ).setParseAction( pushFirst ) ) expr << addterm + ZeroOrMore( ( compop + addterm ).setParseAction( pushFirst ) ) bnf = expr pattern = bnf + StringEnd() # map operator symbols to corresponding arithmetic operations opn = { "+" : ( lambda a,b: numpy.add( a, b ) ),