def defineParsers(): #Enable a fast parsing mode with caching. ParserElement.enablePackrat() #end of line terminates statements, so it is not regular whitespace ParserElement.setDefaultWhitespaceChars('\t ') func_call = Forward() #forward declaration because this is a recursive rule #The "terminal" rules symbol = Word(alphas+'_-', alphanums+'_-') .setParseAction(action_symbol) q_symbol = quotedString .setParseAction(action_q_symbol) bracket_term = Literal("(").suppress() - func_call \ + Literal(")").suppress() word = symbol | q_symbol | bracket_term #The function call #Parse: "foo | bar | baz" or "foo" pipeline = (word + ZeroOrMore("|" - word)) .setParseAction(action_pipeline) #Parse "foo|bar op1 op2 op3" func_call << (pipeline - ZeroOrMore(word)) .setParseAction(action_func_call) #High level structure of program line = LineEnd() | func_call - LineEnd() #empty line or function call program = ZeroOrMore(line) + StringEnd() #multiple lines are a program #define the comments program.ignore('%' + restOfLine) #no tab expansion program.parseWithTabs() #return additional func_call parser to make testing more easy return program, func_call
def __init__(self, alphabet): self.operators = alphabet.getOperators() self.constants = alphabet.getConstants() self.notNeedSpace = alphabet.notNeedSpace() self.ffactory = FormulaFactory() self.__createGram() ParserElement.enablePackrat()
def _int_expression(): from pyparsing import Word, alphanums, nums, Forward, ZeroOrMore, Combine, CaselessLiteral, srange, ParserElement, Optional ParserElement.enablePackrat() entry_name = Word(alphanums + ' _+:.-/') integer = Combine(Optional('-') + Word(nums)).addParseAction(lambda s,l,t: [Constant(int(t[0]))]) hex = Combine(CaselessLiteral("0x") + Word(srange("[0-9a-fA-F]"))).addParseAction(lambda s,l,t:[Constant(int(t[0][2:], 16))]) named_reference = ('${' + entry_name + '}').addParseAction(lambda s,l,t:ValueResult(t[1])) length_reference = ('len{' + entry_name + '}').addParseAction(lambda s,l,t:LengthResult(t[1])) expression = Forward() factor = hex | integer | named_reference | length_reference | ('(' + expression + ')').addParseAction(lambda s,l,t:t[1]) entry = factor for ops in _operators: op_parse = reduce(operator.or_, [(character + entry).addParseAction(_half(op)) for character, op in ops]) entry = (entry + ZeroOrMore(op_parse)).addParseAction(_collapse) expression << entry return expression
nums, alphas, Combine, oneOf, opAssoc, operatorPrecedence, QuotedString, Literal, ParserElement, ParseException, Forward, Group, Suppress, Optional, Regex) ParserElement.enablePackrat() from sqlalchemy import and_, or_, func #from sqlalchemy.orm import aliased import operator import re def dbobject(obj): return getattr(obj, '__moyadbobject__', lambda: obj)() @implements_to_string class DBExpressionError(Exception): hide_py_traceback = True
def make_amr_parser(): """ Pyparsing parser for AMRs. This will return an abstract syntax tree that needs to be converted into an AMR using ast_to_amr. """ def debug(s, loc, tok): if len(tok) > 1: flat = [tok[0]] + tok[1:] else: flat = tok return flat def parse_concept_expr(s, loc, tok): node_name = tok[0] concept_name = None roles = [] if len(tok) > 1: if type(tok[1]) is tuple: roles = tok[1:] else: concept_name = tok[1] if len(tok) > 2: roles = tok[2:] return (node_name, concept_name, roles) ParserElement.enablePackrat() # Hopefully no bug in here... def parse_role(s,loc,tok): if len(tok) >= 2: r, ch = tok[0], [] for v in tok[1:]: if isinstance(v, StrLiteral): # Parse the node alignment and move it to the edge parts = v.replace(" ","").rsplit("~",1) if len(parts) >= 2: v, align = parts v = StrLiteral(v) r = "%s~%s" % (r.strip(), align.strip()) elif isinstance(v, SpecialValue): parts = v.replace(" ","").rsplit("~",1) if len(parts) >= 2: v, align = parts v = StrLiteral(v) r = "%s~%s" % (r.strip(), align.strip()) ch.append(v) return r, ch else: return tok[0] # Number are all mapped to the same node in the graph because of interning parse_quantity = lambda s, loc, tok: StrLiteral(" ".join(tok)) #float(tok[0]) if "." in tok[0] else int(tok[0]) parse_string_literal = lambda s, loc, tok: StrLiteral(" ".join(tok)) parse_special_value = lambda s, loc, tok: SpecialValue(" ".join(tok)) lpar = Literal( "(" ).suppress() rpar = Literal( ")" ).suppress() quantity = Word(nums+".,").setParseAction(parse_quantity) node_name = Word(alphas+nums+"""@-_.~$/<>%&!+\*?^`"'""") #Word(alphas+nums+"_@.") lit_string = Literal('"').suppress() + CharsNotIn('"') + Literal('"').suppress() concept_name = lit_string | Word(alphas+nums+"""-_.,`~$/<>%&!+\*?^"'""") role_name = Word(alphas+nums+"""-_.,~$/<>%&!+\*:?^`"'""") | Literal("#").suppress()+Word(alphas+nums+"[]-$_").setParseAction(lambda s, loc, tok: NonterminalLabel(tok[0])) special_attr = (Literal("-") | Literal("interrogative") | Literal("SHOULD") | Literal("MUST") | Literal("HAVE-TO")| Literal("WOULD") | Literal("CAN") | Literal("DARE-TO")| Literal("BE-TO") | Literal("MAY") | Literal("GOING-TO") | Literal("MIGHT") | Literal("USED-TO")) + Optional(Literal("~")+Word(alphas+nums+".")) expr = Forward() value = expr |\ quantity.setParseAction(parse_quantity) |\ special_attr.setParseAction(parse_special_value) | \ node_name |\ (lit_string + Optional(Literal("~")+Word(alphas+nums+"."))).setParseAction(parse_string_literal) valuelist = Forward() valuelist << (value + Literal(",").suppress() + valuelist | value).setParseAction(debug) role = (Literal(":").suppress() + role_name + valuelist).setParseAction(parse_role) expr.setParseAction(parse_concept_expr) expr << (lpar + node_name + Optional(Literal("/").suppress() + concept_name) + ZeroOrMore(role) + rpar) return expr
# This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. import math import re import pyparsing from pyparsing import (ParserElement, Word, oneOf, Literal, CaselessLiteral, Regex, Optional, Suppress, Forward, FollowedBy, NotAny, Group, OneOrMore, ZeroOrMore, nums, alphas, ParseResults) ParserElement.enablePackrat() # Significant speedup. from . import operators from .units import Q_ # Definitions. ######################################################### # These are reserved words that cannot be variables, constants, or units. keyword = oneOf("to") # Special symbols for currencies. currency_symbols = oneOf("€ £ $ ₪ ¥ ¥ ₩ ₩ ฿ ₹") # Identifier, must start with unicode letter, can then contain unicode # letter, unicode number, or underscore. identifier = currency_symbols | ( NotAny( keyword ) + Regex(r'[^\W\d_]\w*') )
def make_grammar(): from pyparsing import (ParserElement, Literal, Word, Forward, Optional, QuotedString, Combine, ZeroOrMore, Keyword, alphas, alphanums, nums) ParserElement.enablePackrat() plus = Literal("+") minus = Literal("-") mul = Literal("*") div = Literal("/") floordiv = Literal("//") mod = Literal("%") lt = Literal("<") le = Literal("<=") gt = Literal(">") ge = Literal(">=") lshift = Literal("<<") rshift = Literal(">>") equal = Literal("==") | Literal("=") | Literal("!=") bitwise_not = Literal("~") bitwise_and = Literal("&") bitwise_or = Literal("|") bitwise_xor = Literal("^") logical_not = Literal("!") | Keyword("not") logical_and = Literal("&&") | Literal("and") | Keyword("AND") logical_or = Literal("||") | Keyword("or") | Keyword("OR") ident = Word(alphas + "_", alphanums + "_") functionname = Word(alphas + "_", alphanums + "_") unit = Word(alphas) int_number = Word(nums) float_number = Combine(Word(nums) + Optional(Literal(".") + Word(nums))) number = (float_number | int_number) + Optional(unit) lparent = Literal("(").suppress() rparent = Literal(")").suppress() relational_op = (lt | le | gt | ge) shift = (lshift | rshift) add_op = (plus | minus) mul_op = (mul | floordiv | div | mod) expr = Forward() string = (QuotedString('"') | QuotedString("'")) primary_expr = ident | number | string | (lparent + expr + rparent) def make_op(s, loc, toks): if len(toks) == 1: return toks[0] else: def loop(lhs, rest): if len(rest) == 0: return lhs else: return loop(Operator(rest[0], lhs, rest[1]), rest[2:]) return loop(Operator(toks[1], toks[0], toks[2]), toks[3:]) def make_unary(s, loc, toks): if len(toks) == 1: return toks[0] else: return UnaryOperator(toks[0], make_unary(s, loc, toks[1:])) argument_expression_list = expr + ZeroOrMore(Literal(",").suppress() + expr) function_expression = (functionname + lparent + argument_expression_list + rparent) postfix_expression = function_expression | primary_expr unary_expr = ZeroOrMore(bitwise_not | logical_not | minus | plus) + postfix_expression cast_expresion = unary_expr | postfix_expression mult_expr = cast_expresion + ZeroOrMore(mul_op + cast_expresion) # noqa: E221 add_expr = mult_expr + ZeroOrMore(add_op + mult_expr) # noqa: E221 shift_expr = add_expr + ZeroOrMore(shift + add_expr) # noqa: E221 relational_expr = shift_expr + ZeroOrMore(relational_op + shift_expr) # noqa: E221 equality_expr = relational_expr + ZeroOrMore(equal + relational_expr) # noqa: E221 bitwise_and_expr = equality_expr + ZeroOrMore(bitwise_and + equality_expr) # noqa: E221 bitwise_xor_expr = bitwise_and_expr + ZeroOrMore(bitwise_xor + bitwise_and_expr) # noqa: E221 bitwise_or_expr = bitwise_xor_expr + ZeroOrMore(bitwise_or + bitwise_xor_expr) # noqa: E221 logical_and_expr = bitwise_or_expr + ZeroOrMore(logical_and + bitwise_or_expr) # noqa: E221 logical_or_expr = logical_and_expr + ZeroOrMore(logical_or + logical_and_expr) # noqa: E221 expr <<= logical_or_expr function_expression.setParseAction(Function) int_number.setParseAction(lambda s, loc, toks: int(toks[0])) float_number.setParseAction(lambda s, loc, toks: float(toks[0])) number.setParseAction(Number) string.setParseAction(String) ident.setParseAction(Variable) unary_expr.setParseAction(make_unary) mult_expr.setParseAction(make_op) add_expr.setParseAction(make_op) shift_expr.setParseAction(make_op) relational_expr.setParseAction(make_op) equality_expr.setParseAction(make_op) bitwise_and_expr.setParseAction(make_op) bitwise_xor_expr.setParseAction(make_op) bitwise_or_expr.setParseAction(make_op) logical_and_expr.setParseAction(make_op) logical_or_expr.setParseAction(make_op) return expr
from ..context.modifiers import ExpressionModifiers from ..moyaexceptions import MoyaException from ..errors import LogicError from operator import methodcaller import logging import operator import re import sys from operator import truth import threading log = logging.getLogger("moya.runtime") ParserElement.enablePackrat(None) sys.setrecursionlimit(10000) VERSION = 2 @implements_to_string class ExpressionError(Exception): def __init__(self, exp, msg=None, col=None, original=None): super(ExpressionError, self).__init__() self.exp = exp self.msg = msg or "" self.original = original self.col = col
from distutils.version import StrictVersion from pyparsing import (ParserElement, Forward, Combine, Optional, Word, Literal, CaselessKeyword, CaselessLiteral, Group, FollowedBy, LineEnd, OneOrMore, ZeroOrMore, nums, alphas, alphanums, printables, delimitedList, quotedString, __version__) ParserElement.enablePackrat() grammar = Forward() expression = Forward() # Literals intNumber = Combine(Optional('-') + Word(nums))('integer') floatNumber = Combine(Optional('-') + Word(nums) + Literal('.') + Word(nums))('float') sciNumber = Combine((floatNumber | intNumber) + CaselessLiteral('e') + intNumber)('scientific') aString = quotedString('string') # Use lookahead to match only numbers in a list (can't remember why this # is necessary) afterNumber = FollowedBy(",") ^ FollowedBy(")") ^ FollowedBy(LineEnd()) number = Group((sciNumber + afterNumber) | (floatNumber + afterNumber) | (intNumber + afterNumber))('number') boolean = Group(CaselessKeyword("true") | CaselessKeyword("false"))('boolean')
# Add // and % to multOp & EvalMultOp # Keep integer values as integers until something converts them # Allow longer var names # Based on: # # eval_arith.py # # Copyright 2009, Paul McGuire # # Expansion on the pyparsing example simpleArith.py, to include evaluation # of the parsed tokens. from pyparsing import Word, nums, alphas, Combine, oneOf, Optional, \ opAssoc, operatorPrecedence, ParserElement ParserElement.enablePackrat() # Add memoization to parsing logic to increase performance class EvalConstant(): "Class to evaluate a parsed constant or variable" def __init__(self, tokens): self.value = tokens[0] def eval(self, vars_): if self.value in vars_: return vars_[self.value] else: # Try to return an int, then a float, and finally a string try: return int( self.value ) except: pass try:
LineEnd, LineStart, Literal, nums, oneOf, OneOrMore, ParserElement, printables, SkipTo, StringEnd, StringStart, White, Word, ) ParserElement.enablePackrat() # faster ParserElement.setDefaultWhitespaceChars(" \t") # use significant newlines # Data types # ------------------------------------------------------------------------------------------ # integer integer = Word(nums + "+-") # integer # floating point fp = Combine(Word(nums + "+-") + Literal(".") + Word(nums)) # fortran real exp = oneOf("E e D d") real = Combine(fp("base") + exp.setParseAction(lambda x: "e") + integer("exponent"))
def disable_packrat() -> None: ParserElement.enablePackrat(cache_size_limit=None)
def __init__(self): ParserElement.enablePackrat() unit_years = (CaselessLiteral("years") | CaselessLiteral("year") | CaselessLiteral("y")) years = ( Word(nums).setParseAction(lambda s, l, t: [int(t[0])])("years") + unit_years) unit_months = (CaselessLiteral("months") | CaselessLiteral("month") | CaselessLiteral("mo")) months = ( Word(nums).setParseAction(lambda s, l, t: [int(t[0])])("months") + unit_months) unit_weeks = (CaselessLiteral("weeks") | CaselessLiteral("week") | CaselessLiteral("w")) weeks = ( Word(nums).setParseAction(lambda s, l, t: [int(t[0])])("weeks") + unit_weeks) unit_days = (CaselessLiteral("days") | CaselessLiteral("day") | CaselessLiteral("d")) days = ( Word(nums).setParseAction(lambda s, l, t: [int(t[0])])("days") + unit_days) unit_hours = (CaselessLiteral("hours") | CaselessLiteral("hour") | CaselessLiteral("hrs") | CaselessLiteral("hr") | CaselessLiteral("h")) hours = ( Word(nums).setParseAction(lambda s, l, t: [int(t[0])])("hours") + unit_hours) unit_minutes = (CaselessLiteral("minutes") | CaselessLiteral("minute") | CaselessLiteral("mins") | CaselessLiteral("min") | CaselessLiteral("m")) minutes = ( Word(nums).setParseAction(lambda s, l, t: [int(t[0])])("minutes") + unit_minutes) unit_seconds = (CaselessLiteral("seconds") | CaselessLiteral("second") | CaselessLiteral("secs") | CaselessLiteral("sec") | CaselessLiteral("s")) seconds = ( Word(nums).setParseAction(lambda s, l, t: [int(t[0])])("seconds") + unit_seconds) time_unit = years | months | weeks | days | hours | minutes | seconds time_unit_separators = Optional(Literal(",")) + Optional( CaselessLiteral("and")) full_time = time_unit + ZeroOrMore( Suppress(Optional(time_unit_separators)) + time_unit) every_time = Group(CaselessLiteral("every") + full_time)("every") in_opt_time = Group(Optional(CaselessLiteral("in")) + full_time)("in") in_req_time = Group(CaselessLiteral("in") + full_time)("in") reminder_text_capture = SkipTo(every_time | in_req_time | StringEnd()).setParseAction( tokenMap(str.strip)) reminder_text_optional_prefix = Optional( Suppress(CaselessLiteral("to"))) reminder_text = reminder_text_optional_prefix + reminder_text_capture( "text") in_every_text = in_opt_time + every_time + reminder_text every_in_text = every_time + in_req_time + reminder_text in_text_every = in_opt_time + reminder_text + every_time every_text_in = every_time + reminder_text + in_req_time text_in_every = reminder_text + in_req_time + every_time text_every_in = reminder_text + every_time + in_req_time in_text = in_opt_time + reminder_text text_in = reminder_text + in_req_time every_text = every_time + reminder_text text_every = reminder_text + every_time template = (in_every_text | every_in_text | in_text_every | every_text_in | text_in_every | text_every_in | in_text | text_in | every_text | text_every) self.parser = template
# ############################################################################# from __future__ import print_function import re import ast import sys from copy import deepcopy from collections import namedtuple from mdtraj.utils.six import PY2 from pyparsing import (Word, ParserElement, MatchFirst, Keyword, opAssoc, quotedString, alphas, alphanums, infixNotation, Group, ParseException, OneOrMore) import astunparse # this number arises from the current selection language, if the cache size is exceeded, it hurts performance a bit. ParserElement.enablePackrat(cache_size_limit=304) __all__ = ['parse_selection'] # ############################################################################ # Globals # ############################################################################ NUMS = '.0123456789' THIS_ATOM = ast.Name(id='atom', ctx=ast.Load(), SINGLETON=True) RE_MODULE = ast.Name(id='re', ctx=ast.Load(), SINGLETON=True) SELECTION_GLOBALS = {'re': re} _ParsedSelection = namedtuple('_ParsedSelection', ['expr', 'source', 'astnode']) # ############################################################################