def np(words, fn=gr_opt_quoted_string, action=None): p = Keyword(words[0], caseless=True) for w in words[1:]: p = p | Keyword(w, caseless=True) p = p + gr_eq + fn p.setParseAction(action) return p
def np(words, fn = gr_opt_quoted_string, action=None): p = Keyword(words[0], caseless=True) for w in words[1:]: p = p | Keyword(w, caseless=True) p = p + gr_eq + fn p.setParseAction(action) return p
def __createGram(self): if self.notNeedSpace: lNot = Keyword(self.operators['not']) else: lNot = Literal(self.operators['not']) lAnd = Literal(self.operators['and']) lOr = Literal(self.operators['or']) lImp = Literal(self.operators['impL']) lEqu = Literal(self.operators['equ']) lTrue = Keyword(self.constants['true']) lFalse = Keyword(self.constants['false']) lVar = Word(alphas, alphanums+'_') lVar.setParseAction(self.ffactory.createLogicVariable) lTrue.setParseAction(self.ffactory.createLogicTruth) lFalse.setParseAction(self.ffactory.createLogicFalse) factor = lTrue | lFalse | lVar expression = myparsing.operatorPrecedence(factor, [ (lNot, 1, opAssoc.RIGHT, self.ffactory.createNotOperation), (lAnd, 2, opAssoc.LEFT, self.ffactory.createAndOperation), (lOr, 2, opAssoc.LEFT, self.ffactory.createOrOperation), (lImp, 2, opAssoc.LEFT, self.ffactory.createImpicationOperation), (lEqu, 2, opAssoc.LEFT, self.ffactory.createEquvalenceOperation) ], [('(', ')'), ('[', ']'), ('{', '}')]) self.final = expression + StringEnd()
def np(words, fn = gr_opt_quoted_string, action=nullDebugAction): p = Keyword(words[0], caseless=True).setDebug(bacula_tools.DEBUG) for w in words[1:]: p = p | Keyword(w, caseless=True).setDebug(bacula_tools.DEBUG) p = p + gr_eq + fn p.setParseAction(action) return p
def enumeration_type_definition() -> Token: enumeration_literal = unqualified_identifier() positional_enumeration = enumeration_literal + ZeroOrMore( comma() - enumeration_literal) positional_enumeration.setParseAction( lambda t: [(k, Number(v)) for v, k in enumerate(t.asList())]) element_value_association = enumeration_literal + Keyword( "=>") - numeric_literal() element_value_association.setParseAction(lambda t: (t[0], t[2])) named_enumeration = element_value_association + ZeroOrMore( comma() - element_value_association) boolean_literal = Keyword("True") | Keyword("False") boolean_literal.setParseAction(lambda t: t[0] == "True") boolean_aspect_definition = Optional(Keyword("=>") - boolean_literal) boolean_aspect_definition.setParseAction(lambda t: (t if t else ["=>", True])) always_valid_aspect = Literal("Always_Valid") - boolean_aspect_definition always_valid_aspect.setParseAction(parse_aspect) enumeration_aspects = Keyword("with") - delimitedList( size_aspect() | always_valid_aspect) enumeration_aspects.setParseAction(parse_aspects) return (Literal("(") - (named_enumeration | positional_enumeration) - Literal(")") - enumeration_aspects).setName("Enumeration")
def __init__(self): # speed up infixNotation considerably at the price of some cache memory ParserElement.enablePackrat() boolean = Keyword('True') | Keyword('False') none = Keyword('None') integer = Word(nums) real = Combine(Word(nums) + "." + Word(nums)) string = (QuotedString('"', escChar='\\') | QuotedString("'", escChar='\\')) regex = QuotedString('/', escChar='\\') identifier = Word(alphas, alphanums + '_') dereference = infixNotation(identifier, [ (Literal('.'), 2, opAssoc.LEFT, EvalArith), ]) result = (Keyword('bad') | Keyword('fail') | Keyword('good') | Keyword('ignore') | Keyword('unknown')) rval = boolean | none | real | integer | string | regex | result | dereference rvallist = Group( Suppress('[') + Optional(delimitedList(rval)) + Suppress(']')) rvalset = Group( Suppress('{') + Optional(delimitedList(rval)) + Suppress('}')) operand = rval | rvallist | rvalset # parse actions replace the parsed tokens with an instantiated object # which we can later call into for evaluation of its content boolean.setParseAction(EvalBoolean) none.setParseAction(EvalNone) integer.setParseAction(EvalInteger) real.setParseAction(EvalReal) string.setParseAction(EvalString) regex.setParseAction(EvalRegex) identifier.setParseAction(EvalIdentifier) result.setParseAction(EvalResult) rvallist.setParseAction(EvalList) rvalset.setParseAction(EvalSet) identity_test = Keyword('is') + ~Keyword('not') | Combine( Keyword('is') + Keyword('not'), adjacent=False, joinString=' ') membership_test = Keyword('in') | Combine( Keyword('not') + Keyword('in'), adjacent=False, joinString=' ') comparison_op = oneOf('< <= > >= != == isdisjoint') comparison = identity_test | membership_test | comparison_op self.parser = infixNotation(operand, [ (Literal('**'), 2, opAssoc.LEFT, EvalPower), (oneOf('+ - ~'), 1, opAssoc.RIGHT, EvalModifier), (oneOf('* / // %'), 2, opAssoc.LEFT, EvalArith), (oneOf('+ -'), 2, opAssoc.LEFT, EvalArith), (oneOf('<< >>'), 2, opAssoc.LEFT, EvalArith), (Literal('&'), 2, opAssoc.LEFT, EvalArith), (Literal('^'), 2, opAssoc.LEFT, EvalArith), (Literal('|'), 2, opAssoc.LEFT, EvalArith), (comparison, 2, opAssoc.LEFT, EvalLogic), (Keyword('not'), 1, opAssoc.RIGHT, EvalModifier), (Keyword('and'), 2, opAssoc.LEFT, EvalLogic), (Keyword('or'), 2, opAssoc.LEFT, EvalLogic), (Keyword('->'), 2, opAssoc.LEFT, EvalArith), ])
def parse_imp(input): # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # # <fact> ::= relation(constant, ...). # # <rule> ::= name(constant, ... ) :- name(constant, ...), ... . # idChars = alphas + "_+*!=<>" pNAME = Word(idChars, idChars + "0123456789") pRELATION = pNAME + \ "(" + Group(pNAME + ZeroOrMore(Suppress(",") + pNAME)) + ")" pRELATION.setParseAction(lambda result: (result[0], result[2])) pDECL_RELATION = pRELATION + "." pDECL_RELATION.setParseAction(lambda result: result[0]) pRULE = pRELATION + ":-" + \ Group(pRELATION + ZeroOrMore(Suppress(",") + pRELATION)) + "." pRULE.setParseAction(lambda result: (result[0], result[2])) pFACT = (pDECL_RELATION ^ NoMatch()) pQUERY = pRELATION + "?" pQUERY.setParseAction(lambda result: (result[0][0], result[0][1])) pTOP_QUERY = pQUERY.copy() pTOP_QUERY.setParseAction(lambda result: { "result": "query", "stmt": result[0] }) pTOP_RULE = pRULE.copy() pTOP_RULE.setParseAction(lambda result: {"result": "rule", "rule": result}) pTOP_FACT = pFACT.copy() pTOP_FACT.setParseAction(lambda result: { "result": "fact", "decl": result[0] }) pQUIT = Keyword("#quit") pQUIT.setParseAction(lambda result: {"result": "quit"}) pENV = Keyword("#env") pENV.setParseAction(lambda result: {"result": "env"}) pTOP = (pQUIT ^ pENV ^ pTOP_RULE ^ pTOP_FACT ^ pTOP_QUERY) result = pTOP.parseString(input)[0] return result # the first element of the result is the expression
def integer_type_definition() -> Token: range_type_aspects = Keyword("with") - size_aspect() range_type_aspects.setParseAction(parse_aspects) range_type_definition = (Keyword("range") - mathematical_expression() - Suppress(Literal("..")) - mathematical_expression() - range_type_aspects) range_type_definition.setName("RangeInteger") modular_type_definition = Keyword("mod") - mathematical_expression() modular_type_definition.setName("ModularInteger") return range_type_definition | modular_type_definition
def parse (input): # parse a string into an element of the abstract representation # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # ( if <expr> <expr> <expr> ) # ( let ( ( <name> <expr> ) ) <expr ) # ( + <expr> <expr> ) # ( * <expr> <expr> ) # idChars = alphas+"_+*-?!=<>" pIDENTIFIER = Word(idChars, idChars+"0123456789") pIDENTIFIER.setParseAction(lambda result: EId(result[0])) # A name is like an identifier but it does not return an EId... pNAME = Word(idChars,idChars+"0123456789") pINTEGER = Word("-0123456789","0123456789") pINTEGER.setParseAction(lambda result: EInteger(int(result[0]))) pBOOLEAN = Keyword("true") | Keyword("false") pBOOLEAN.setParseAction(lambda result: EBoolean(result[0]=="true")) pEXPR = Forward() pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")" pIF.setParseAction(lambda result: EIf(result[2],result[3],result[4])) pBINDING = "(" + pNAME + pEXPR + ")" pBINDING.setParseAction(lambda result: (result[1],result[2])) pLET = "(" + Keyword("let") + "(" + pBINDING + ")" + pEXPR + ")" pLET.setParseAction(lambda result: ELet([result[3]],result[5])) pPLUS = "(" + Keyword("+") + pEXPR + pEXPR + ")" pPLUS.setParseAction(lambda result: ECall("+",[result[2],result[3]])) pTIMES = "(" + Keyword("*") + pEXPR + pEXPR + ")" pTIMES.setParseAction(lambda result: ECall("*",[result[2],result[3]])) pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pIF | pLET | pPLUS | pTIMES) result = pEXPR.parseString(input)[0] return result # the first element of the result is the expression
def parse_string(self, string): '''Populate a new object from a string. Parsing is hard, so we're going to call out to the pyparsing library here. I hope you installed it! ''' from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith gr_eq = Literal('=') gr_stripped_string = quotedString.copy().setParseAction(removeQuotes) gr_opt_quoted_string = gr_stripped_string | restOfLine gr_name = Keyword('name', caseless=True) + gr_eq + gr_opt_quoted_string gr_name.setParseAction(lambda x, y=self: y._set_name(x[2])) gr_yn = Keyword('yes', caseless=True).setParseAction( replaceWith('1')) | Keyword('no', caseless=True).setParseAction( replaceWith('0')) gr_phrase = Group( OneOrMore(gr_stripped_string | Word(alphanums)) + gr_eq + gr_opt_quoted_string) def np(words, fn=gr_opt_quoted_string, action=print): p = Keyword(words[0], caseless=True) for w in words[1:]: p = p | Keyword(w, caseless=True) p = p + gr_eq + fn p.setParseAction(action) return p gr_ifsc = np(PList('Ignore File Set Changes'), gr_yn, action=self._parse_setter(IGNORECHANGES)) gr_evss = np(PList('Enable VSS'), gr_yn, action=self._parse_setter(VSSENABLED)) gr_i_option = Group( Keyword(OPTIONS, caseless=True) + nestedExpr('{', '}', Regex('[^\}]+', re.MULTILINE))) gr_e_option = gr_i_option.copy() gr_i_file = gr_phrase.copy() gr_e_file = gr_phrase.copy() gr_inc = Keyword('include', caseless=True) + nestedExpr( '{', '}', OneOrMore(gr_i_option | gr_i_file)) gr_inc.addParseAction(self._parse_add_entry) gr_exc = Keyword('exclude', caseless=True) + nestedExpr( '{', '}', OneOrMore(gr_e_option | gr_e_file)) gr_exc.addParseAction(self._parse_add_entry) gr_res = OneOrMore(gr_name | gr_inc | gr_exc | gr_ifsc | gr_evss) result = gr_res.parseString(string, parseAll=True) return 'Fileset: ' + self[NAME]
def message_type_definition() -> Token: first_aspect = Keyword("First") - Keyword("=>") - mathematical_expression() first_aspect.setParseAction(parse_aspect) length_aspect = Keyword("Length") - Keyword( "=>") - mathematical_expression() length_aspect.setParseAction(parse_aspect) component_aspects = Keyword("with") - delimitedList(first_aspect | length_aspect) component_aspects.setParseAction(parse_aspects) then = locatedExpr( Keyword("then") - (Keyword("null") | unqualified_identifier()) - Group(Optional(component_aspects)) - Group(Optional(value_constraint()))) then.setParseAction(parse_then) then_list = then + ZeroOrMore(comma() - then) then_list.setParseAction(lambda t: [t.asList()]) component_item = (~Keyword("end") + ~CaselessKeyword("Message") - unqualified_identifier() + Literal(":") - qualified_identifier() - Optional(then_list) - semicolon()) component_item.setParseAction(lambda t: Component(t[0], t[2], t[3]) if len(t) >= 4 else Component(t[0], t[2])) component_item.setName("Component") null_component_item = Keyword("null") - then - semicolon() null_component_item.setParseAction(lambda t: Component(None, None, [t[1]])) null_component_item.setName("NullComponent") component_list = Group( Optional(null_component_item) - component_item - ZeroOrMore(component_item)) component_list.setParseAction(lambda t: t.asList()) return (Keyword("message") - component_list - Keyword("end message") | Keyword("null message")).setName("Message")
def parse(input): # parse a string into an element of the abstract representation # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # ( if <expr> <expr> <expr> ) # ( let ( ( <name> <expr> ) ) <expr ) # ( + <expr> <expr> ) # ( * <expr> <expr> ) # idChars = alphas + "_+*-?!=<>" pIDENTIFIER = Word(idChars, idChars + "0123456789") pIDENTIFIER.setParseAction(lambda result: EId(result[0])) # A name is like an identifier but it does not return an EId... pNAME = Word(idChars, idChars + "0123456789") pINTEGER = Word("-0123456789", "0123456789") pINTEGER.setParseAction(lambda result: EInteger(int(result[0]))) pBOOLEAN = Keyword("true") | Keyword("false") pBOOLEAN.setParseAction(lambda result: EBoolean(result[0] == "true")) pEXPR = Forward() pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")" pIF.setParseAction(lambda result: EIf(result[2], result[3], result[4])) pBINDING = "(" + pNAME + pEXPR + ")" pBINDING.setParseAction(lambda result: (result[1], result[2])) pLET = "(" + Keyword("let") + "(" + pBINDING + ")" + pEXPR + ")" pLET.setParseAction(lambda result: ELet([result[3]], result[5])) pPLUS = "(" + Keyword("+") + pEXPR + pEXPR + ")" pPLUS.setParseAction(lambda result: ECall("+", [result[2], result[3]])) pTIMES = "(" + Keyword("*") + pEXPR + pEXPR + ")" pTIMES.setParseAction(lambda result: ECall("*", [result[2], result[3]])) pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pIF | pLET | pPLUS | pTIMES) result = pEXPR.parseString(input)[0] return result # the first element of the result is the expression
def build_parser(root_directory, path, fake_root=os.getcwd(), file_reader=None): from pyparsing import nestedExpr from pyparsing import QuotedString from pyparsing import Group from pyparsing import restOfLine from pyparsing import Word from pyparsing import alphanums from pyparsing import cStyleComment from pyparsing import OneOrMore from pyparsing import ZeroOrMore from pyparsing import Optional from pyparsing import Forward from pyparsing import Literal from pyparsing import Keyword root = Forward() include_handler = IncludeHandler( root_directory, path, root, fake_root=fake_root, file_reader=file_reader) # relaxed grammar identifier = Word(alphanums + "-_.:/") comment = ("//" + restOfLine).suppress() \ | ("#" + restOfLine).suppress() \ | cStyleComment endstmt = Literal(";").suppress() argument = QuotedString('"') \ | identifier arguments = ZeroOrMore(argument) statements = Forward() section = nestedExpr("{", "}", statements) include = Keyword("include").suppress() + QuotedString('"') regular = identifier + Group(arguments) + Optional(section, default=[]) statement = include.setParseAction(include_handler.pyparsing_call) \ | regular.setParseAction(include_handler.pyparsing_mark) statements << OneOrMore(statement + endstmt) root << Optional(statements) root.ignore(comment) setattr( root, 'parse_file', lambda f, root=root: root.parseFile(f, parseAll=True)) return root
def build(parsers: dict): comma = Literal(",") rb = Literal(")") lb = Literal("(") srb = Literal("]") slb = Literal("[") number = Regex(r"0|[1-9][0-9]*") string = quotedString() name = Word(alphanums) label = Keyword(STRONG) | Keyword(WEAK) | Literal(SHORT_WEAK) param = Combine(Keyword(PARAM) + slb + number + srb) marker = Keyword(RESULT) | Keyword(TRUE) | Keyword(FALSE) | Keyword( THIS) | Keyword(_THIS) | param function = Keyword(GET) get = Literal(GETATTR) operator1 = Literal(MUL) | Literal(DIV) | Literal(MOD) operator2 = Literal(ADD) | Literal(SUB) operator3 = Literal(EQUAL) | Literal(NOT_EQUAL) operator3 |= And(Keyword(word) for word in IS_NOT.split(" ")) | Keyword(IS) operator4 = Literal(GREATER_OR_EQUAL) | Literal(GREATER) | Literal( LOWER_OR_EQUAL) | Literal(LOWER) operator5 = Keyword(AND) operator6 = Keyword(OR) operator7 = Keyword(FOLLOW) expression = Forward() string_st = string.setParseAction(parsers[STRING]) name_st = name.setParseAction(parsers[STRING]) marker_st = marker.setParseAction(parsers[MARKER]) tuple_st = expression + ZeroOrMore(comma + expression) round_invocation_st = (lb + Optional(tuple_st) + rb).setParseAction( parsers[INVOCATION]) function_st = (function + Suppress(round_invocation_st)).setParseAction( parsers[FUNCTION]) getattr_st = (marker_st | name_st) + OneOrMore( (get + Suppress(name_st)).setParseAction(parsers[OPERATOR])) atom_st = (lb + expression + rb) | function_st | string_st | getattr_st | marker_st operator_st = atom_st + ZeroOrMore( (operator1 + Suppress(atom_st)).setParseAction(parsers[OPERATOR])) operator_st = operator_st + ZeroOrMore( (operator2 + Suppress(operator_st)).setParseAction(parsers[OPERATOR])) operator_st = operator_st + ZeroOrMore( (operator3 + Suppress(operator_st)).setParseAction(parsers[OPERATOR])) operator_st = operator_st + ZeroOrMore( (operator4 + Suppress(operator_st)).setParseAction(parsers[OPERATOR])) operator_st = operator_st + ZeroOrMore( (operator5 + Suppress(operator_st)).setParseAction(parsers[OPERATOR])) operator_st = operator_st + ZeroOrMore( (operator6 + Suppress(operator_st)).setParseAction(parsers[OPERATOR])) operator_st = operator_st + ZeroOrMore( (operator7 + Suppress(operator_st)).setParseAction(parsers[OPERATOR])) expression << operator_st getattr_st.enablePackrat() statement = (Optional(label, STRONG) + Suppress(expression)).setParseAction(parsers[LABEL]) return ZeroOrMore(statement) + StringEnd()
def ParseExpression(cls, source): # atoms boolean = Keyword('#f') | Keyword('#t') boolean.setParseAction(lambda s, l, t: SchemeExpression.make_boolean(t[ 0] == '#t').expression) symbol = Word(alphanums + '-_') symbol.setParseAction( lambda s, l, t: SchemeExpression.make_symbol(t[0]).expression) integer = Word(nums) integer.setParseAction( lambda s, l, t: SchemeExpression.make_integer(t[0]).expression) string = QuotedString('"', multiline=True) string.setParseAction( lambda s, l, t: SchemeExpression.make_string(t[0]).expression) element = integer | boolean | symbol | string # lists lexpr = Forward() vexpr = Forward() lparen = Literal('(').suppress() rparen = Literal(')').suppress() hashsym = Literal('#').suppress() # vectors lexpr << Group(lparen + ZeroOrMore(element ^ lexpr ^ vexpr) + rparen) lexpr.setParseAction(lambda s, l, t: SchemeExpression.make_list(t[0])) vexpr << Group(hashsym + lparen + ZeroOrMore(element ^ lexpr ^ vexpr) + rparen) vexpr.setParseAction( lambda s, l, t: SchemeExpression.make_vector(t[0])) # final... sexpr = element | vexpr | lexpr sexpr.keepTabs = True # this seems to be necessary to fix a problem with pyparsing result = sexpr.parseString(source)[0] return cls(SchemeExpression._flatten(result))
def ParseExpression(cls, source): # atoms boolean = Keyword('#f') | Keyword('#t') boolean.setParseAction(lambda s, l, t: SchemeExpression.make_boolean(t[ 0] == '#t').expression) symbol = Word(alphanums + '-_') symbol.setParseAction( lambda s, l, t: SchemeExpression.make_symbol(t[0]).expression) integer = Word(nums) integer.setParseAction( lambda s, l, t: SchemeExpression.make_integer(t[0]).expression) string = dblQuotedString string.setParseAction(lambda s, l, t: SchemeExpression.make_string(t[ 0][1:-1]).expression) element = integer | boolean | symbol | dblQuotedString # lists lexpr = Forward() vexpr = Forward() lparen = Literal('(').suppress() rparen = Literal(')').suppress() hashsym = Literal('#').suppress() # vectors lexpr << Group(lparen + ZeroOrMore(element ^ lexpr ^ vexpr) + rparen) lexpr.setParseAction(lambda s, l, t: SchemeExpression.make_list(*t[0])) vexpr << Group(hashsym + lparen + ZeroOrMore(element ^ lexpr ^ vexpr) + rparen) vexpr.setParseAction( lambda s, l, t: SchemeExpression.make_vector(*t[0])) # final... sexpr = element | vexpr | lexpr result = sexpr.parseString(source)[0] return cls(SchemeExpression._flatten(result))
def parse_string(self, string): '''Populate a new object from a string. Parsing is hard, so we're going to call out to the pyparsing library here. I hope you installed it! ''' from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith gr_eq = Literal('=') gr_stripped_string = quotedString.copy().setParseAction( removeQuotes ) gr_opt_quoted_string = gr_stripped_string | restOfLine gr_name = Keyword('name', caseless=True) + gr_eq + gr_opt_quoted_string gr_name.setParseAction(lambda x, y=self: y._set_name(x[2])) gr_yn = Keyword('yes', caseless=True).setParseAction(replaceWith('1')) | Keyword('no', caseless=True).setParseAction(replaceWith('0')) gr_phrase = Group(OneOrMore(gr_stripped_string | Word(alphanums)) + gr_eq + gr_opt_quoted_string) def np(words, fn = gr_opt_quoted_string, action=print): p = Keyword(words[0], caseless=True) for w in words[1:]: p = p | Keyword(w, caseless=True) p = p + gr_eq + fn p.setParseAction(action) return p gr_ifsc = np(PList('Ignore File Set Changes'), gr_yn, action=self._parse_setter(IGNORECHANGES)) gr_evss = np(PList('Enable VSS'), gr_yn, action=self._parse_setter(VSSENABLED)) gr_i_option = Group(Keyword(OPTIONS, caseless=True) + nestedExpr('{','}', Regex('[^\}]+', re.MULTILINE))) gr_e_option = gr_i_option.copy() gr_i_file = gr_phrase.copy() gr_e_file = gr_phrase.copy() gr_inc = Keyword('include', caseless=True) + nestedExpr('{','}', OneOrMore(gr_i_option | gr_i_file)) gr_inc.addParseAction(self._parse_add_entry) gr_exc = Keyword('exclude', caseless=True) + nestedExpr('{','}', OneOrMore(gr_e_option | gr_e_file)) gr_exc.addParseAction(self._parse_add_entry) gr_res = OneOrMore(gr_name | gr_inc | gr_exc | gr_ifsc | gr_evss) result = gr_res.parseString(string, parseAll=True) return 'Fileset: ' + self[NAME]
def __init__(self): FALSE = Keyword("false") NULL = Keyword("null") TRUE = Keyword("true") FALSE.setParseAction(replaceWith(False)) NULL.setParseAction(replaceWith(None)) TRUE.setParseAction(replaceWith(True)) pattern = Forward() label = Word(alphas, alphanums + "_").setResultsName("layer_name") configurable_param = nestedExpr(content=pattern) arg = (NULL ^ FALSE ^ TRUE ^ pyparsing_common.number ^ (Word(alphanums + "*_") + ~Word("=")) ^ configurable_param) args = arg[...].setResultsName("args") args.setParseAction(self.convert_list) options = Dict(Group(Word(alphanums + "_") + Suppress("=") + arg))[...].setResultsName("options") options.setParseAction(self.convert_dict) pattern <<= label + args + options pattern.setParseAction(Pattern) self.pattern = pattern
statedef.extend([ "class %s(object):" % baseStateClass, " def __str__(self):", " return self.__class__.__name__", " def next_state(self):", " return self._next_state_class()" ]) # define all state classes statedef.extend("class {}({}): pass".format(s, baseStateClass) for s in states) statedef.extend("{}._next_state_class = {}".format(s, fromTo[s]) for s in states if s in fromTo) return indent + ("\n" + indent).join(statedef) + "\n" stateMachine.setParseAction(expand_state_definition) def expand_named_state_definition(source, loc, tokens): indent = " " * (col(loc, source) - 1) statedef = [] # build list of states and transitions states = set() transitions = set() baseStateClass = tokens.name + "State" fromTo = {} for tn in tokens.transitions: states.add(tn.fromState) states.add(tn.toState)
def _parse_line(self): """ Parses a single line, and returns a node representing the active context Further lines processed are expected to be children of the active context, or children of its accestors. ------------------------------------------------ Basic grammar is as follows: line = <mako>|<nemo>|<string> <mako> We don't parse normally parse tags, so the following info is sketchy. Mako tags are recognized as anythign that starts with: - <% - %> - %CLOSETEXT - </% Mako Control tags however are parsed, and required to adhere to the same indentation rules as Nemo tags. mako_control = <start>|<middle>|<end> start = (for|if|while) <inner>: middle = (else|elif): end = endfor|endwhile nemo = % ( <mako_control>|<nemo_statement> ) nemo_statement = .<quote><string><quote>|#<quote><string><quote>|<words> <quote> = '|" Notes: Quotes are required to be balanced. Quotes preceded by a \ are ignored. <string> = * words = \w+ """ #if self.debug: print '\t ' + str(self._current_node) # PyParser setParseAction's actually execute during parsing, # So we need closures in order to change the current scope def depth_from_indentation(function): """ Set the depth as the start of the match """ def wrap(start, values): #print 'Depth %d | %d %s' %(self._depth, start, values) #self._depth = start self._current_node = function(values) #print self._current_node return '' return wrap def depth_from_match(function): """ Set the depth as the start of the match """ def wrap(start, values): #print 'Depth %d | %d %s' %(self._depth, start, values) #print self._current_node self._depth = start self._current_node = function(values) #print self._current_node return '' return wrap def depth_from_nemo_tag(function): """ Start of the match is where the nemo tag is. Pass the other values to the wrapped function """ def wrap(start, values): # print 'Depth %d | %d %s' %(self._depth, start, values) self._depth = start tokens = values[1] self._current_node = function(tokens) #print self._current_node return '' return wrap # Match HTML from pyparsing import NotAny, MatchFirst html = restOfLine html.setParseAction(depth_from_indentation(self._add_html_node)) # Match Mako control tags nemo_tag = Literal('%') begin = Keyword('for') | Keyword('if') | Keyword('while') middle = Keyword('else') | Keyword('elif') end = Keyword('endfor') | Keyword('endif') | Keyword('endwhile') control = nemo_tag + (begin | middle | end) begin.setParseAction(depth_from_indentation(self._add_nesting_mako_control_node) ) middle.setParseAction(depth_from_indentation(self._add_mako_middle_node)) end.setParseAction(depth_from_indentation(self._add_mako_control_leaf)) # Match Nemo tags argument_name = Word(alphas,alphanums+"_-:") argument_value = quotedString regular_argument = argument_name + Literal('=') + argument_value class_name = Literal('.').setParseAction(lambda x: 'class=') id_name = Literal('#').setParseAction(lambda x: 'id=') special_argument = (class_name | id_name) + argument_value argument = Combine(special_argument) | Combine(regular_argument) # Match single Nemo statement (Part of a multi-line) inline_nemo_html = Word(alphas) + Group(ZeroOrMore(argument)) inline_nemo_html.setParseAction(depth_from_match(self._add_nemo_node)) # Match first nemo tag on the line (the one that may begin a multi-statement expression) nemo_html = nemo_tag + Group(Word(alphanums+"_-:") + Group(ZeroOrMore(argument))) nemo_html.setParseAction(depth_from_nemo_tag(self._add_nemo_node)) # Match a multi-statement expression. Nemo statements are seperated by |. Anything after || is treated as html separator = Literal('|').suppress() html_separator = Literal('||') # | Literal('|>') nemo_list = nemo_html + ZeroOrMore( separator + inline_nemo_html ) inline_html = html.copy() inline_html.setParseAction(depth_from_match(self._add_inline_html_node)) nemo_multi = nemo_list + Optional(html_separator + inline_html) # Match empty Nemo statement empty = nemo_tag + Empty() empty.setParseAction(depth_from_indentation(self._add_blank_nemo_node)) # Match unused Mako tags mako_tags = Literal('<%') | Literal('%>') | Literal('%CLOSETEXT') | Literal('</%') mako = mako_tags mako_tags.setParseAction(depth_from_indentation(self._add_html_node)) # Matches General nemo = (control | nemo_multi | empty) line = mako_tags | nemo | html # Depth Calculation (deprecated?) self._depth = len(self._c) - len(self._c.strip()) #try: line.parseString(self._c)
def SPICE_BNF(): global bnf if not bnf: # punctuation colon = Literal(":").suppress() lbrace = Literal("{").suppress() rbrace = Literal("}").suppress() lbrack = Literal("[").suppress() rbrack = Literal("]").suppress() lparen = Literal("(").suppress() rparen = Literal(")").suppress() equals = Literal("=").suppress() comma = Literal(",").suppress() semi = Literal(";").suppress() # primitive types int8_ = Keyword("int8").setParseAction(replaceWith(ptypes.int8)) uint8_ = Keyword("uint8").setParseAction(replaceWith(ptypes.uint8)) int16_ = Keyword("int16").setParseAction(replaceWith(ptypes.int16)) uint16_ = Keyword("uint16").setParseAction(replaceWith(ptypes.uint16)) int32_ = Keyword("int32").setParseAction(replaceWith(ptypes.int32)) uint32_ = Keyword("uint32").setParseAction(replaceWith(ptypes.uint32)) int64_ = Keyword("int64").setParseAction(replaceWith(ptypes.int64)) uint64_ = Keyword("uint64").setParseAction(replaceWith(ptypes.uint64)) # keywords channel_ = Keyword("channel") enum32_ = Keyword("enum32").setParseAction(replaceWith(32)) enum16_ = Keyword("enum16").setParseAction(replaceWith(16)) enum8_ = Keyword("enum8").setParseAction(replaceWith(8)) flags32_ = Keyword("flags32").setParseAction(replaceWith(32)) flags16_ = Keyword("flags16").setParseAction(replaceWith(16)) flags8_ = Keyword("flags8").setParseAction(replaceWith(8)) channel_ = Keyword("channel") server_ = Keyword("server") client_ = Keyword("client") protocol_ = Keyword("protocol") typedef_ = Keyword("typedef") struct_ = Keyword("struct") message_ = Keyword("message") image_size_ = Keyword("image_size") bytes_ = Keyword("bytes") cstring_ = Keyword("cstring") switch_ = Keyword("switch") default_ = Keyword("default") case_ = Keyword("case") identifier = Word(alphas, alphanums + "_") enumname = Word(alphanums + "_") integer = ( Combine(CaselessLiteral("0x") + Word(nums + "abcdefABCDEF")) | Word(nums + "+-", nums)).setName("int").setParseAction(cvtInt) typename = identifier.copy().setParseAction( lambda toks: ptypes.TypeRef(str(toks[0]))) # This is just normal "types", i.e. not channels or messages typeSpec = Forward() attributeValue = integer ^ identifier attribute = Group( Combine("@" + identifier) + Optional(lparen + delimitedList(attributeValue) + rparen)) attributes = Group(ZeroOrMore(attribute)) arraySizeSpecImage = Group(image_size_ + lparen + integer + comma + identifier + comma + identifier + rparen) arraySizeSpecBytes = Group(bytes_ + lparen + identifier + comma + identifier + rparen) arraySizeSpecCString = Group(cstring_ + lparen + rparen) arraySizeSpec = lbrack + Optional( identifier ^ integer ^ arraySizeSpecImage ^ arraySizeSpecBytes ^ arraySizeSpecCString, default="") + rbrack variableDef = Group(typeSpec + Optional("*", default=None) + identifier + Optional(arraySizeSpec, default=None) + attributes - semi) \ .setParseAction(parseVariableDef) switchCase = Group(Group(OneOrMore(default_.setParseAction(replaceWith(None)) + colon | Group(case_.suppress() + Optional("!", default="") + identifier) + colon)) + variableDef) \ .setParseAction(lambda toks: ptypes.SwitchCase(toks[0][0], toks[0][1])) switchBody = Group(switch_ + lparen + delimitedList(identifier,delim='.', combine=True) + rparen + lbrace + Group(OneOrMore(switchCase)) + rbrace + identifier + attributes - semi) \ .setParseAction(lambda toks: ptypes.Switch(toks[0][1], toks[0][2], toks[0][3], toks[0][4])) messageBody = structBody = Group(lbrace + ZeroOrMore(variableDef | switchBody) + rbrace) structSpec = Group(struct_ + identifier + structBody + attributes).setParseAction( lambda toks: ptypes.StructType( toks[0][1], toks[0][2], toks[0][3])) # have to use longest match for type, in case a user-defined type name starts with a keyword type, like "channel_type" typeSpec << (structSpec ^ int8_ ^ uint8_ ^ int16_ ^ uint16_ ^ int32_ ^ uint32_ ^ int64_ ^ uint64_ ^ typename).setName("type") flagsBody = enumBody = Group( lbrace + delimitedList(Group(enumname + Optional(equals + integer))) + Optional(comma) + rbrace) messageSpec = Group(message_ + messageBody + attributes ).setParseAction(lambda toks: ptypes.MessageType( None, toks[0][1], toks[0][2])) | typename channelParent = Optional(colon + typename, default=None) channelMessage = Group(messageSpec + identifier + Optional(equals + integer, default=None) + semi) \ .setParseAction(lambda toks: ptypes.ChannelMember(toks[0][1], toks[0][0], toks[0][2])) channelBody = channelParent + Group(lbrace + ZeroOrMore( server_ + colon | client_ + colon | channelMessage) + rbrace) enum_ = (enum32_ | enum16_ | enum8_) flags_ = (flags32_ | flags16_ | flags8_) enumDef = Group(enum_ + identifier + enumBody + attributes - semi).setParseAction(lambda toks: ptypes.EnumType( toks[0][0], toks[0][1], toks[0][2], toks[0][3])) flagsDef = Group(flags_ + identifier + flagsBody + attributes - semi).setParseAction(lambda toks: ptypes.FlagsType( toks[0][0], toks[0][1], toks[0][2], toks[0][3])) messageDef = Group(message_ + identifier + messageBody + attributes - semi).setParseAction( lambda toks: ptypes.MessageType( toks[0][1], toks[0][2], toks[0][3])) channelDef = Group(channel_ + identifier + channelBody - semi).setParseAction( lambda toks: ptypes.ChannelType( toks[0][1], toks[0][2], toks[0][3])) structDef = Group(struct_ + identifier + structBody + attributes - semi).setParseAction(lambda toks: ptypes.StructType( toks[0][1], toks[0][2], toks[0][3])) typedefDef = Group(typedef_ + identifier + typeSpec + attributes - semi).setParseAction(lambda toks: ptypes.TypeAlias( toks[0][1], toks[0][2], toks[0][3])) definitions = typedefDef | structDef | enumDef | flagsDef | messageDef | channelDef protocolChannel = Group(typename + identifier + Optional(equals + integer, default=None) + semi) \ .setParseAction(lambda toks: ptypes.ProtocolMember(toks[0][1], toks[0][0], toks[0][2])) protocolDef = Group(protocol_ + identifier + Group(lbrace + ZeroOrMore(protocolChannel) + rbrace) + semi) \ .setParseAction(lambda toks: ptypes.ProtocolType(toks[0][1], toks[0][2])) bnf = ZeroOrMore(definitions) + protocolDef + StringEnd() singleLineComment = "//" + restOfLine bnf.ignore(singleLineComment) bnf.ignore(cStyleComment) return bnf
def init_grammar(self): """Set up the parsing classes Any changes to the grammar of the config file be done here. """ # Some syntax that we need, but don't care about SEMICOLON = (Suppress(";")) EQUALS = Suppress("=") # Top Section FILE_NAME = Word(alphas + nums + '-_.') alignment_def = Keyword('alignment') + EQUALS\ + FILE_NAME + SEMICOLON alignment_def.setParseAction(self.set_alignment) tree_def = Keyword('user_tree_topology') + EQUALS\ + FILE_NAME + SEMICOLON tree_def.setParseAction(self.set_user_tree) def simple_option(name): opt = Keyword(name) + EQUALS +\ Word(alphas + nums + '-_') + SEMICOLON opt.setParseAction(self.set_simple_option) return opt branch_def = simple_option('branchlengths') MODEL_NAME = Word(alphas + nums + '+' + ' ' + '_') model_list = delimitedList(MODEL_NAME) model_def = 'models' + EQUALS + model_list + SEMICOLON model_def.setParseAction(self.set_models) model_selection_def = simple_option("model_selection") top_section = alignment_def + Optional(tree_def) + branch_def + \ model_def + model_selection_def # Data Block Parsing column = Word(nums) block_name = Word(alphas + '_-' + nums) block_def = column("start") +\ Optional(Suppress("-") + column("end")) +\ Optional(Suppress("\\") + column("step")) block_def.setParseAction(self.define_range) block_list_def = Group(OneOrMore(Group(block_def))) user_subset_def = Optional("charset") + block_name("name") + \ EQUALS + block_list_def("parts") + SEMICOLON user_subset_def.setParseAction(self.define_user_subset) block_def_list = OneOrMore(Group(user_subset_def)) block_section = Suppress("[data_blocks]") + block_def_list block_def_list.setParseAction(self.check_blocks) # Scheme Parsing scheme_name = Word(alphas + '_-' + nums) # Make a copy, cos we set a different action on it user_subset_ref = block_name.copy() user_subset_ref.setParseAction(self.check_block_exists) subset = Group(Suppress("(") + delimitedList(user_subset_ref("name")) + Suppress(")")) subset.setParseAction(self.define_subset_grouping) scheme = Group(OneOrMore(subset)) scheme_def = scheme_name("name") + \ EQUALS + scheme("scheme") + SEMICOLON scheme_def.setParseAction(self.define_scheme) scheme_list = OneOrMore(Group(scheme_def)) scheme_algo = simple_option("search") scheme_section = \ Suppress("[schemes]") + scheme_algo + Optional(scheme_list) # We've defined the grammar for each section. # Here we just put it all together self.config_parser = ( top_section + block_section + scheme_section + stringEnd)
GT = Literal('>') GTE = Literal('>=') LT = Literal('<') LTE = Literal('<=') one_sided_range = Group((GTE | GT | LTE | LT)('op') + valid_word('bound'))('onesidedrange') term = (_range | one_sided_range | regex | wildcard | phrase | single_term) clause << (Optional(field_name + COLON, default='__default_field__')('field') + (term('term') | Group(LPAR + query_expr + RPAR)('subquery'))) clause.addParseAction(SearchTerm) query_expr << infixNotation(clause, [ (required_modifier | prohibit_modifier, 1, opAssoc.RIGHT, SearchModifier), (NOT.setParseAction(lambda: 'NOT'), 1, opAssoc.RIGHT, SearchNot), (AND.setParseAction(lambda: 'AND'), 2, opAssoc.LEFT, SearchAnd), (Optional(OR).setParseAction(lambda: 'OR'), 2, opAssoc.LEFT, SearchOr), ]) class QueryParser: DEFAULT_FIELD = 'text' def parse(self, query, default_field=None): default_field = default_field or QueryParser.DEFAULT_FIELD return repr(query_expr.parseString(query)[0]).replace( '__default_field__', default_field)
" def __str__(self):", " return self.__class__.__name__", " def next_state(self):", " return self._next_state_class()" ]) # define all state classes statedef.extend( "class %s(%s): pass" % (s,baseStateClass) for s in states ) statedef.extend( "%s._next_state_class = %s" % (s,fromTo[s]) for s in states if s in fromTo ) return indent + ("\n"+indent).join(statedef)+"\n" stateMachine.setParseAction(expand_state_definition) def expand_named_state_definition(source,loc,tokens): indent = " " * (col(loc,source)-1) statedef = [] # build list of states and transitions states = set() transitions = set() baseStateClass = tokens.name + "State" fromTo = {} for tn in tokens.transitions: states.add(tn.fromState) states.add(tn.toState) transitions.add(tn.transition)
def parse (input): # parse a string into an element of the abstract representation # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # ( if <expr> <expr> <expr> ) # ( let ( ( <name> <expr> ) ) <expr ) # ( function ( <name> ... ) <expr> ) # ( ref <expr> ) # ( <expr> <expr> ... ) # idChars = alphas+"_+*-?!=<>" pIDENTIFIER = Word(idChars, idChars+"0123456789") pIDENTIFIER.setParseAction(lambda result: EId(result[0])) # A name is like an identifier but it does not return an EId... pNAME = Word(idChars,idChars+"0123456789") pNAMES = ZeroOrMore(pNAME) pNAMES.setParseAction(lambda result: [result]) pINTEGER = Word("0123456789") pINTEGER.setParseAction(lambda result: EValue(VInteger(int(result[0])))) pBOOLEAN = Keyword("true") | Keyword("false") pBOOLEAN.setParseAction(lambda result: EValue(VBoolean(result[0]=="true"))) pEXPR = Forward() pEXPRS = ZeroOrMore(pEXPR) pEXPRS.setParseAction(lambda result: [result]) pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")" pIF.setParseAction(lambda result: EIf(result[2],result[3],result[4])) pBINDING = "(" + pNAME + pEXPR + ")" pBINDING.setParseAction(lambda result: (result[1],result[2])) pBINDINGS = ZeroOrMore(pBINDING) pBINDINGS.setParseAction(lambda result: [ result ]) pLET = "(" + Keyword("let") + "(" + pBINDINGS + ")" + pEXPR + ")" pLET.setParseAction(lambda result: ELet(result[3],result[5])) pCALL = "(" + pEXPR + pEXPRS + ")" pCALL.setParseAction(lambda result: ECall(result[1],result[2])) pFUN = "(" + Keyword("function") + "(" + pNAMES + ")" + pEXPR + ")" pFUN.setParseAction(lambda result: EFunction(result[3],result[5])) pFUNrec = "(" + Keyword("function") + pNAME + "(" + pNAMES + ")" + pEXPR + ")" pFUNrec.setParseAction(lambda result: EFunction(result[4],result[6],name=result[2])) pCLASS = "(" + Keyword("class") + "(" + pNAMES + ")" + Keyword("(") + pBINDINGS + ")" + Keyword("(") + pBINDINGS + Keyword(")") + ")" pCLASS.setParseAction(lambda result: EClass(result[3],result[6],result[9])) pNEW = "(" + Keyword("new") + pEXPR + pEXPRS +")" pNEW.setParseAction(lambda result: ENew(result[2],result[3])) pWITH = "(" + Keyword("with") + pEXPR + pEXPR +")" pWITH.setParseAction(lambda result: EWithObj(result[2],result[3])) pDO = "(" + Keyword("do") + pEXPRS + ")" pDO.setParseAction(lambda result: EDo(result[2])) pWHILE = "(" + Keyword("while") + pEXPR + pEXPR + ")" pWHILE.setParseAction(lambda result: EWhile(result[2],result[3])) pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pIF | pLET | pFUN | pFUNrec| pCLASS | pNEW | pWITH | pDO | pWHILE | pCALL) # can't attach a parse action to pEXPR because of recursion, so let's duplicate the parser pTOPEXPR = pEXPR.copy() pTOPEXPR.setParseAction(lambda result: {"result":"expression","expr":result[0]}) pDEFINE = "(" + Keyword("define") + pNAME + pEXPR + ")" pDEFINE.setParseAction(lambda result: {"result":"value", "name":result[2], "expr":result[3]}) pDEFUN = "(" + Keyword("defun") + pNAME + "(" + pNAMES + ")" + pEXPR + ")" pDEFUN.setParseAction(lambda result: {"result":"function", "name":result[2], "params":result[4], "body":result[6]}) pABSTRACT = "#abs" + pEXPR pABSTRACT.setParseAction(lambda result: {"result":"abstract", "expr":result[1]}) pQUIT = Keyword("#quit") pQUIT.setParseAction(lambda result: {"result":"quit"}) pTOP = (pDEFUN | pDEFINE | pQUIT | pABSTRACT | pTOPEXPR) result = pTOP.parseString(input)[0] return result # the first element of the result is the expression
def parse_natural (input): # parse a string into an element of the abstract representation # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # ( expr ) # <expr> ? <expr> : <expr> # let ( <bindings> ) <expr> # <expr> + <expr> # <expr> * <expr> # <expr> - <expr> # <name> ( <expr-seq> ) # # <bindings> ::= <name> = <expr> , <bindings> # <name> = <expr> # # <expr-seq> ::= <expr> , <expr-seq> # <expr> idChars = alphas+"_+*-?!=<>" pIDENTIFIER = Word(idChars, idChars+"0123456789") pIDENTIFIER.setParseAction(lambda result: EId(result[0])) # A name is like an identifier but it does not return an EId... pNAME = Word(idChars,idChars+"0123456789") pINTEGER = Word("-0123456789","0123456789") pINTEGER.setParseAction(lambda result: EInteger(int(result[0]))) pBOOLEAN = Keyword("true") | Keyword("false") pBOOLEAN.setParseAction(lambda result: EBoolean(result[0]=="true")) pEXPR = Forward() pEXPR_META = Forward() pEXPR_REST = Forward() pSINGLE_EXPR = "(" + pEXPR_META + ")" pSINGLE_EXPR.setParseAction(lambda result:result[1]) pBINDING = pNAME + Keyword("=") + pEXPR pBINDING.setParseAction(lambda result: (result[0], result[2])) pMULT_BINDING = pBINDING + ZeroOrMore(Suppress(",") + pBINDING) pMULT_BINDING.setParseAction(lambda result: [(r[0],r[1]) for r in result]) pLET = Keyword("let") + "(" + pMULT_BINDING + ")" + pEXPR pLET.setParseAction(lambda result:pLET_exps_unpack_nat(result)) pIF = pEXPR + Keyword("?") + pEXPR + Keyword(":") + pEXPR pIF.setParseAction(lambda result: EIf(result[0], result[2], result[4])) pUSR_FUNC = pNAME + "(" + pEXPR + ZeroOrMore(Suppress(",") + pEXPR) + ")" pUSR_FUNC.setParseAction(lambda result: ECall(result[0], result[2:-1])) pEXPR_FIRST = pINTEGER | pIDENTIFIER | pSINGLE_EXPR | pLET | pUSR_FUNC pALGEBRA = pEXPR_FIRST + pEXPR_REST pALGEBRA.setParseAction(lambda result: ECall(result[1], [result[0], result[2]])) pPLUS = Keyword("+") + pEXPR pPLUS.setParseAction(lambda result: result) pTIMES = Keyword("*") + pEXPR pTIMES.setParseAction(lambda result: result) pMINUS = Keyword("-") + pEXPR pMINUS.setParseAction(lambda result: result) pEXPR_REST << (pTIMES | pPLUS | pMINUS) pEXPR << (pLET | pUSR_FUNC | pALGEBRA | pINTEGER | pBOOLEAN | pIDENTIFIER | pNAME | pSINGLE_EXPR) pEXPR_META << (pIF | pEXPR) result = pEXPR_META.parseString(input)[0] if type(result) == dict: return {"result":"function", "name":result["name"], "params":result["params"], "body":result["body"]} else: return {"result":"expression", "expr":result}
def create_bnf(allow_tuple=False, free_word=False): cvt_int = lambda toks: int(toks[0]) cvt_real = lambda toks: float(toks[0]) cvt_bool = lambda toks: toks[0].lower == 'true' cvt_none = lambda toks: [None] cvt_tuple = lambda toks : tuple(toks.asList()) cvt_dict = lambda toks: dict(toks.asList()) # define punctuation as suppressed literals (lparen, rparen, lbrack, rbrack, lbrace, rbrace, colon) = map(Suppress,"()[]{}:") integer = Combine(Optional(oneOf("+ -")) + Word(nums)).setName("integer") integer.setParseAction(cvt_int) boolean = Keyword("False", caseless = True) boolean.setParseAction(cvt_bool) none = Keyword("None", caseless = True) none.setParseAction(cvt_none) real = Combine(Optional(oneOf("+ -"))+ Word(nums) + "." + Optional(Word(nums)) + Optional("e" + Optional(oneOf("+ -")) + Word(nums))).setName("real") real.setParseAction(cvt_real) tuple_str = Forward() list_str = Forward() dict_str = Forward() if free_word: string = Word(alphas8bit + "_-/.+**" + alphanums) else: string = Word(alphas8bit + alphas, alphas8bit + alphanums + "_" ) list_item = (none | boolean | real | integer | list_str | tuple_str | dict_str | quotedString.setParseAction(removeQuotes) | string ) list_item2 = list_item | Empty().setParseAction(lambda: [None]) tuple_inner = Optional(delimitedList(list_item)) + Optional(Suppress(",")) tuple_inner.setParseAction(cvt_tuple) tuple_str << (Suppress("(") + tuple_inner + Suppress(")")) list_inner = Optional(delimitedList(list_item) + Optional(Suppress(","))) list_inner.setParseAction(lambda toks: list(toks)) list_str << (lbrack + list_inner + rbrack) dict_entry = Group(list_item + colon + list_item2) dict_inner = delimitedList(dict_entry) + Optional(Suppress(",")) dict_inner.setParseAction(cvt_dict) dict_str << (lbrace + Optional(dict_inner) + rbrace) dict_or_tuple = dict_inner | tuple_inner if allow_tuple: return dict_or_tuple else: return dict_inner
def parse (input): # parse a string into an element of the abstract representation # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # ( if <expr> <expr> <expr> ) # ( let ( ( <name> <expr> ) ) <expr> ) # ( <name> <expr> ... ) # idChars = alphas+"_+*-?!=<>" pIDENTIFIER = Word(idChars, idChars+"0123456789") pIDENTIFIER.setParseAction(lambda result: EId(result[0])) # A name is like an identifier but it does not return an EId... pNAME = Word(idChars,idChars+"0123456789") pNAMES = ZeroOrMore(pNAME) pNAMES.setParseAction(lambda result: [result]) pINTEGER = Word("-0123456789","0123456789") pINTEGER.setParseAction(lambda result: EInteger(int(result[0]))) pBOOLEAN = Keyword("true") | Keyword("false") pBOOLEAN.setParseAction(lambda result: EBoolean(result[0]=="true")) pEXPR = Forward() pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")" pIF.setParseAction(lambda result: EIf(result[2],result[3],result[4])) pBINDING = "(" + pNAME + pEXPR + ")" pBINDING.setParseAction(lambda result: (result[1],result[2])) pBINDINGS = OneOrMore(pBINDING) pBINDINGS.setParseAction(lambda result: [ result ]) pLET = "(" + Keyword("let") + "(" + pBINDINGS + ")" + pEXPR + ")" pLET.setParseAction(lambda result: ELet(result[3],result[5])) pEXPRS = ZeroOrMore(pEXPR) pEXPRS.setParseAction(lambda result: [result]) pCALL = "(" + pNAME + pEXPRS + ")" pCALL.setParseAction(lambda result: ECall(result[1],result[2])) pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pIF | pLET | pCALL) # can't attach a parse action to pEXPR because of recursion, so let's duplicate the parser pTOPEXPR = pEXPR.copy() pTOPEXPR.setParseAction(lambda result: {"result":"expression","expr":result[0]}) pDEFUN = "(" + Keyword("defun") + pNAME + "(" + pNAMES + ")" + pEXPR + ")" pDEFUN.setParseAction(lambda result: {"result":"function", "name":result[2], "params":result[4], "body":result[6]}) pTOP = (pDEFUN | pTOPEXPR) result = pTOP.parseString(input)[0] return result # the first element of the result is the expression
def init_grammar(self): """Set up the parsing classes Any changes to the grammar of the config file be done here. """ # Some syntax that we need, but don't bother looking at SEMICOLON = (Suppress(";")) EQUALS = Suppress("=") OPENB = Suppress("(") CLOSEB = Suppress(")") BACKSLASH = Suppress("\\") DASH = Suppress("-") # Top Section FILENAME = Word(alphas + nums + '-_.') alignmentdef = Keyword('alignment') + EQUALS + FILENAME + SEMICOLON alignmentdef.setParseAction(self.set_alignment) treedef = Keyword('user_tree_topology') + EQUALS + FILENAME + SEMICOLON treedef.setParseAction(self.set_user_tree) def simple_option(name): opt = Keyword(name) + EQUALS + Word(alphas + nums + '-_') + SEMICOLON opt.setParseAction(self.set_simple_option) return opt branchdef = simple_option('branchlengths') MODELNAME = Word(alphas + nums + '+') modellist = delimitedList(MODELNAME) modeldef = Keyword("models") + EQUALS + Group( ( CaselessKeyword("all") | CaselessKeyword("mrbayes") | CaselessKeyword("raxml") | CaselessKeyword("beast") | CaselessKeyword("all_protein") | CaselessKeyword( "all_protein_gamma") | CaselessKeyword("all_protein_gammaI") )("predefined") | Group(modellist)("userlist")) + SEMICOLON modeldef.setParseAction(self.set_models) modseldef = simple_option("model_selection") topsection = alignmentdef + Optional(treedef) + branchdef + \ modeldef + modseldef # Partition Parsing column = Word(nums) partname = Word(alphas + '_-' + nums) partdef = column("start") +\ Optional(DASH + column("end")) +\ Optional(BACKSLASH + column("step")) partdef.setParseAction(self.define_range) partdeflist = Group(OneOrMore(Group(partdef))) partition = Optional("charset") + partname("name") + \ EQUALS + partdeflist("parts") + SEMICOLON partition.setParseAction(self.define_partition) partlist = OneOrMore(Group(partition)) partsection = Suppress("[data_blocks]") + partlist # Scheme Parsing schemename = Word(alphas + '_-' + nums) partnameref = partname.copy( ) # Make a copy, cos we set a different action on it partnameref.setParseAction(self.check_part_exists) subset = Group(OPENB + delimitedList(partnameref("name")) + CLOSEB) subset.setParseAction(self.define_subset) scheme = Group(OneOrMore(subset)) schemedef = schemename("name") + \ EQUALS + scheme("scheme") + SEMICOLON schemedef.setParseAction(self.define_schema) schemelist = OneOrMore(Group(schemedef)) schemealgo = simple_option("search") schemesection = \ Suppress("[schemes]") + schemealgo + Optional(schemelist) # We've defined the grammar for each section. Here we just put it all together self.config_parser = ( topsection + partsection + schemesection + stringEnd)
def _setup_QASMParser(): """ Routine to initialise and return parsing blocks """ class _Op: """ Class to set up quantum operations """ def __init__(self, name, argParser, version="OPENQASM 2.0", qop=False, keyOverride=None): global cops global qops global _reservedKeys if name in qops or name in cops: raise IOError(dupTokenWarning.format("Operation", name)) self.operation = name if keyOverride is not None: self.parser = (keyOverride + argParser).addParseAction( lambda s, l, t: _override_keyword(t, name)) else: self.parser = CaselessKeyword(name)("keyword") + argParser self.version = parse_version(version) self.parser.addParseAction( lambda s, l, t: _set_version(t, self.version)) _reservedKeys.append(name) if qop: qops[name] = self else: cops[name] = self class _Routine(): """ Class to set up quantum gates, circuits, etc. """ def __init__(self, name, pargs=False, spargs=False, gargs=False, qargs=False, returnables=False, prefixes=None, version="OPENQASM 2.0"): global blocks global _reservedKeys if name in qops or name in cops: raise IOError(dupTokenWarning.format("Routine", name)) self.operation = name self.parser = Keyword(name)("keyword") + validName("gateName") if prefixes: localPrefixParser = Each(map(Optional, map( Keyword, prefixes))).addParseAction(prefix_setter) else: localPrefixParser = prefixParser self.parser = localPrefixParser + self.parser # Handle different args req = [] if pargs: req.append(Optional(pargParser)("pargs")) if spargs: req.append(Optional(spargParser)("spargs")) if gargs: req.append(Optional(gargParser)("gargs")) self.parser = self.parser + Each(req) if qargs: self.parser = self.parser + qargParser("qargs") if returnables: self.parser = self.parser + Optional(returnParser) self.version = parse_version(version) self.parser.addParseAction( lambda s, l, t: _set_version(t, self.version)) _reservedKeys.append(name) blocks[name] = self class _Block(): """ Class to set up blocks such as if, for, etc. """ def __init__(self, name, detParser, version="OPENQASM 2.0"): global blocks global _reservedKeys self.operation = name self.parser = Keyword(name)("keyword") + detParser self.version = parse_version(version) self.parser.addParseAction( lambda s, l, t: _set_version(t, self.version)) _reservedKeys.append(name) blocks[name] = self sign = Word("+-", exact=1) number = Word(nums) expo = Combine(CaselessLiteral("e") + Optional(sign) + number).setResultsName("exponent") pi = CaselessKeyword("pi") bitstring = Combine(OneOrMore(oneOf("0 1")) + Literal("b")) integer = Combine(number + Optional(expo)) real = Combine( Optional(sign) + (("." + number) ^ (number + "." + Optional(number))) + Optional(expo)) validName = Forward() lineEnd = Literal(";") _is_ = Keyword("to").suppress() _in_ = Keyword("in") _to_ = Literal("->").suppress() commentSyntax = "//" commentOpenStr = "/*" commentCloseStr = "*/" commentOpenSyntax = Literal(commentOpenStr) commentCloseSyntax = Literal(commentCloseStr) dirSyntax = "***" dirOpenStr = f"{dirSyntax} begin" dirCloseStr = f"{dirSyntax} end" dirSyntax = Keyword(dirSyntax) dirOpenSyntax = CaselessLiteral(dirOpenStr) dirCloseSyntax = CaselessLiteral(dirCloseStr) intFunc = oneOf("abs powrem countof fllog") realFunc = oneOf("abs powrem arcsin arccos arctan sin cos tan exp ln sqrt") boolFunc = oneOf("andof orof xorof") inL, inS, inR = map(Suppress, "[:]") vBar = Suppress("|") bSlash = Suppress("\\") brL, brR = map(Suppress, "()") intExp = Forward() realExp = Forward() boolExp = Forward() index = intExp.setResultsName("index") interval = Optional(intExp.setResultsName("start"), default=None) + inS \ + Optional(intExp.setResultsName("end"), default=None) \ + Optional(inS + Optional(intExp.setResultsName("step"), default=1)) interRef = Group(inL + interval + inR) loopRef = Group( inL + intExp.setResultsName("start") + inS + intExp.setResultsName("end") + Optional(inS + Optional(intExp.setResultsName("step"), default=1)) + inR) ref = inL + Group(delimitedList(index ^ interval))("ref") + inR regNoRef = validName("var") regRef = Group(validName("var") + Optional(ref)) regMustRef = Group(validName("var") + ref) regListNoRef = Group(delimitedList(regNoRef)) regListRef = Group(delimitedList(regRef)) inPlaceAlias = vBar + regListRef + vBar validQarg = regRef | inPlaceAlias aliasQarg = Group(regRef) | inPlaceAlias inPlaceCreg = bSlash + delimitedList(regRef | bitstring) + bSlash validCreg = (regRef | inPlaceCreg) def set_maths_type(toks, mathsType): """ Set logical or integer or floating point """ toks["type"] = mathsType intVar = integer | regRef realVar = real | integer | pi | regRef boolVar = interRef | regRef | realExp | intExp | validCreg | bitstring intFuncVar = (intFunc + brL + Group(Optional(delimitedList(intVar)))("args") + brR).setParseAction(Function) realFuncVar = ((realFunc ^ intFunc) + brL + Group(Optional(delimitedList(realVar)))("args") + brR).setParseAction(Function) boolFuncVar = (boolFunc + brL + Group(Optional(delimitedList(boolVar)))("args") + brR).setParseAction(Function) mathOp = [(oneOf("- +"), 1, opAssoc.RIGHT, Binary), (oneOf("^"), 2, opAssoc.LEFT, Binary), (oneOf("* / div"), 2, opAssoc.LEFT, Binary), (oneOf("+ -"), 2, opAssoc.LEFT, Binary)] logOp = [(oneOf("! not"), 1, opAssoc.RIGHT, Binary), (oneOf("and or xor"), 2, opAssoc.LEFT, Binary), (oneOf("< <= == != >= >"), 2, opAssoc.LEFT, Binary), (oneOf("in"), 2, opAssoc.LEFT, Binary)] intExp <<= infixNotation( intFuncVar | intVar, mathOp).setParseAction(lambda s, l, t: set_maths_type(t, "int")) realExp <<= infixNotation( realFuncVar | realVar, mathOp).setParseAction(lambda s, l, t: set_maths_type(t, "float")) boolExp <<= infixNotation( boolFuncVar | boolVar, logOp).setParseAction(lambda s, l, t: set_maths_type(t, "bool")) mathExp = intExp ^ realExp ^ boolExp cregExp = bitstring("bit") ^ validCreg("reg") prefixes = ["unitary"] callMods = ["CTRL", "INV"] def prefix_setter(toks): """ Pull out prefixes of gate calls and add them into list """ for prefix in prefixes: toks[prefix] = prefix in toks.asList() prefixParser = Each(map(Optional, map(Keyword, prefixes))).addParseAction(prefix_setter) pargParser = brL + delimitedList(validName)("pargs") + brR spargParser = inL + delimitedList(validName)("spargs") + inR gargParser = ungroup( nestedExpr("<", ">", delimitedList(ungroup(validName)), None)) qargParser = delimitedList(regRef) callQargParser = delimitedList(validQarg) callPargParser = brL + delimitedList(realExp) + brR callSpargParser = inL + delimitedList(intExp) + inR fullArgParser = Each( (Optional(pargParser("pargs")), Optional(spargParser("spargs")), Optional(gargParser("gargs")))) callArgParser = Each( (Optional(callPargParser("pargs")), Optional(callSpargParser("spargs")), Optional(gargParser("gargs")))) returnParser = Optional(_to_ + validCreg("byprod")) modifiers = ZeroOrMore(Combine(oneOf(callMods) + Suppress("-"))) commentLine = Literal(commentSyntax).suppress() + restOfLine("comment") commentBlock = cStyleComment("comment").addParseAction( removeQuotes).addParseAction(removeQuotes) comment = commentLine | commentBlock comment.addParseAction(lambda s, l, t: _set_version(t, (0, 0, 0))) directiveName = Word(alphas).setParseAction(downcaseTokens) directiveArgs = CharsNotIn(";") _Op("directive", directiveName("directive") + Suppress(White() * (1, )) + directiveArgs("args"), version="REQASM 1.0", keyOverride=(~dirOpenSyntax + ~dirCloseSyntax + dirSyntax)) def split_args(toks): """ Split directive arguments out """ toks[0]["keyword"] = "directive" toks[0]["args"] = toks[0]["args"].strip().split(" ") directiveStatement = directiveName("directive") + restOfLine("args") + \ Group(ZeroOrMore(Combine(Optional(White(" ")) + ~dirCloseSyntax + Word(printables+" "))))("block") directiveBlock = ungroup( nestedExpr( dirOpenSyntax, dirCloseSyntax, content=directiveStatement, ignoreExpr=(comment | quotedString )).setWhitespaceChars("\n").setParseAction(split_args)) directiveBlock.addParseAction(lambda s, l, t: _set_version(t, (2, 1, 0))) # Programming lines _Op("version", Empty(), version=(0, 0, 0), keyOverride=Combine( oneOf(versions)("type") + White() + real("versionNumber"))("version")) _Op("include", quotedString("file").addParseAction(removeQuotes)) # Gate-like structures _Op("opaque", validName("name") + fullArgParser + Optional(qargParser("qargs")) + returnParser, keyOverride=prefixParser + "opaque") _Routine("gate", pargs=True, qargs=True) _Routine("circuit", pargs=True, qargs=True, spargs=True, returnables=True, version="REQASM 1.0") # Variable-like structures _Op("creg", regRef("arg")) _Op("qreg", regRef("arg")) _Op("cbit", Group(regNoRef)("arg"), version="REQASM 1.0") _Op("qbit", Group(regNoRef)("arg"), version="REQASM 1.0") _Op("defAlias", regMustRef("alias"), keyOverride="alias", version="REQASM 1.0") # No more on-definition aliases _Op("alias", regRef("alias") + _is_ + aliasQarg("target"), keyOverride="set", version="REQASM 1.0") _Op("val", validName("var") + Literal("=").suppress() + mathExp("val"), version="REQASM 1.0") _Op("set", (Group(regRef)("var") ^ inPlaceCreg("var")) + Literal("=").suppress() + cregExp("val"), version="REQASM 1.0") # Operations-like structures _Op("measure", regRef("qreg") + _to_ + regRef("creg"), qop=True) _Op("barrier", regListNoRef("args")) _Op("output", regRef("value"), qop=True, version="REQASM 1.0") _Op("reset", regRef("qreg")) _Op("exit", Empty(), version="REQASM 1.0") _Op("free", validName("target"), version="REQASM 1.0") _Op("next", validName("loopVar"), qop=True, version="REQASM 1.0") _Op("finish", (Literal("quantum process") | validName)("loopVar"), qop=True, version="REQASM 1.0") _Op("end", validName("process"), qop=True, version="REQASM 1.0") # Special gate call handler callGate = Combine(Group(modifiers)("mods") + \ validName("gate")) + \ callArgParser + \ callQargParser("qargs").addParseAction(lambda s, l, t: _override_keyword(t, "call")) + \ returnParser callGate.addParseAction(lambda s, l, t: _set_version(t, (1, 2, 0))) # Block structures _Block("for", validName("var") + _in_ + loopRef("range"), version="REQASM 1.0") _Block("if", "(" + boolExp("cond") + ")", version="REQASM 1.0") _Block("while", "(" + boolExp("cond") + ")", version="OMEQASM 1.0") qopsParsers = list(map(lambda qop: qop.parser, qops.values())) + [callGate, directiveBlock] blocksParsers = list(map(lambda block: block.parser, blocks.values())) _Op("if", blocks["if"].parser + Group(Group(Group(Or(qopsParsers))))("block"), version="OPENQASM 2.0", keyOverride=Empty()) _Op("for", blocks["for"].parser + Group(Group(Group(Or(qopsParsers))))("block"), version="REQASM 1.0", keyOverride=Empty()) _Op("while", blocks["while"].parser + Group(Group(Group(Or(qopsParsers))))("block"), version="OMEQASM 1.0", keyOverride=Empty()) # Set-up line parsers reservedNames = Or(map(Keyword, _reservedKeys)) validName <<= (~reservedNames) + Word(alphas, alphanums + "_") copsParsers = list(map(lambda cop: cop.parser, cops.values())) operations = (( (Or(copsParsers) ^ Or(qopsParsers)) | # Classical/Quantum Operations callGate | # Gate parsers White() # Blank Line ) + lineEnd.suppress()) ^ directiveBlock # ; or Directives validLine = Forward() codeBlock = nestedExpr("{", "}", Suppress(White()) ^ Group(validLine), (quotedString)) validLine <<= ( ((operations + Optional(comment)) ^ (Or(blocksParsers) + codeBlock("block") + Optional(lineEnd)) ^ comment)) # Whole line comment testLine = Forward() dummyCodeBlock = nestedExpr( "{", "}", testLine, (directiveBlock | quotedString | comment)) + Optional(lineEnd) ignoreSpecialBlocks = (~commentOpenSyntax + ~commentCloseSyntax + ~dirOpenSyntax + ~dirCloseSyntax) testLine <<= ( comment | # Comments directiveBlock | # Directives (ignoreSpecialBlocks + ZeroOrMore(CharsNotIn("{}")) + dummyCodeBlock) | # Block operations (ignoreSpecialBlocks + ZeroOrMore(CharsNotIn("{};")) + lineEnd) ) # QASM Instructions testKeyword = (dirSyntax.setParseAction( lambda s, l, t: _override_keyword(t, "directive")) | Word(alphas)("keyword")) code = (Group(directiveBlock)) | Group(validLine) return code, testLine, testKeyword, reservedNames, mathExp
def parse_morphology(filename, filename_toparse): global current_section_name current_section_name = '' converted_file = open(filename, 'w') put_string = 'from neuron import h\ndef shape_3D(self):\n' converted_file.write(put_string) ntabs = 1 # from here on, add a tab to all lines # define lists of characters for a..z and 1..9 uppercase = lowercase.upper() lowercaseplus = lowercase+('_') lowercaseplus = lowercaseplus+(uppercase) nonzero = ''.join([str(i) for i in range(1, 10)]) COMMA = Literal(',') EQUALS = Literal('=') MINUS = Literal('-') PERIOD = Literal('.') LCURL = Literal('{') RCURL = Literal('}') LBRACK = Literal('(') RBRACK = Literal(')') LSQUARE = Literal('[') RSQUARE = Literal(']') PTSCLEAR = Literal('{pt3dclear()').suppress() PTSCLEARNL = Literal('{\npt3dclear()\n').suppress() integer = Word(nums) single_section = Word(lowercaseplus, min = 2) single_section.setResultsName('SINGLE') integer_var = Word(lowercase, exact = 1) double = Group(Optional(MINUS) + integer + Optional(PERIOD + integer)) operand = integer ^ integer_var operator = Word('+-*/', exact=1) unaryoperation = operand binaryoperation = operand + operator + operand operation = unaryoperation ^ binaryoperation array_section = Group(single_section + LSQUARE.suppress() + operation + RSQUARE.suppress()) array_section.setResultsName('ARRAY') section = single_section ^ array_section section_location = Group(section + LBRACK.suppress() + double + RBRACK.suppress()) create = Keyword('create').suppress() + section + ZeroOrMore(COMMA.suppress() + section) create.setParseAction(print_create(converted_file, ntabs)) connect = Keyword('connect').suppress() + section_location + COMMA.suppress() + section_location connect.setParseAction(print_connect(converted_file, ntabs)) for_loop = Keyword('for').suppress() + integer_var + EQUALS.suppress() + integer + COMMA.suppress() + integer # NOTE TO FUTURE SELF: for loops can only have one line of code in this implementation for_loop.setParseAction(print_for_loop(converted_file, ntabs)) point_add = Literal('pt3dadd(').suppress() + double + COMMA.suppress() + double + COMMA.suppress() + double + COMMA.suppress() + double + RBRACK.suppress() point_add.setParseAction(print_point_add(converted_file, ntabs)) point_style = Literal('pt3dstyle(').suppress() + double + COMMA.suppress() + double + COMMA.suppress() + double + COMMA.suppress() + double + RBRACK.suppress() point_style.setParseAction(print_point_style(converted_file, ntabs)) geom_define_pre = section + (PTSCLEAR ^ PTSCLEARNL) geom_define_body = OneOrMore(point_add ^ point_style) + RCURL.suppress() geom_define_pre.setParseAction(update_current_section(converted_file, ntabs)) geom_define = geom_define_pre + geom_define_body expression = (connect ^ for_loop ^ geom_define ^ create) codeblock = OneOrMore(expression) test_str = 'Ia_node[0] {\npt3dclear()\n pt3dadd( 47, 76, 92.5, 3.6) }' #file_to_parse = open('../../tempdata/Ia_geometry') file_to_parse = open(filename_toparse) tokens = codeblock.parseString(file_to_parse.read())
def parse_imp (input): # parse a string into an element of the abstract representation # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # ( if <expr> <expr> <expr> ) # ( function ( <name ... ) <expr> ) # ( <expr> <expr> ... ) # # <decl> ::= var name = expr ; # # <stmt> ::= if <expr> <stmt> else <stmt> # while <expr> <stmt> # name <- <expr> ; # print <expr> ; # <block> # # <block> ::= { <decl> ... <stmt> ... } # # <toplevel> ::= <decl> # <stmt> # idChars = alphas+"_+*-?!=<>" pIDENTIFIER = Word(idChars, idChars+"0123456789") #### NOTE THE DIFFERENCE pIDENTIFIER.setParseAction(lambda result: EPrimCall(oper_deref,[EId(result[0])])) # A name is like an identifier but it does not return an EId... pNAME = Word(idChars,idChars+"0123456789") pNAMES = ZeroOrMore(pNAME) pNAMES.setParseAction(lambda result: [result]) pINTEGER = Word("0123456789") pINTEGER.setParseAction(lambda result: EValue(VInteger(int(result[0])))) pBOOLEAN = Keyword("true") | Keyword("false") pBOOLEAN.setParseAction(lambda result: EValue(VBoolean(result[0]=="true"))) pEXPR = Forward() pEXPRS = ZeroOrMore(pEXPR) pEXPRS.setParseAction(lambda result: [result]) pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")" pIF.setParseAction(lambda result: EIf(result[2],result[3],result[4])) def mkFunBody (params,body): bindings = [ (p,ERefCell(EId(p))) for p in params ] return ELet(bindings,body) pFUN = "(" + Keyword("function") + "(" + pNAMES + ")" + pEXPR + ")" pFUN.setParseAction(lambda result: EFunction(result[3],mkFunBody(result[3],result[5]))) pCALL = "(" + pEXPR + pEXPRS + ")" pCALL.setParseAction(lambda result: ECall(result[1],result[2])) pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pIF | pFUN | pCALL) pDECL_VAR = "var" + pNAME + "=" + pEXPR + ";" pDECL_VAR.setParseAction(lambda result: (result[1],result[3])) # hack to get pDECL to match only PDECL_VAR (but still leave room # to add to pDECL later) pDECL = ( pDECL_VAR | NoMatch() ) pDECLS = ZeroOrMore(pDECL) pDECLS.setParseAction(lambda result: [result]) pSTMT = Forward() pSTMT_IF_1 = "if" + pEXPR + pSTMT + "else" + pSTMT pSTMT_IF_1.setParseAction(lambda result: EIf(result[1],result[2],result[4])) pSTMT_IF_2 = "if" + pEXPR + pSTMT pSTMT_IF_2.setParseAction(lambda result: EIf(result[1],result[2],EValue(VBoolean(True)))) pSTMT_WHILE = "while" + pEXPR + pSTMT pSTMT_WHILE.setParseAction(lambda result: EWhile(result[1],result[2])) pSTMT_PRINT = "print" + pEXPR + ";" pSTMT_PRINT.setParseAction(lambda result: EPrimCall(oper_print,[result[1]])); pSTMT_UPDATE = pNAME + "<-" + pEXPR + ";" pSTMT_UPDATE.setParseAction(lambda result: EPrimCall(oper_update,[EId(result[0]),result[2]])) pSTMTS = ZeroOrMore(pSTMT) pSTMTS.setParseAction(lambda result: [result]) def mkBlock (decls,stmts): bindings = [ (n,ERefCell(expr)) for (n,expr) in decls ] return ELet(bindings,EDo(stmts)) pSTMT_BLOCK = "{" + pDECLS + pSTMTS + "}" pSTMT_BLOCK.setParseAction(lambda result: mkBlock(result[1],result[2])) pSTMT << ( pSTMT_IF_1 | pSTMT_IF_2 | pSTMT_WHILE | pSTMT_PRINT | pSTMT_UPDATE | pSTMT_BLOCK ) # can't attach a parse action to pSTMT because of recursion, so let's duplicate the parser pTOP_STMT = pSTMT.copy() pTOP_STMT.setParseAction(lambda result: {"result":"statement", "stmt":result[0]}) pTOP_DECL = pDECL.copy() pTOP_DECL.setParseAction(lambda result: {"result":"declaration", "decl":result[0]}) pABSTRACT = "#abs" + pSTMT pABSTRACT.setParseAction(lambda result: {"result":"abstract", "stmt":result[1]}) pQUIT = Keyword("#quit") pQUIT.setParseAction(lambda result: {"result":"quit"}) pTOP = (pQUIT | pABSTRACT | pTOP_DECL | pTOP_STMT ) result = pTOP.parseString(input)[0] return result # the first element of the result is the expression
doftype = Keyword("x")|Keyword("y")|Keyword("z")|Keyword("angle_x")|Keyword("angle_y")|Keyword("angle_z") dof = (doftype+Optional(paren+real+paren)) dofs = Keyword("set_dof")+tag+OneOrMore(dof ) jgroup = Keyword("set_jump_group")+tag+OneOrMore(tag) symline = (sname|eline|anchor|vcoords|connect|dofs |jgroup)+LineEnd() symfile = ZeroOrMore(symline)+StringEnd() real .setParseAction( lambda s,l,t: float(t[0]) ) sname .setParseAction(tuple) anchor .setParseAction(tuple) efirst .setParseAction(lambda s,l,t: (t[0],t[2])) erest .setParseAction(lambda s,l,t: ( t[1],t[4],t[6])) eline .setParseAction(lambda s,l,t: ('E',tuple(t[2:]))) vcoord .setParseAction(lambda s,l,t: tuple(t[1:])) vcoords.setParseAction(lambda s,l,t: ('xyz',tuple(t[1:-1]))) subunit.setParseAction(lambda s,l,t: ("subunit",t[1:])) connect.setParseAction(lambda s,l,t: ("connect",tuple([x for x in t[1:] if not x=='\n']))) dof .setParseAction(lambda s,l,t: ('dof',(t[0],t[2]) if len(t)>2 else (t[0],))) dofs .setParseAction(lambda s,l,t: ('dofs',tuple(t[1:]))) jgroup .setParseAction(lambda s,l,t: ('jump_group',t[1:])) symline.setParseAction(lambda s,l,t: tuple([x for x in t if not x=='\n'])) print dof .parseString("x(20.4244497891662)") print dof .parseString("angle_x") print dofs .parseString("set_dof JUMP0_0_to_com x(20.4244497891662)") #print dofs .parseString("set_dof JUMP0_0_to_subunit angle_x angle_y angle_z") #print dofs .parseString("set_dof JUMP0_0 x(3.81281197834553) angle_x") # print connect.parseString("connect_virtual JUMP0_0_to_com VRT0_0 VRT0_0_base") # print connect.parseString("connect_virtual JUMP0_0_to_subunit TEST SUBUNIT")
def parse (input): # parse a string into an element of the abstract representation # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # ( if <expr> <expr> <expr> ) # ( let ( ( <name> <expr> ) ) <expr ) # ( + <expr> <expr> ) # ( * <expr> <expr> ) # idChars = alphas+"_+*-?!=<>" pIDENTIFIER = Word(idChars, idChars+"0123456789") pIDENTIFIER.setParseAction(lambda result: EId(result[0])) # A name is like an identifier but it does not return an EId... pNAME = Word(idChars,idChars+"0123456789") pINTEGER = Word("-0123456789","0123456789") pINTEGER.setParseAction(lambda result: EInteger(int(result[0]))) pBOOLEAN = Keyword("true") | Keyword("false") pBOOLEAN.setParseAction(lambda result: EBoolean(result[0]=="true")) pEXPR = Forward() pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")" pIF.setParseAction(lambda result: EIf(result[2],result[3],result[4])) pBINDING = "(" + pNAME + pEXPR + ")" pBINDING.setParseAction(lambda result: (result[1],result[2])) pMULT_BINDING = OneOrMore(pBINDING) pMULT_BINDING.setParseAction(lambda result: [(r[0],r[1]) for r in result]) pLET = "(" + Keyword("let") + "(" + pMULT_BINDING + ")" + pEXPR + ")" pLET.setParseAction(lambda result: pLET_exps_unpack(result)) pPLUS = "(" + Keyword("+") + pEXPR + pEXPR + ")" pPLUS.setParseAction(lambda result: ECall("+",[result[2],result[3]])) pTIMES = "(" + Keyword("*") + pEXPR + pEXPR + ")" pTIMES.setParseAction(lambda result: ECall("*",[result[2],result[3]])) pUSR_FUNC = "(" + pNAME + OneOrMore(pEXPR)+ ")" pUSR_FUNC.setParseAction(lambda result: ECall(result[1], result[2:-1])) pDEF_FUNC = "(" + Keyword("defun") + pNAME + "(" + OneOrMore(pNAME) + ")" + pEXPR + ")" pDEF_FUNC.setParseAction(lambda result: pDEF_FUNC_unpack_and_add_to_dict(result)) pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pNAME | pIF | pLET | pPLUS | pTIMES | pUSR_FUNC | pDEF_FUNC) result = pEXPR.parseString(input)[0] if type(result) == dict: return {"result":"function", "name":result["name"], "params":result["params"], "body":result["body"]} else: return {"result":"expression", "expr":result}
Forward, Empty, quotedString, oneOf, removeQuotes, delimitedList, nums, alphas, alphanums, Keyword, CaselessLiteral) word_free = Word(alphas + '][@_-/.+**' + alphanums) word_strict = Word(alphas, alphas + alphanums + '_' ) (lparen, rparen, lbrack, rbrack, lbrace, rbrace, colon, equal_sign) = map(Suppress, '()[]{}:=') integer = Combine(Optional(oneOf('+ -')) + Word(nums)).setName('integer') cvt_int = lambda toks: int(toks[0]) integer.setParseAction(cvt_int) boolean_true = Keyword('True', caseless=True) boolean_true.setParseAction(lambda x: True) boolean_false = Keyword('False', caseless=True) boolean_false.setParseAction(lambda x: False) boolean = boolean_true | boolean_false none = Keyword('None', caseless=True) cvt_none = lambda toks: [None] none.setParseAction(cvt_none) e = CaselessLiteral("e") real = (Combine(Optional(oneOf('+ -')) + Word(nums) + '.' + Optional(Word(nums)) + Optional(e + Optional(oneOf('+ -')) + Word(nums))) | Combine(Optional(oneOf('+ -')) + Word(nums)
def parse(input): # parse a string into an element of the abstract representation # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # ( if <expr> <expr> <expr> ) # ( let ( ( <name> <expr> ) ) <expr ) # ( function ( <name> ... ) <expr> ) # ( ref <expr> ) # ( <expr> <expr> ... ) # idChars = alphas + "_+*-?!=<>" pIDENTIFIER = Word(idChars, idChars + "0123456789") pIDENTIFIER.setParseAction(lambda result: EId(result[0])) # A name is like an identifier but it does not return an EId... pNAME = Word(idChars, idChars + "0123456789") pNAMES = ZeroOrMore(pNAME) pNAMES.setParseAction(lambda result: [result]) pINTEGER = Word("0123456789") pINTEGER.setParseAction(lambda result: EValue(VInteger(int(result[0])))) pBOOLEAN = Keyword("true") | Keyword("false") pBOOLEAN.setParseAction( lambda result: EValue(VBoolean(result[0] == "true"))) pEXPR = Forward() pEXPRS = ZeroOrMore(pEXPR) pEXPRS.setParseAction(lambda result: [result]) pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")" pIF.setParseAction(lambda result: EIf(result[2], result[3], result[4])) pBINDING = "(" + pNAME + pEXPR + ")" pBINDING.setParseAction(lambda result: (result[1], result[2])) pBINDINGS = ZeroOrMore(pBINDING) pBINDINGS.setParseAction(lambda result: [result]) pLET = "(" + Keyword("let") + "(" + pBINDINGS + ")" + pEXPR + ")" pLET.setParseAction(lambda result: ELet(result[3], result[5])) pCALL = "(" + pEXPR + pEXPRS + ")" pCALL.setParseAction(lambda result: ECall(result[1], result[2])) pFUN = "(" + Keyword("function") + "(" + pNAMES + ")" + pEXPR + ")" pFUN.setParseAction(lambda result: EFunction(result[3], result[5])) pFUNrec = "(" + Keyword( "function") + pNAME + "(" + pNAMES + ")" + pEXPR + ")" pFUNrec.setParseAction( lambda result: EFunction(result[4], result[6], name=result[2])) pCLASS = "(" + Keyword("class") + "(" + pNAMES + ")" + Keyword( "(") + pBINDINGS + ")" + Keyword("(") + pBINDINGS + Keyword(")") + ")" pCLASS.setParseAction( lambda result: EClass(result[3], result[6], result[9])) pNEW = "(" + Keyword("new") + pEXPR + pEXPRS + ")" pNEW.setParseAction(lambda result: ENew(result[2], result[3])) pWITH = "(" + Keyword("with") + pEXPR + pEXPR + ")" pWITH.setParseAction(lambda result: EWithObj(result[2], result[3])) pDO = "(" + Keyword("do") + pEXPRS + ")" pDO.setParseAction(lambda result: EDo(result[2])) pWHILE = "(" + Keyword("while") + pEXPR + pEXPR + ")" pWHILE.setParseAction(lambda result: EWhile(result[2], result[3])) pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pIF | pLET | pFUN | pFUNrec | pCLASS | pNEW | pWITH | pDO | pWHILE | pCALL) # can't attach a parse action to pEXPR because of recursion, so let's duplicate the parser pTOPEXPR = pEXPR.copy() pTOPEXPR.setParseAction(lambda result: { "result": "expression", "expr": result[0] }) pDEFINE = "(" + Keyword("define") + pNAME + pEXPR + ")" pDEFINE.setParseAction(lambda result: { "result": "value", "name": result[2], "expr": result[3] }) pDEFUN = "(" + Keyword("defun") + pNAME + "(" + pNAMES + ")" + pEXPR + ")" pDEFUN.setParseAction( lambda result: { "result": "function", "name": result[2], "params": result[4], "body": result[6] }) pABSTRACT = "#abs" + pEXPR pABSTRACT.setParseAction(lambda result: { "result": "abstract", "expr": result[1] }) pQUIT = Keyword("#quit") pQUIT.setParseAction(lambda result: {"result": "quit"}) pTOP = (pDEFUN | pDEFINE | pQUIT | pABSTRACT | pTOPEXPR) result = pTOP.parseString(input)[0] return result # the first element of the result is the expression
def simple_option(name): opt = Keyword(name) + EQUALS +\ Word(alphas + nums + '-_') + SEMICOLON opt.setParseAction(self.set_simple_option) return opt
istart = orig_test.setResultsName('istart') eqsign = p.Literal('=').setParseAction(lambda s,loc,toks: [' = ']) comma = p.Literal(',').setParseAction(lambda s,loc,toks: [', ']) iend = orig_test.setResultsName('iend') istep = orig_test.setResultsName('istep') do_stmt = do_kwd + ivar + eqsign + istart + comma + iend + Optional(comma + istep)\ + EOLL do_stmt.setParseAction(lambda s,loc,toks: [''.join(toks), '', 'enddo']) if_expr = (Suppress('(') + orig_test + Suppress(')')) | orig_test if_expr.setParseAction(lambda s,loc,toks: [' (' + ''.join(toks) + ')' ]) if_expr_2 = if_expr.copy() if_expr_2.setParseAction(lambda s,loc,toks: [' (' + ''.join(toks) + ')' + ' then' ]) if_stmt = Keyword('if') + if_expr_2 + Optional(Suppress('then')) + EOLL if_stmt.setParseAction(lambda s,loc,toks: [''.join(toks), '', 'endif']) elseif_stmt = Keyword('elseif') + if_expr_2 + Optional(Suppress('then')) + EOLL elseif_stmt.setParseAction(lambda s,loc,toks: [''.join(toks), '']) while_kwd = Keyword('while').setParseAction(lambda s,loc,toks: [' while']) dowhile_stmt = Keyword('do') + while_kwd + if_expr + EOLL dowhile_stmt.setParseAction(lambda s,loc,toks: [''.join(toks), '', 'enddo']) selectcase_kwd = Keyword('select') + Keyword('case') selectcase_kwd.setParseAction(lambda s,loc,toks: [' '.join(toks)]) selectcase_stmt = selectcase_kwd + if_expr + EOLL selectcase_stmt.setParseAction(lambda s,loc,toks: [''.join(toks), '', 'end select']) where_stmt = Keyword('where') + if_expr + EOLL where_stmt.setParseAction(lambda s,loc,toks: [''.join(toks), '', 'end where'])
def RawInputSpecParser(): '''Syntax of the INPUT statement (and nothing else).''' with PyParsingDefaultWhitespaceChars(DEFAULT_WHITESPACE_CHARS): INPUT = CaselessKeyword('INPUT').suppress() FOR = CaselessKeyword('for').suppress() IN = CaselessKeyword('in').suppress() target = Forward() anonymous_var = Keyword('_') # Keywords cannot be used as variable names (we still allow "INPUT" as it never occurs inside the spec) input_keyword = FOR | IN var = (~input_keyword + ~anonymous_var + Word(alphas + '_', alphanums + '_')).setName('variable') tuple_match = lpar + target + ZeroOrMore(comma + target) + Optional( comma) + rpar # The target of an assignment, supporting tuple unpacking as a simple form of pattern matching in addition to plain variables target << (anonymous_var | var | tuple_match) # anonymous_var.setParseAction(lambda: i.AnonymousVariable()) var.setParseAction(lambda t: i.Variable(str(t[0]))) tuple_match.setParseAction(lambda t: i.TupleMatch(t)) # Accessing objects, some examples: # - just access a variable directly: node # - access a field on a variable: node.label # - accessing a fixed index in a collection: some_tuple[3] # - chainable: node.neighbors[2].label field_accessor = dot + py_identifier('name') subscript = integer | QuotedString('"', escChar='\\') subscript_accessor = lbracket + subscript('key') + rbracket accessor = var('var') + Group( ZeroOrMore(field_accessor | subscript_accessor))('path') # field_accessor.setParseAction(lambda t: i.Attribute(t.name)) subscript_accessor.setParseAction(lambda t: i.Subscript(t.key)) accessor.setParseAction(lambda t: i.Accessor(t.var, t.path)) # Iterating over objects iteration = FOR + target('target') + IN + accessor('accessor') iterations = Group(ZeroOrMore(iteration)) # iteration.setParseAction(lambda t: i.Iteration(t.target, t.accessor)) predicate_args = Group( Optional(accessor + ZeroOrMore(comma + accessor) + Optional(comma))) predicate_spec = predicate_name('pred') + lpar + predicate_args( 'args') + rpar + iterations('iters') + semicolon predicate_specs = Group(ZeroOrMore(predicate_spec)) # predicate_spec.setParseAction( lambda t: i.Predicate(t.pred, t.args, t.iters)) # Allow optional types, e.g., Set<Node> etc. input_type = Forward() input_type << (py_qualified_identifier('type_name') + Group( Optional(langle + input_type + ZeroOrMore(comma + input_type) + rangle))('type_args')) input_arg = Group((input_type('type') + var('name')) | var('name')) input_args = Group( Optional(input_arg + ZeroOrMore(comma + input_arg) + Optional(comma))) input_statement = INPUT + lpar + input_args( 'args') + rpar + lbrace + predicate_specs('preds') + rbrace # input_statement.setParseAction(lambda t: i.InputSpec( (x.name for x in t.args), t.preds)) return input_statement
def init_grammar(self): """Set up the parsing classes Any changes to the grammar of the config file be done here. """ # Some syntax that we need, but don't care about SEMICOLON = (Suppress(";")) EQUALS = Suppress("=") # Top Section FILE_NAME = Word(alphas + nums + '-_.') alignment_def = Keyword('alignment') + EQUALS\ + FILE_NAME + SEMICOLON alignment_def.setParseAction(self.set_alignment) tree_def = Keyword('user_tree_topology') + EQUALS\ + FILE_NAME + SEMICOLON tree_def.setParseAction(self.set_user_tree) def simple_option(name): opt = Keyword(name) + EQUALS +\ Word(alphas + nums + '-_') + SEMICOLON opt.setParseAction(self.set_simple_option) return opt branch_def = simple_option('branchlengths') MODEL_NAME = Word(alphas + nums + '+' + ' ' + '_') model_list = delimitedList(MODEL_NAME) model_def = 'models' + EQUALS + model_list + SEMICOLON model_def.setParseAction(self.set_models) model_selection_def = simple_option("model_selection") top_section = alignment_def + Optional(tree_def) + branch_def + \ model_def + model_selection_def # Data Block Parsing column = Word(nums) block_name = Word(alphas + '_-' + nums) block_def = column("start") +\ Optional(Suppress("-") + column("end")) +\ Optional(Suppress("\\") + column("step")) block_def.setParseAction(self.define_range) block_list_def = Group(OneOrMore(Group(block_def))) user_subset_def = Optional("charset") + block_name("name") + \ EQUALS + block_list_def("parts") + SEMICOLON user_subset_def.setParseAction(self.define_user_subset) block_def_list = OneOrMore(Group(user_subset_def)) block_section = Suppress("[data_blocks]") + block_def_list block_def_list.setParseAction(self.check_blocks) # Scheme Parsing scheme_name = Word(alphas + '_-' + nums) # Make a copy, cos we set a different action on it user_subset_ref = block_name.copy() user_subset_ref.setParseAction(self.check_block_exists) subset = Group( Suppress("(") + delimitedList(user_subset_ref("name")) + Suppress(")")) subset.setParseAction(self.define_subset_grouping) scheme = Group(OneOrMore(subset)) scheme_def = scheme_name("name") + \ EQUALS + scheme("scheme") + SEMICOLON scheme_def.setParseAction(self.define_scheme) scheme_list = OneOrMore(Group(scheme_def)) scheme_algo = simple_option("search") scheme_section = \ Suppress("[schemes]") + scheme_algo + Optional(scheme_list) # We've defined the grammar for each section. # Here we just put it all together self.config_parser = (top_section + block_section + scheme_section + stringEnd)
import re from pyparsing import ( Word, Keyword, NotAny, alphanums, nums, alphas, OneOrMore, srange, ZeroOrMore, Regex ) from whispy_lispy import ast int_literal = Word(nums) + NotAny('.') int_literal.setParseAction(ast.Int.from_parsed_result) float_literal = Word(nums) + Word('.') + Word(nums) float_literal.setParseAction(ast.Float.from_parsed_result) bool_literal = Keyword('#t') | Keyword('#f') bool_literal.setParseAction(ast.Bool.from_parsed_result) string_literal = Regex(r'\".*?(?<!\\)\"', re.DOTALL) string_literal.setParseAction(ast.String.from_parse_result) grammar = OneOrMore(float_literal | int_literal | bool_literal | string_literal)
# -*- coding: utf-8 -*- """Contains constants""" from pyparsing import Keyword, MatchFirst, oneOf from ..exc import PlaceholderAminoAcidWarning from ... import language aa_single = oneOf(list(language.amino_acid_dict.keys())) aa_single.setParseAction(lambda s, l, t: [language.amino_acid_dict[t[0]]]) aa_triple = oneOf(list(language.amino_acid_dict.values())) #: In biological literature, the X is used to denote a truncation. Text mining efforts often encode X as an amino #: acid, for which we will throw an error using :func:`handle_aa_placeholder` aa_placeholder = Keyword('X') def handle_aa_placeholder(line, position, tokens): """Raises an exception when encountering a placeholder amino acid, ``X``""" raise PlaceholderAminoAcidWarning(-1, line, position, tokens[0]) aa_placeholder.setParseAction(handle_aa_placeholder) amino_acid = MatchFirst([aa_triple, aa_single, aa_placeholder])
class Parser(object): """Lexical and Syntax analysis""" @property def semantic_analyser(self): return self._AST.semantic_analyser def __init__(self): self._AST = Syntax_tree() # keywords self.int_ = Keyword('Int') self.false_ = Keyword('False') self.true_ = Keyword('True') self.bit_ = Combine(Optional(Literal("@")) + Keyword('Bit')) self.sbox_ = Keyword('Sbox') self.l_shift_ = Keyword('<<') self.r_shift_ = Keyword('>>') self.circ_l_shift_ = Keyword('<<<') self.circ_r_shift_ = Keyword('>>>') self.bit_val = self.false_ ^ self.true_ self.if_ = Keyword('if') self.for_ = Keyword('for') self.return_ = Keyword('return') self.void_ = Keyword('void') self.ID = NotAny(self.sbox_ ^ self.int_ ^ self.bit_ ^ self.false_ ^ self.true_ ^ self.if_ ^ self.for_ ^ self.sbox_) + Word(alphas + '_', alphanums + '_') # NOQA self.ID_ = NotAny(self.sbox_ ^ self.int_ ^ self.bit_ ^ self.false_ ^ self.true_ ^ self.if_ ^ self.for_ ^ self.sbox_) + Word(alphas + '_', alphanums + '_') # Other Tokens self.l_bracket = Literal('(') self.r_bracket = Literal(')') self.eq_set = Literal('=')("set") self.term_st = Literal(';') self.b_2_num = Combine(Literal("0b") + Word("01")) self.b_2_num.setParseAction(self.semantic_analyser.convert_base_to_str) self.b_16_num = Combine(Literal("0x") + Word(srange("[0-9a-fA-F]"))) self.b_16_num.setParseAction(self.semantic_analyser.convert_base_to_str) self.b_10_num = Word(nums) self.bit_and = Literal('&') self.bit_or = Keyword('|') self.bit_xor = Keyword('^') self.bit_not = Literal('~') self.eq_compare = Literal('==') self.neq_compare = Literal('!=') self.l_brace = Literal('{') self.r_brace = Literal('}') self.bin_add = Literal('+') self.bin_mult = Literal('*') self.bin_sub = Literal('-') self.bin_mod = Literal('%') self.bin_div = Literal('/') self.g_than = Literal('>') self.ge_than = Literal('>=') self.l_than = Literal('<') self.le_than = Literal('<=') self.log_and = Keyword('&&') self.log_or = Keyword('||') self.l_sq_b = Literal('[') self.r_sq_b = Literal(']') # Operator Productions self.log_op = self.log_and ^ self.log_or self.comparison_op = self.g_than ^ self.ge_than ^ self.l_than ^ self.le_than ^ self.eq_compare ^ self.neq_compare self.arith_op = self.bin_add ^ self.bin_mult ^ self.bin_sub ^ self.bin_mod ^ self.bin_div self.bitwise_op = self.bit_and ^ self.bit_or ^ self.bit_xor ^ self.bit_not ^ self.l_shift_ ^ self.r_shift_ ^ self.circ_l_shift_ ^ self.circ_r_shift_ # Grammar self.stmt = Forward() self.for_loop = Forward() self.cast = Forward() self.seq_val = Forward() self.int_value = self.b_2_num ^ self.b_16_num ^ self.b_10_num self.expr = Forward() self.function_call = Forward() self.index_select = Forward() self.seq_ = Forward() self.operand = Forward() self.seq_range = Forward() # #######Operands self.sbox_call = Group((self.ID ^ self.seq_val) + ~White() + Literal(".") + ~White() + self.sbox_ + ~White() + self.l_bracket + (self.ID ^ self.int_value) + self.r_bracket) self.operand = self.index_select | self.seq_val | self.function_call | self.ID | self.int_value | self.cast | self.bit_val self.seq_val.setParseAction(lambda t: ['Seq_val'] + [t.asList()]) self.index_select.setParseAction(lambda t: ['index_select'] + [t.asList()]) self.function_call.setParseAction(lambda t: ['function_call'] + [t.asList()]) self.ID.setParseAction(lambda t: ['ID'] + [t.asList()]) self.int_value.setParseAction(lambda t: ['Int_val'] + [t.asList()]) self.cast.setParseAction(lambda t: ['cast'] + [t.asList()]) self.bit_val.setParseAction(lambda t: ['Bit_val'] + [t.asList()]) self.seq_range.setParseAction(lambda t: ['seq_range'] + [t.asList()]) # #######Expressions self.expr = Group(infixNotation(Group(self.operand), [(self.bitwise_op, 2, opAssoc.LEFT, self.nest_operand_pairs), (self.comparison_op, 2, opAssoc.LEFT, self.nest_operand_pairs), (self.log_op, 2, opAssoc.LEFT, self.nest_operand_pairs), (self.arith_op, 2, opAssoc.LEFT, self.nest_operand_pairs)])) # self.expr.setParseAction(self.expr_p) self.int_size = Combine(Optional(Literal("@")) + self.int_)("decl") + ~White() + Suppress(self.l_bracket) + self.expr + Suppress(self.r_bracket) self.sbox_size = self.sbox_ + ~White() + Suppress(self.l_bracket) + self.expr + Suppress(self.r_bracket) self.seq_range << self.expr + Suppress(Literal(":")) + self.expr self.seq_val << Suppress(self.l_sq_b) + Optional(Group(delimitedList(self.expr))) + Suppress(self.r_sq_b) self.seq_ << (self.int_size | self.bit_ | self.sbox_size)("type") +\ Group(OneOrMore(~White() + Suppress(self.l_sq_b) + self.expr + Suppress(self.r_sq_b)))("seq_size") self.function_call << self.ID("function_name") + ~White() + Suppress(self.l_bracket) +\ Optional(Group(delimitedList(self.expr)))("param_list") + Suppress(self.r_bracket) self.cast << Suppress(self.l_bracket) + Group((self.seq_ | self.int_size | self.bit_)) +\ Suppress(self.r_bracket) + (self.expr)("target") self.index_select << (self.ID("ID") ^ (Suppress(self.l_bracket) + self.cast + Suppress(self.r_bracket))("cast")) + ~White() +\ Group(OneOrMore(Suppress(self.l_sq_b) + Group(delimitedList(self.expr ^ Group(Group(self.seq_range))))("index") + Suppress(self.r_sq_b))) # ####### Declarations self.id_set = Group((Group(self.index_select) | self.ID_) + self.eq_set + self.expr) self.id_set.setParseAction(self.AST.id_set) self.int_decl = Group(self.int_size + delimitedList(Group((self.ID_("ID") + self.eq_set + self.expr("set_value")) | self.ID_("ID")))("value")) # NOQA self.int_decl.setParseAction(self.AST.int_decl) self.bit_decl = Group(self.bit_("decl") + delimitedList(Group(self.ID_("ID")) ^ Group(self.ID_("ID") + self.eq_set + self.expr("set_value")))("value")) self.bit_decl.setParseAction(self.AST.bit_decl) self.seq_decl = Group(self.seq_("decl") + Group(self.ID)("ID") + Optional(self.eq_set + Group(self.expr))("value")) self.seq_decl.setParseAction(self.AST.seq_decl) self.decl = self.bit_decl ^ self.int_decl ^ self.seq_decl # ###### Statements self.return_stmt = Group(self.return_ + self.expr) self.return_stmt.setParseAction(self.AST.return_stmt) self.function_start = Literal("{") self.function_start.setParseAction(self.AST.function_start) self.function_end = Literal("}") self.function_decl = Group((Group(self.seq_) | Group(self.int_size) | Group(self.bit_) | Group(self.void_))("return_type") + Group(self.ID)("func_ID") + Suppress(self.l_bracket) + Group(Optional(delimitedList(Group((self.seq_ | self.int_size | self.bit_) + Group(self.ID)))))("func_param") + # NOQA Suppress(self.r_bracket) + Suppress(self.function_start) + Group(self.stmt)("body") + Suppress(self.r_brace)) self.function_decl.setParseAction(self.AST.function_decl) self.for_init = Literal('(') self.for_init.setParseAction(self.AST.begin_for) self.for_terminator = Literal(';') self.for_terminator.setParseAction(self.AST.for_terminator) self.for_increment = Literal(';') self.for_increment.setParseAction(self.AST.for_increment) self.terminator_expr = Group(infixNotation(Group(self.operand), [(self.log_op, 2, opAssoc.LEFT, self.nest_operand_pairs), (self.bitwise_op, 2, opAssoc.LEFT, self.nest_operand_pairs), (self.comparison_op, 2, opAssoc.LEFT, self.nest_operand_pairs), (self.arith_op, 2, opAssoc.LEFT, self.nest_operand_pairs)])) self.terminator_expr.setParseAction(self.AST.terminator_expr) self.for_body = Literal('{') self.for_body.setParseAction(self.AST.for_body) self.end_for = Literal('}') self.end_for.setParseAction(self.AST.end_for) self.for_loop << Group(self.for_ + ~White() + Suppress(self.for_init) + Optional(delimitedList(self.decl ^ self.id_set))("init") + Suppress(self.for_terminator) + Optional(self.terminator_expr) + Suppress(self.for_increment) + Optional(delimitedList(self.id_set))("increm") + Suppress(self.r_bracket) + Suppress(self.for_body) + self.stmt("loop_body") + Suppress(self.end_for)) self.if_condition = Suppress(self.l_bracket) + self.expr + Suppress(self.r_bracket) self.if_condition.setParseAction(self.AST.if_cond) self.if_.setParseAction(self.AST.begin_if) self.if_body_st = Literal('{') self.if_body_st.setParseAction(self.AST.if_body_st) self.if_body_end = Literal('}') self.if_body_end.setParseAction(self.AST.if_body_end) self.if_stmt = Group(self.if_ + self.if_condition("if_cond") + Suppress(self.if_body_st) + Group(self.stmt).setResultsName("body") + Suppress(self.if_body_end)) self.single_expr = self.expr + Suppress(self.term_st) self.single_expr.setParseAction(self.AST.stand_alone_expr) self.stmt << ZeroOrMore(self.decl + Suppress(self.term_st) ^ self.function_decl ^ self.id_set + Suppress(self.term_st) ^ self.single_expr ^ self.for_loop ^ self.if_stmt ^ self.return_stmt + Suppress(self.term_st) ^ self.sbox_call + Suppress(self.term_st)) self.grammar_test = self.stmt + StringEnd() # Allows single statements to be parsed self.grammar = ZeroOrMore(self.function_decl ^ self.seq_decl + Suppress(self.term_st)) + StringEnd() def nest_operand_pairs(self, tokens): tokens = tokens[0] ret = ParseResults(tokens[:3]) remaining = iter(tokens[3:]) done = False while not done: next_pair = (next(remaining, None), next(remaining, None)) if next_pair == (None, None): done = True break ret = ParseResults([ret]) ret += ParseResults(list(next_pair)) return [ret] @property def AST(self): return self._AST @AST.setter def AST(self, value): self._AST = value def analyse_tree_test(self, AST): return self.semantic_analyser.analyse(AST) def parse_test_unit(self, data_in): """Parses single statements""" try: res = self.grammar_test.parseString(data_in) except ParseException as details: print("The following error occured:") print(details) return False if type(res[0]) is not bool: pass # print(res[0].dump()) return [res, True] def parse_test_AST_semantic(self, data_in): """Parses single statements and returns AST""" try: self.grammar_test.parseString(data_in) except ParseException as details: print("The following error occured:") print(details) return False return self.AST def parse_test_integration(self, data_in): """Only Parses Statements in functions""" try: res = self.grammar.parseString(data_in) except ParseException as details: print("The following error occured:") print(details) return False # if type(res[0]) is not bool: # print(res[0].dump()) return [res, True] def parse(self, data_in): """Prod parsing entry point""" self.grammar.parseString(data_in) if self.semantic_analyser.analyse(self.AST, True) is True: return self.semantic_analyser.IR.translate()
def SPICE_BNF(): global bnf if not bnf: # punctuation colon = Literal(":").suppress() lbrace = Literal("{").suppress() rbrace = Literal("}").suppress() lbrack = Literal("[").suppress() rbrack = Literal("]").suppress() lparen = Literal("(").suppress() rparen = Literal(")").suppress() equals = Literal("=").suppress() comma = Literal(",").suppress() semi = Literal(";").suppress() # primitive types int8_ = Keyword("int8").setParseAction(replaceWith(ptypes.int8)) uint8_ = Keyword("uint8").setParseAction(replaceWith(ptypes.uint8)) int16_ = Keyword("int16").setParseAction(replaceWith(ptypes.int16)) uint16_ = Keyword("uint16").setParseAction(replaceWith(ptypes.uint16)) int32_ = Keyword("int32").setParseAction(replaceWith(ptypes.int32)) uint32_ = Keyword("uint32").setParseAction(replaceWith(ptypes.uint32)) int64_ = Keyword("int64").setParseAction(replaceWith(ptypes.int64)) uint64_ = Keyword("uint64").setParseAction(replaceWith(ptypes.uint64)) # keywords enum32_ = Keyword("enum32").setParseAction(replaceWith(32)) enum16_ = Keyword("enum16").setParseAction(replaceWith(16)) enum8_ = Keyword("enum8").setParseAction(replaceWith(8)) flags32_ = Keyword("flags32").setParseAction(replaceWith(32)) flags16_ = Keyword("flags16").setParseAction(replaceWith(16)) flags8_ = Keyword("flags8").setParseAction(replaceWith(8)) channel_ = Keyword("channel") server_ = Keyword("server") client_ = Keyword("client") protocol_ = Keyword("protocol") typedef_ = Keyword("typedef") struct_ = Keyword("struct") message_ = Keyword("message") image_size_ = Keyword("image_size") bytes_ = Keyword("bytes") cstring_ = Keyword("cstring") switch_ = Keyword("switch") default_ = Keyword("default") case_ = Keyword("case") identifier = Word(alphas, alphanums + "_") enumname = Word(alphanums + "_") integer = ( (Combine(CaselessLiteral("0x") + Word(nums + "abcdefABCDEF")) | Word(nums + "+-", nums)) .setName("int") .setParseAction(cvtInt) ) typename = identifier.copy().setParseAction(lambda toks: ptypes.TypeRef(str(toks[0]))) # This is just normal "types", i.e. not channels or messages typeSpec = Forward() attributeValue = integer ^ identifier attribute = Group(Combine("@" + identifier) + Optional(lparen + delimitedList(attributeValue) + rparen)) attributes = Group(ZeroOrMore(attribute)) arraySizeSpecImage = Group(image_size_ + lparen + integer + comma + identifier + comma + identifier + rparen) arraySizeSpecBytes = Group(bytes_ + lparen + identifier + comma + identifier + rparen) arraySizeSpecCString = Group(cstring_ + lparen + rparen) arraySizeSpec = ( lbrack + Optional( identifier ^ integer ^ arraySizeSpecImage ^ arraySizeSpecBytes ^ arraySizeSpecCString, default="" ) + rbrack ) variableDef = Group( typeSpec + Optional("*", default=None) + identifier + Optional(arraySizeSpec, default=None) + attributes - semi ).setParseAction(parseVariableDef) switchCase = Group( Group( OneOrMore( default_.setParseAction(replaceWith(None)) + colon | Group(case_.suppress() + Optional("!", default="") + identifier) + colon ) ) + variableDef ).setParseAction(lambda toks: ptypes.SwitchCase(toks[0][0], toks[0][1])) switchBody = Group( switch_ + lparen + delimitedList(identifier, delim=".", combine=True) + rparen + lbrace + Group(OneOrMore(switchCase)) + rbrace + identifier + attributes - semi ).setParseAction(lambda toks: ptypes.Switch(toks[0][1], toks[0][2], toks[0][3], toks[0][4])) messageBody = structBody = Group(lbrace + ZeroOrMore(variableDef | switchBody) + rbrace) structSpec = Group(struct_ + identifier + structBody + attributes).setParseAction( lambda toks: ptypes.StructType(toks[0][1], toks[0][2], toks[0][3]) ) # have to use longest match for type, in case a user-defined type name starts with a keyword type, like "channel_type" typeSpec << ( structSpec ^ int8_ ^ uint8_ ^ int16_ ^ uint16_ ^ int32_ ^ uint32_ ^ int64_ ^ uint64_ ^ typename ).setName("type") flagsBody = enumBody = Group( lbrace + delimitedList(Group(enumname + Optional(equals + integer))) + Optional(comma) + rbrace ) messageSpec = ( Group(message_ + messageBody + attributes).setParseAction( lambda toks: ptypes.MessageType(None, toks[0][1], toks[0][2]) ) | typename ) channelParent = Optional(colon + typename, default=None) channelMessage = Group( messageSpec + identifier + Optional(equals + integer, default=None) + semi ).setParseAction(lambda toks: ptypes.ChannelMember(toks[0][1], toks[0][0], toks[0][2])) channelBody = channelParent + Group( lbrace + ZeroOrMore(server_ + colon | client_ + colon | channelMessage) + rbrace ) enum_ = enum32_ | enum16_ | enum8_ flags_ = flags32_ | flags16_ | flags8_ enumDef = Group(enum_ + identifier + enumBody + attributes - semi).setParseAction( lambda toks: ptypes.EnumType(toks[0][0], toks[0][1], toks[0][2], toks[0][3]) ) flagsDef = Group(flags_ + identifier + flagsBody + attributes - semi).setParseAction( lambda toks: ptypes.FlagsType(toks[0][0], toks[0][1], toks[0][2], toks[0][3]) ) messageDef = Group(message_ + identifier + messageBody + attributes - semi).setParseAction( lambda toks: ptypes.MessageType(toks[0][1], toks[0][2], toks[0][3]) ) channelDef = Group(channel_ + identifier + channelBody + attributes - semi).setParseAction( lambda toks: ptypes.ChannelType(toks[0][1], toks[0][2], toks[0][3], toks[0][4]) ) structDef = Group(struct_ + identifier + structBody + attributes - semi).setParseAction( lambda toks: ptypes.StructType(toks[0][1], toks[0][2], toks[0][3]) ) typedefDef = Group(typedef_ + identifier + typeSpec + attributes - semi).setParseAction( lambda toks: ptypes.TypeAlias(toks[0][1], toks[0][2], toks[0][3]) ) definitions = typedefDef | structDef | enumDef | flagsDef | messageDef | channelDef protocolChannel = Group(typename + identifier + Optional(equals + integer, default=None) + semi).setParseAction( lambda toks: ptypes.ProtocolMember(toks[0][1], toks[0][0], toks[0][2]) ) protocolDef = Group( protocol_ + identifier + Group(lbrace + ZeroOrMore(protocolChannel) + rbrace) + semi ).setParseAction(lambda toks: ptypes.ProtocolType(toks[0][1], toks[0][2])) bnf = ZeroOrMore(definitions) + protocolDef + StringEnd() singleLineComment = "//" + restOfLine bnf.ignore(singleLineComment) bnf.ignore(cStyleComment) return bnf
# stateMachine.py # # module to define .pystate import handler # #import imputil import sys import os import types import urlparse DEBUG = False from pyparsing import Word, Group, ZeroOrMore, alphas, \ alphanums, ParserElement, ParseException, ParseSyntaxException, \ Empty, LineEnd, OneOrMore, col, Keyword, pythonStyleComment, \ StringEnd, traceParseAction ident = Word(alphas+"_", alphanums+"_$") pythonKeywords = """and as assert break class continue def del elif else except exec finally for from global if import in is lambda None not or pass print raise return try while with yield True False""" pythonKeywords = set(pythonKeywords.split()) def no_keywords_allowed(s,l,t): wd = t[0] if wd in pythonKeywords: errmsg = "cannot not use keyword '%s' " \ "as an identifier" % wd raise ParseException(s,l,errmsg) ident.setParseAction(no_keywords_allowed) stateTransition = ident("fromState") + "->" + ident("toState") stateMachine = Keyword("statemachine") + \
def parse(input): # parse a string into an element of the abstract representation # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # ( if <expr> <expr> <expr> ) # ( let ( ( <name> <expr> ) ) <expr ) # ( function ( <name> ... ) <expr> ) # ( ref <expr> ) # ( <expr> <expr> ... ) # idChars = alphas + "_+*-?!=<>" pIDENTIFIER = Word(idChars, idChars + "0123456789") pIDENTIFIER.setParseAction(lambda result: EId(result[0])) # A name is like an identifier but it does not return an EId... pNAME = Word(idChars, idChars + "0123456789") pNAMES = ZeroOrMore(pNAME) pNAMES.setParseAction(lambda result: [result]) pINTEGER = Word("0123456789") pINTEGER.setParseAction(lambda result: EValue(VInteger(int(result[0])))) pBOOLEAN = Keyword("true") | Keyword("false") pBOOLEAN.setParseAction( lambda result: EValue(VBoolean(result[0] == "true"))) pEXPR = Forward() pEXPRS = ZeroOrMore(pEXPR) pEXPRS.setParseAction(lambda result: [result]) pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")" pIF.setParseAction(lambda result: EIf(result[2], result[3], result[4])) pBINDING = "(" + pNAME + pEXPR + ")" pBINDING.setParseAction(lambda result: (result[1], result[2])) pBINDINGS = ZeroOrMore(pBINDING) pBINDINGS.setParseAction(lambda result: [result]) def makeLet(bindings, body): params = [param for (param, exp) in bindings] args = [exp for (param, exp) in bindings] return ECall(EFunction(params, body), args) pLET = "(" + Keyword("let") + "(" + pBINDINGS + ")" + pEXPR + ")" pLET.setParseAction(lambda result: makeLet(result[3], result[5])) pCALL = "(" + pEXPR + pEXPRS + ")" pCALL.setParseAction(lambda result: ECall(result[1], result[2])) pFUN = "(" + Keyword("function") + "(" + pNAMES + ")" + pEXPR + ")" pFUN.setParseAction(lambda result: EFunction(result[3], result[5])) pFUNrec = "(" + Keyword( "function") + pNAME + "(" + pNAMES + ")" + pEXPR + ")" pFUNrec.setParseAction( lambda result: EFunction(result[4], result[6], name=result[2])) def makeDo(exprs): result = exprs[-1] for e in reversed(exprs[:-1]): # space is not an allowed identifier in the syntax! result = makeLet([(" ", e)], result) return result pDO = "(" + Keyword("do") + pEXPRS + ")" pDO.setParseAction(lambda result: makeDo(result[2])) def makeWhile(cond, body): return makeLet( [(" while", EFunction([], EIf(cond, makeLet([(" ", body)], ECall(EId(" while"), [])), EValue(VNone())), name=" while"))], ECall(EId(" while"), [])) pWHILE = "(" + Keyword("while") + pEXPR + pEXPR + ")" pWHILE.setParseAction(lambda result: makeWhile(result[2], result[3])) pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pIF | pLET | pFUN | pFUNrec | pDO | pWHILE | pCALL) # can't attach a parse action to pEXPR because of recursion, so let's duplicate the parser pTOPEXPR = pEXPR.copy() pTOPEXPR.setParseAction(lambda result: { "result": "expression", "expr": result[0] }) pDEFINE = "(" + Keyword("define") + pNAME + pEXPR + ")" pDEFINE.setParseAction(lambda result: { "result": "value", "name": result[2], "expr": result[3] }) pDEFUN = "(" + Keyword("defun") + pNAME + "(" + pNAMES + ")" + pEXPR + ")" pDEFUN.setParseAction( lambda result: { "result": "function", "name": result[2], "params": result[4], "body": result[6] }) pABSTRACT = "#abs" + pEXPR pABSTRACT.setParseAction(lambda result: { "result": "abstract", "expr": result[1] }) pQUIT = Keyword("#quit") pQUIT.setParseAction(lambda result: {"result": "quit"}) pTOP = (pDEFUN | pDEFINE | pQUIT | pABSTRACT | pTOPEXPR) result = pTOP.parseString(input)[0] return result # the first element of the result is the expression
# stateMachine.py # # module to define .pystate import handler # #import imputil import sys import os import types import urlparse DEBUG = False from pyparsing import Word, Group, ZeroOrMore, alphas, \ alphanums, ParserElement, ParseException, ParseSyntaxException, \ Empty, LineEnd, OneOrMore, col, Keyword, pythonStyleComment, \ StringEnd, traceParseAction ident = Word(alphas + "_", alphanums + "_$") pythonKeywords = """and as assert break class continue def del elif else except exec finally for from global if import in is lambda None not or pass print raise return try while with yield True False""" pythonKeywords = set(pythonKeywords.split()) def no_keywords_allowed(s, l, t): wd = t[0] if wd in pythonKeywords: errmsg = "cannot not use keyword '%s' " \ "as an identifier" % wd raise ParseException(s, l, errmsg) ident.setParseAction(no_keywords_allowed) stateTransition = ident("fromState") + "->" + ident("toState")
def parse (input): # parse a string into an element of the abstract representation # Grammar: # # <expr> ::= <integer> # true # false # <identifier> # ( if <expr> <expr> <expr> ) # ( let ( ( <name> <expr> ) ... ) <expr ) # ( function ( <name> ... ) <expr> ) # ( <expr> <expr> ... ) # ( call/cc <expr>) # idChars = alphas+"_+*-?!=<>" pIDENTIFIER = Word(idChars, idChars+"0123456789") pIDENTIFIER.setParseAction(lambda result: EId(result[0])) # A name is like an identifier but it does not return an EId... pNAME = Word(idChars,idChars+"0123456789") pNAMES = ZeroOrMore(pNAME) pNAMES.setParseAction(lambda result: [result]) pINTEGER = Word("0123456789") pINTEGER.setParseAction(lambda result: EValue(VInteger(int(result[0])))) pBOOLEAN = Keyword("true") | Keyword("false") pBOOLEAN.setParseAction(lambda result: EValue(VBoolean(result[0]=="true"))) pEXPR = Forward() pEXPRS = ZeroOrMore(pEXPR) pEXPRS.setParseAction(lambda result: [result]) pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")" pIF.setParseAction(lambda result: EIf(result[2],result[3],result[4])) pBINDING = "(" + pNAME + pEXPR + ")" pBINDING.setParseAction(lambda result: (result[1],result[2])) pBINDINGS = ZeroOrMore(pBINDING) pBINDINGS.setParseAction(lambda result: [ result ]) def makeLet (bindings,body): params = [ param for (param,exp) in bindings ] args = [ exp for (param,exp) in bindings ] return ECall(EFunction(params,body),args) pLET = "(" + Keyword("let") + "(" + pBINDINGS + ")" + pEXPR + ")" pLET.setParseAction(lambda result: makeLet(result[3],result[5])) pCALL = "(" + pEXPR + pEXPRS + ")" pCALL.setParseAction(lambda result: ECall(result[1],result[2])) pFUN = "(" + Keyword("function") + "(" + pNAMES + ")" + pEXPR + ")" pFUN.setParseAction(lambda result: EFunction(result[3],result[5])) pFUNrec = "(" + Keyword("function") + pNAME + "(" + pNAMES + ")" + pEXPR + ")" pFUNrec.setParseAction(lambda result: EFunction(result[4],result[6],name=result[2])) def makeDo (exprs): result = exprs[-1] for e in reversed(exprs[:-1]): # space is not an allowed identifier in the syntax! result = makeLet([(" ",e)],result) return result pDO = "(" + Keyword("do") + pEXPRS + ")" pDO.setParseAction(lambda result: makeDo(result[2])) def makeWhile (cond,body): return makeLet([(" while", EFunction([],EIf(cond,makeLet([(" ",body)],ECall(EId(" while"),[])),EValue(VNone())),name=" while"))], ECall(EId(" while"),[])) pWHILE = "(" + Keyword("while") + pEXPR + pEXPR + ")" pWHILE.setParseAction(lambda result: makeWhile(result[2],result[3])) pCALLCC = "(" + Keyword("call/cc") + pEXPR + ")" pCALLCC.setParseAction(lambda result: ECallCC(result[2])) pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pIF | pLET | pFUN | pFUNrec| pDO | pWHILE | pCALLCC | pCALL) # can't attach a parse action to pEXPR because of recursion, so let's duplicate the parser pTOPEXPR = pEXPR.copy() pTOPEXPR.setParseAction(lambda result: {"result":"expression","expr":result[0]}) pDEFINE = "(" + Keyword("define") + pNAME + pEXPR + ")" pDEFINE.setParseAction(lambda result: {"result":"value", "name":result[2], "expr":result[3]}) pDEFUN = "(" + Keyword("defun") + pNAME + "(" + pNAMES + ")" + pEXPR + ")" pDEFUN.setParseAction(lambda result: {"result":"function", "name":result[2], "params":result[4], "body":result[6]}) pABSTRACT = "#abs" + pEXPR pABSTRACT.setParseAction(lambda result: {"result":"abstract", "expr":result[1]}) pQUIT = Keyword("#quit") pQUIT.setParseAction(lambda result: {"result":"quit"}) pTOP = (pDEFUN | pDEFINE | pQUIT | pABSTRACT | pTOPEXPR) result = pTOP.parseString(input)[0] return result # the first element of the result is the expression
tr_matrix_data = Suppress("(tr)") + matrix_data tr_matrix_data.setParseAction(mark_transposed) set_slice_component = number | symbol | '*' set_slice_record = LPAREN + NotAny('tr') + delimitedList( set_slice_component) + RPAREN set_slice_record.setParseAction(SliceRecord) _set_record = set_slice_record | matrix_data | tr_matrix_data | Suppress( ":=") set_record = simple_data | _set_record non_dimen_set_record = non_dimen_simple_data | _set_record set_def_stmt = Keyword("set") + symbol + Optional(subscript_domain) + \ Optional(Keyword("dimen") + integer.setResultsName('dimen')) + END set_def_stmt.setParseAction(SetDefStmt) set_member = LBRACKET + delimitedList(data) + RBRACKET set_stmt = Keyword("set") + symbol + Optional(set_member).setResultsName("member") + \ Group(non_dimen_set_record + ZeroOrMore(Optional(Suppress(',')) + set_record)) \ .setResultsName("records") + END set_stmt.setParseAction(SetStmt) subscript = single param_data = data | '.' plain_data = param_data | subscript + ZeroOrMore( Optional(Suppress(',')) + subscript) + param_data # should not match a single (tr) plain_data_record = Group(NotAny('(tr)') + plain_data + NotAny(plain_data) | \
alphanums, Keyword, ) word_free = Word(alphas8bit + "_-/.+**" + alphanums) word_strict = Word(alphas8bit + alphas, alphas8bit + alphanums + "_") (lparen, rparen, lbrack, rbrack, lbrace, rbrace, colon) = map(Suppress, "()[]{}:") integer = Combine(Optional(oneOf("+ -")) + Word(nums)).setName("integer") cvt_int = lambda toks: int(toks[0]) integer.setParseAction(cvt_int) boolean = Keyword("False", caseless=True) cvt_bool = lambda toks: toks[0].lower == "true" boolean.setParseAction(cvt_bool) none = Keyword("None", caseless=True) cvt_none = lambda toks: [None] none.setParseAction(cvt_none) real = Combine( Optional(oneOf("+ -")) + Word(nums) + "." + Optional(Word(nums)) + Optional("e" + Optional(oneOf("+ -")) + Word(nums)) ).setName("real") cvt_real = lambda toks: float(toks[0]) real.setParseAction(cvt_real)