def _module_decl(self): """ BNF from IEEE1364-2005 module_declaration ::= { attribute_instance } module_keyword module_identifier [ module_parameter_port_list ] list_of_ports ; { module_item } endmodule | { attribute_instance } module_keyword module_identifier [ module_parameter_port_list ] [ list_of_port_declarations ] ; { non_port_module_item } endmodule module_keyword ::= module | macromodule module_identifier ::= identifier Unimplemented features: { attribute_instance } """ ENDMODULE = Suppress(Keyword('endmodule')) def f_new_module(t): try: new_module = ElemModule(name=t.ModuleHeader.Name) except Exception as e: print(e) for each_port in t.ModuleHeader.PortList: if isinstance(each_port, ElemExportPort): new_module.add_port(each_port) else: elem_list = VlogPModule.build_elem_port_list(each_port) for n in elem_list: np = ElemExportPort(name=n.name) np.set_port(name=n.name, obj=n) new_module.put(name=n.name, symbol=n) new_module.add_port(np) return new_module # Names in module parsing result # module_decl = ['ModuleHeader': ['Name': ..., # 'PortList':[...]], # 'ModuleItems': ...] module_hdr = Group( Suppress(oneOf("module macromodule")) + identifier('Name') + Optional(self.module_parameter_port_list)('ParamDeclList') + Optional(LPARENTH + Group( Optional( delimitedList( # Group(oneOf("input output") + # (netDecl1Arg | netDecl2Arg | netDecl3Arg)) | self.port_declaration | self.port('Port')))).setResultsName('PortList') + RPARENTH) + SEMI).setResultsName('ModuleHeader').setParseAction(f_new_module) module_hdr.setDebug(False) self.module_decl = Group( module_hdr.addParseAction( lambda t: self.push_scope(t.ModuleHeader.scope)) + Group(ZeroOrMore(self.module_item)).setResultsName('ModuleItems') + ENDMODULE).setParseAction(lambda t: self.pop_scope()) return self.module_decl
class SExpressionParser(object): def __init__(self): self.lpar = Literal('(') self.rpar = Literal(')') self.word_chars = ''.join(c for c in printables if c not in ('()')) self.word = Word(self.word_chars) | quotedString self.atom = self.word self.expression = Forward() self.composite_expression = ( Suppress(self.lpar) + ZeroOrMore(self.expression) + Suppress(self.rpar))('composite_expression') self.composite_expression.addParseAction( self._composite_expression_to_tuple) self.expression << (self.atom | self.composite_expression) self.expressions = Group(ZeroOrMore(self.expression))('expressions') self.expressions.addParseAction(self._expressions_to_tuple) def parse_expression(self, instring): return self.expression.parseString(instring, parseAll=True)[0] def parse_expressions(self, instring): return self.expressions.parseString(instring, parseAll=True)[0] @staticmethod def _composite_expression_to_tuple(toks): return SExpression(toks.composite_expression) @staticmethod def _expressions_to_tuple(toks): return SExpressionList(toks.expressions)
IntegerTok.addParseAction(lambda toks: int("".join(toks))) UnsignedIntTok = Word(pyparsing.nums) UnsignedIntTok.addParseAction(lambda toks: int(toks[0])) FloatTok = Optional(Literal('-')) + Word( pyparsing.nums) + Optional(Literal('.') + Optional(Word(pyparsing.nums))) FloatTok.addParseAction(lambda toks: float("".join(toks))) HexStringTok = Word(pyparsing.hexnums) HexStringTok.addParseAction(lambda toks: int(toks[0], base=16)) UnquotedStringTok = ZeroOrMore( White()).suppress() + CharsNotIn("()\"\'" + " \r\n") UnquotedStringTok.addParseAction(lambda toks: "".join(toks).strip()) QuotedStringTok = Group(dblQuotedString() ^ sglQuotedString()) QuotedStringTok.addParseAction(lambda toks: "".join(toks[0]).strip('"')) AnystringTok = QuotedStringTok ^ UnquotedStringTok LeftParenTok = Literal('(').suppress() RightParenTok = Literal(')').suppress() BoolTrueTok = Keyword("yes", caseless=True) | Keyword("true", caseless=True) BoolTrueTok.addParseAction(lambda: True) BoolFalseTok = Keyword("no", caseless=True) | Keyword("false", caseless=True) BoolFalseTok.addParseAction(lambda: False) BooleanTok = BoolTrueTok | BoolFalseTok def _paren_stmt(keyword, *values, store=True): """ Create a parser for a parenthesized list with an initial keyword.
def __init__(self): # define SQL tokens selectStmt = Forward() selectToken = Keyword("select", caseless=True) fromToken = Keyword("from", caseless=True) asToken = Keyword("as", caseless=True) whereToken = Keyword("where", caseless=True) semicolon = Literal(";") ident = Word( alphas, alphanums + "_$" ).setName("identifier") columnName = delimitedList( ident, ".", combine=True ) #columnName.setParseAction(upcaseTokens) columnNameList = Group( columnName + ZeroOrMore("," + columnName)) # selectableList = Forward() columnRvalList = Forward() functionExpr = ident + Optional("."+ident) + Literal('(') + columnRvalList + Literal(')') alias = Forward() identExpr = functionExpr | ident self.identExpr = identExpr # Debug self.functionExpr = functionExpr # Debug alias = ident.copy() selectableName = identExpr | columnName selectableList = Group( selectableName + ZeroOrMore(","+selectableName)) columnRef = columnName functionSpec = functionExpr valueExprPrimary = functionSpec | columnRef numPrimary = valueExprPrimary ## | numericValFunc factor = Optional(Literal("+") | Literal("-")) + numPrimary muldiv = oneOf("* /") term = Forward() term << factor + Optional(muldiv + factor) numericExpr = Forward() addsub = oneOf("+ -") numericExpr << term + Optional(addsub + numericExpr) arithop = oneOf("+ - * /") columnNumericExpr = Forward() cTerm = valueExprPrimary testme = valueExprPrimary + arithop + valueExprPrimary columnNumericExpr << cTerm + Optional(arithop + columnNumericExpr) colNumExpList = Group( columnNumericExpr + ZeroOrMore(","+columnNumericExpr)) valueExpr = numericExpr ## | stringExpr | dateExpr | intervalExpr derivedColumn = valueExpr + Optional(asToken + alias) selectSubList = derivedColumn + ZeroOrMore("," + derivedColumn) tableName = delimitedList( ident, ".", combine=True ) # don't upcase table names anymore # tableName.setParseAction(upcaseTokens) self.tableAction = [] tableName.addParseAction(self.actionWrapper(self.tableAction)) tableName.setResultsName("table") tableAlias = tableName + asToken + ident.setResultsName("aliasName") tableAlias.setResultsName("alias") genericTableName = tableAlias | tableName genericTableName = genericTableName.setResultsName("tablename") tableNameList = Group( genericTableName + ZeroOrMore("," + genericTableName)) whereExpression = Forward() and_ = Keyword("and", caseless=True) or_ = Keyword("or", caseless=True) in_ = Keyword("in", caseless=True) between_ = Keyword("between", caseless=True) E = CaselessLiteral("E") binop = oneOf("= != < > >= <= eq ne lt le gt ge", caseless=True) arithSign = Word("+-",exact=1) realNum = Combine( Optional(arithSign) + ( Word( nums ) + "." + Optional( Word(nums) ) | ( "." + Word(nums) ) ) + Optional( E + Optional(arithSign) + Word(nums) ) ) intNum = Combine( Optional(arithSign) + Word( nums ) + Optional( E + Optional("+") + Word(nums) ) ) # need to add support for alg expressions columnRval = realNum | intNum | quotedString | columnNumericExpr# | numericExpr columnRvalList << Group( columnRval + ZeroOrMore("," + columnRval)) self.whereExpAction = [] namedRv = columnRval.setResultsName("column") whereConditionFlat = Group( ( functionSpec + binop + columnRval) | ( namedRv + binop + columnRval ) | ( namedRv + in_ + "(" + columnRval + ZeroOrMore(","+namedRv) + ")" ) | ( namedRv + in_ + "(" + selectStmt + ")" ) | ( namedRv + between_ + namedRv + and_ + namedRv ) ) whereConditionFlat.addParseAction(self.actionWrapper(self.whereExpAction)) whereCondition = Group(whereConditionFlat | ( "(" + whereExpression + ")" )) # Test code to try to make an expression parse. # print whereConditionFlat.parseString("ABS(o1.ra - o2.ra) < 0.00083 / COS(RADIANS(o2.decl))") # goodFunction = ident + Literal('(') + columnNumericExpr + Literal(')') # print "ADFDSFDSF",testme.parseString("o1.ra - o2.ra", parseAll=True) # print "ADSFDSFAD", goodFunction.parseString("ABS(o1.ra - o2.ra)") #whereExpression << whereCondition.setResultsName("wherecond") #+ ZeroOrMore( ( and_ | or_ ) + whereExpression ) def scAnd(tok): print "scAnd", tok if "TRUE" == tok[0][0]: tok = tok[2] elif "TRUE" == tok[2][0]: tok = tok[0] return tok def scOr(tok): print "scOr", tok if ("TRUE" == tok[0][0]) or ("TRUE" == tok[2][0]): tok = [["TRUE"]] return tok def scWhere(tok): newtok = [] i = 0 while i < len(tok): if str(tok[i]) in ["TRUE",str(["TRUE"])] and (i+1) < len(tok): if str(tok[i+1]).upper() == "AND": i += 2 continue elif str(tok[i+i]).upper() == "OR": break newtok.append(tok[i]) i += 1 return newtok def collapseWhere(tok): #collapse.append(tok[0][1]) if ["TRUE"] == tok.asList()[0][1]: tok = [] return tok andExpr = and_ + whereExpression orExpr = or_ + whereExpression whereExpression << whereCondition + ZeroOrMore( andExpr | orExpr) whereExpression.addParseAction(scWhere) self.selectPart = selectToken + ( '*' | selectSubList ).setResultsName( "columns" ) whereClause = Group(whereToken + whereExpression).setResultsName("where") whereClause.addParseAction(collapseWhere) self.fromPart = fromToken + tableNameList.setResultsName("tables") # define the grammar selectStmt << ( self.selectPart + fromToken + tableNameList.setResultsName( "tables" ) + whereClause) self.simpleSQL = selectStmt + semicolon # define Oracle comment format, and ignore them oracleSqlComment = "--" + restOfLine self.simpleSQL.ignore( oracleSqlComment )
import sys import pyparsing from pprint import pprint from functools import reduce from pyparsing import Literal, Word, Forward, Group, nums, Regex, oneOf, StringEnd, OneOrMore L = Literal Ls = lambda expr: L(expr).suppress() # atoms integer = Word(nums + '-') integer.setParseAction(lambda t: int(t[0])) comma = Ls(',') vector = Group(Ls('<') + integer + comma + integer + comma + integer + Ls('>')) vector.addParseAction(lambda t: tuple(t[0])) p, v, a = L('p'), L('v'), L('a') eq = Ls('=') row = Group( Group(p + eq + vector) + comma + Group(v + eq + vector) + comma + Group(a + eq + vector)) document = OneOrMore(row) + StringEnd() def vadd(x, y): return (x[0] + y[0], x[1] + y[1], x[2] + y[2]) def vmul(c, x): return (c * x[0], c * x[1], c * x[2])
def __init__(self): # define SQL tokens selectStmt = Forward() selectToken = Keyword("select", caseless=True) fromToken = Keyword("from", caseless=True) asToken = Keyword("as", caseless=True) whereToken = Keyword("where", caseless=True) semicolon = Literal(";") ident = Word(alphas, alphanums + "_$").setName("identifier") columnName = delimitedList(ident, ".", combine=True) #columnName.setParseAction(upcaseTokens) columnNameList = Group(columnName + ZeroOrMore("," + columnName)) # selectableList = Forward() columnRvalList = Forward() functionExpr = ident + Optional("." + ident) + Literal( '(') + columnRvalList + Literal(')') alias = Forward() identExpr = functionExpr | ident self.identExpr = identExpr # Debug self.functionExpr = functionExpr # Debug alias = ident.copy() selectableName = identExpr | columnName selectableList = Group(selectableName + ZeroOrMore("," + selectableName)) columnRef = columnName functionSpec = functionExpr valueExprPrimary = functionSpec | columnRef numPrimary = valueExprPrimary ## | numericValFunc factor = Optional(Literal("+") | Literal("-")) + numPrimary muldiv = oneOf("* /") term = Forward() term << factor + Optional(muldiv + factor) numericExpr = Forward() addsub = oneOf("+ -") numericExpr << term + Optional(addsub + numericExpr) arithop = oneOf("+ - * /") columnNumericExpr = Forward() cTerm = valueExprPrimary testme = valueExprPrimary + arithop + valueExprPrimary columnNumericExpr << cTerm + Optional(arithop + columnNumericExpr) colNumExpList = Group(columnNumericExpr + ZeroOrMore("," + columnNumericExpr)) valueExpr = numericExpr ## | stringExpr | dateExpr | intervalExpr derivedColumn = valueExpr + Optional(asToken + alias) selectSubList = derivedColumn + ZeroOrMore("," + derivedColumn) tableName = delimitedList(ident, ".", combine=True) # don't upcase table names anymore # tableName.setParseAction(upcaseTokens) self.tableAction = [] tableName.addParseAction(self.actionWrapper(self.tableAction)) tableName.setResultsName("table") tableAlias = tableName + asToken + ident.setResultsName("aliasName") tableAlias.setResultsName("alias") genericTableName = tableAlias | tableName genericTableName = genericTableName.setResultsName("tablename") tableNameList = Group(genericTableName + ZeroOrMore("," + genericTableName)) whereExpression = Forward() and_ = Keyword("and", caseless=True) or_ = Keyword("or", caseless=True) in_ = Keyword("in", caseless=True) between_ = Keyword("between", caseless=True) E = CaselessLiteral("E") binop = oneOf("= != < > >= <= eq ne lt le gt ge", caseless=True) arithSign = Word("+-", exact=1) realNum = Combine( Optional(arithSign) + (Word(nums) + "." + Optional(Word(nums)) | ("." + Word(nums))) + Optional(E + Optional(arithSign) + Word(nums))) intNum = Combine( Optional(arithSign) + Word(nums) + Optional(E + Optional("+") + Word(nums))) # need to add support for alg expressions columnRval = realNum | intNum | quotedString | columnNumericExpr # | numericExpr columnRvalList << Group(columnRval + ZeroOrMore("," + columnRval)) self.whereExpAction = [] namedRv = columnRval.setResultsName("column") whereConditionFlat = Group((functionSpec + binop + columnRval) | (namedRv + binop + columnRval) | (namedRv + in_ + "(" + columnRval + ZeroOrMore("," + namedRv) + ")") | (namedRv + in_ + "(" + selectStmt + ")") | (namedRv + between_ + namedRv + and_ + namedRv)) whereConditionFlat.addParseAction( self.actionWrapper(self.whereExpAction)) whereCondition = Group(whereConditionFlat | ("(" + whereExpression + ")")) # Test code to try to make an expression parse. # print whereConditionFlat.parseString("ABS(o1.ra - o2.ra) < 0.00083 / COS(RADIANS(o2.decl))") # goodFunction = ident + Literal('(') + columnNumericExpr + Literal(')') # print "ADFDSFDSF",testme.parseString("o1.ra - o2.ra", parseAll=True) # print "ADSFDSFAD", goodFunction.parseString("ABS(o1.ra - o2.ra)") #whereExpression << whereCondition.setResultsName("wherecond") #+ ZeroOrMore( ( and_ | or_ ) + whereExpression ) def scAnd(tok): print "scAnd", tok if "TRUE" == tok[0][0]: tok = tok[2] elif "TRUE" == tok[2][0]: tok = tok[0] return tok def scOr(tok): print "scOr", tok if ("TRUE" == tok[0][0]) or ("TRUE" == tok[2][0]): tok = [["TRUE"]] return tok def scWhere(tok): newtok = [] i = 0 while i < len(tok): if str(tok[i]) in ["TRUE", str(["TRUE"]) ] and (i + 1) < len(tok): if str(tok[i + 1]).upper() == "AND": i += 2 continue elif str(tok[i + i]).upper() == "OR": break newtok.append(tok[i]) i += 1 return newtok def collapseWhere(tok): #collapse.append(tok[0][1]) if ["TRUE"] == tok.asList()[0][1]: tok = [] return tok andExpr = and_ + whereExpression orExpr = or_ + whereExpression whereExpression << whereCondition + ZeroOrMore(andExpr | orExpr) whereExpression.addParseAction(scWhere) self.selectPart = selectToken + ( '*' | selectSubList).setResultsName("columns") whereClause = Group(whereToken + whereExpression).setResultsName("where") whereClause.addParseAction(collapseWhere) self.fromPart = fromToken + tableNameList.setResultsName("tables") # define the grammar selectStmt << (self.selectPart + fromToken + tableNameList.setResultsName("tables") + whereClause) self.simpleSQL = selectStmt + semicolon # define Oracle comment format, and ignore them oracleSqlComment = "--" + restOfLine self.simpleSQL.ignore(oracleSqlComment)