def Syntax(): delimitedList = (lambda x: x + ZeroOrMore(Suppress(",") + x) + Optional(Suppress(","))) dbl_quoted = Suppress('"') + Optional(CharsNotIn('"')) + Suppress('"') lelem = Word(alphanums + "-_") relem = Literal("true") | Literal("false") | Word(nums + ".") | dbl_quoted dict_ = Forward() assignment = lelem + Suppress("=") + (relem | (dict_)) dict_ << Suppress("{") + Group( Dict(delimitedList(Group(assignment))) | (dbl_quoted + Suppress(",") + Word(nums + ".")) | delimitedList(dict_)) + Suppress("}") return Dict(delimitedList(Group(assignment)))
def _define_valued_characters_section(self, heading, characters, parse_action, character_type): """Returns a parser object for a section specifying characters and their valued features. :param heading: section heading :type heading: `str` :param characters: valid characters :type characters: `list` of `str` :param parse_action: parse action for a character :type parse_action: `function` :param character_type: type of characters being described :type character_type: `str` """ heading = Literal('[{}]'.format(heading)) character = Word(''.join(characters), exact=1).setResultsName( 'character') character.setParseAction(self._handle_character) feature = Word(alphas).setResultsName('feature') feature.setParseAction(parse_action) value = Literal('+') ^ Literal('-') ^ Literal('\N{MINUS SIGN}') value.setParseAction(self._handle_feature_value) feature_value = Group(value + feature) feature_values = Group(delimitedList(feature_value)) character_definition = Dict(Group(character + Suppress(':') + feature_values)) character_definitions = Group(OneOrMore(character_definition)).setResultsName(character_type) section = Suppress(heading) + character_definitions return section
def parseText(metadata): try: header = ("# BEGIN DMI" "version = " + OneOrMore(Word(nums + ",.").setResultsName("DMIVer")) + Optional("width = " + Word(nums).setResultsName("width") + "height = " + Word(nums).setResultsName("height"))) states = ( Suppress("state = ") + QuotedString("\"").setResultsName("name") + "dirs = " + Word(nums).setResultsName("dirs") + "frames = " + Word(nums).setResultsName("frames") + Optional("delay = " + OneOrMore(Word(nums + ",.")).setResultsName("delay")) + Optional("loop =" + Word(nums)) + Optional("rewind =" + Word(nums))) full = header + Dict(Optional(OneOrMore( Group(states)))).setResultsName("states") clean = re.sub(r"(\n)|(\t)", "", metadata) print(clean) return full.parseString(clean).asDict() except ParseException as e: e = e raise Exception("Unable to parse DMI metadata")
def parse_all_configs(configs): regs = {} # set up the bnf to be used for each file lbrack = Literal("[").suppress() rbrack = Literal("]").suppress() colon = Literal(":").suppress() semi = Literal(";") comment = semi + Optional(restOfLine) nonrbrack = "".join([c for c in printables if c != "]"]) + " \t" noncolon = "".join([c for c in printables if c != ":"]) + " \t" sectionDef = lbrack + Word(nonrbrack) + rbrack keyDef = ~lbrack + Word(noncolon) + colon + restOfLine bnf = Dict(ZeroOrMore(Group(sectionDef + ZeroOrMore(Group(keyDef))))) bnf.ignore(comment) # bundle into a single var that can be passed to os.path.walk conf_data = (bnf, regs) for conf in configs: if not os.path.exists(conf): continue if os.path.isdir(conf): for dirpath, dirnames, filenames in os.walk(conf): parse_config_dir(conf_data, dirpath, filenames) else: parse_config(bnf, regs, conf) return regs
def jail_parser(self): word = Word(alphanums) token = Word(alphanums + '-_.:') path = Word(alphanums + '-_.:/') end = Literal(';').suppress() jail_name = delimitedList(Word(alphanums + '-.*'), delim='.', combine=True) boolean_parameter = token + end value = dblQuotedString | sglQuotedString | delimitedList(token) | path parameter = token + (Literal('=') | Literal('+=')) + value + end variable = Combine('$' + word) | Combine('$' + '{' + token + '}') variable_definition = variable + '=' + (dblQuotedString | sglQuotedString | path) + end jail_def = (jail_name + Literal('{') + ZeroOrMore( Group(boolean_parameter | parameter | variable_definition)) + Literal('}')) jail = Dict(ZeroOrMore(Group(jail_def | boolean_parameter | parameter))) jail.ignore(cppStyleComment) jail.ignore(pythonStyleComment) return jail
def parse(self, data=""): if data: self.raw = data SPACE = White().suppress() HOST = Literal("Host").suppress() KEY = Word(alphanums + "~*._-/") VALUE = Word(alphanums + "~*._-/") paramValueDef = SkipTo("#" | lineEnd) indentStack = [1] HostDecl = HOST + SPACE + VALUE paramDef = Dict(Group(KEY + SPACE + paramValueDef)) block = indentedBlock(paramDef, indentStack) HostBlock = Dict(Group(HostDecl + block)) try: return OneOrMore(HostBlock).ignore(pythonStyleComment).parseString(self.raw) except ParseException as e: return None
def _grammar(self): ident = Word(alphanums + ".") semi = Literal(";").suppress() # lrb = Literal("(").suppress() # rrb = Literal(")").suppress() lcb = Literal("{").suppress() rcb = Literal("}").suppress() Value = SkipTo(semi) KeyValue = Dict(Group(ident + Value + semi)) Dictionary = Forward() Block = lcb + ZeroOrMore(Dictionary | KeyValue) + rcb Dictionary << Dict(Group(ident + Block)) ParameterFile = ZeroOrMore(Dictionary | KeyValue) ParameterFile.ignore(cStyleComment) ParameterFile.ignore(cppStyleComment) return ParameterFile
def instance(): lit_e = CaselessLiteral('E') plusorminus = Literal('+') | Literal('-') number = Word(nums) integer = Combine(Optional(plusorminus) + number).setParseAction(lambda t: int(t[0])) index = integer.copy().addParseAction(index_check(0)) floatnumber = Combine(integer + Optional(Literal('.') + Optional(number)) + Optional(lit_e + integer)).setParseAction( lambda t: float(t[0])) #comment = Suppress("%") + Word(alphanums + " ") comment = Regex(r"%.*").setName("comment").suppress() linend = Or([comment, LineEnd()]).suppress() section_end = (Literal('#') + LineEnd()).suppress() vertex = (Group( OneOrMore(floatnumber('point') + OneOrMore(White()).suppress())) + linend)('vertex') vertex_header = (Keyword('VERTEX') + linend).suppress() vertex_section = (vertex_header + Group(OneOrMore(vertex))('vertices') + section_end) simplex = ( Group(OneOrMore(index('index') + OneOrMore(White()).suppress())) + linend)('simplex') simplex_header = (Keyword('SIMPLEX') + linend).suppress() simplex_section = (simplex_header + Group(OneOrMore(simplex))('simplices') + section_end) boundarysegment = (Group( index('id') + OneOrMore(index('index') + OneOrMore(White()).suppress())) + linend)('boundarysegment') boundarysegment_header = (Keyword('BOUNDARYSEGMENTS') + linend).suppress() boundarysegment_section = ( boundarysegment_header + Dict(OneOrMore(boundarysegment))('boundarysegments') + section_end) sections = Each([vertex_section, simplex_section, boundarysegment_section]) dgf_header = (Keyword('DGF') + linend).suppress() dgf = (dgf_header + Dict(sections) + OneOrMore(section_end))('dgf') return dgf
def pyparse_gml(): """A pyparsing tokenizer for GML graph format. This is not indented to be called directly. See Also -------- write_gml, read_gml, parse_gml Notes ----- This doesn't implement the complete GML specification for nested attributes for graphs, edges, and nodes. """ global graph try: from pyparsing import \ Literal, CaselessLiteral,Word,\ ZeroOrMore, Group, Dict, Optional, Combine,\ ParseException, restOfLine, White, alphanums, nums,\ OneOrMore,quotedString,removeQuotes,dblQuotedString except ImportError: raise ImportError, \ "Import Error: not able to import pyparsing: http://pyparsing.wikispaces.com/" if not graph: creator = Literal("Creator") + Optional(restOfLine) graphkey = Literal("graph").suppress() lbrack = Literal("[").suppress() rbrack = Literal("]").suppress() pound = ("#") comment = pound + Optional(restOfLine) white = White(" \t\n") point = Literal(".") e = CaselessLiteral("E") integer = Word(nums).setParseAction(lambda s, l, t: [int(t[0])]) real = Combine( Word("+-" + nums, nums) + Optional(point + Optional(Word(nums))) + Optional(e + Word("+-" + nums, nums))).setParseAction( lambda s, l, t: [float(t[0])]) key = Word(alphanums) value = integer ^ real ^ Word(alphanums) ^ quotedString.setParseAction( removeQuotes) keyvalue = Dict(Group(key+OneOrMore(white).suppress()\ +value+OneOrMore(white).suppress())) node = Group(Literal("node") + lbrack + OneOrMore(keyvalue) + rbrack) edge = Group(Literal("edge") + lbrack + OneOrMore(keyvalue) + rbrack) graph = Optional(creator)+\ graphkey + lbrack + ZeroOrMore(edge|node|keyvalue) + rbrack graph.ignore(comment) return graph
def pyparse_gml(): """A pyparsing tokenizer for GML graph format. This is not intended to be called directly. See Also -------- write_gml, read_gml, parse_gml """ try: from pyparsing import \ Literal, CaselessLiteral, Word, Forward,\ ZeroOrMore, Group, Dict, Optional, Combine,\ ParseException, restOfLine, White, alphas, alphanums, nums,\ OneOrMore,quotedString,removeQuotes,dblQuotedString, Regex except ImportError: try: from matplotlib.pyparsing import \ Literal, CaselessLiteral, Word, Forward,\ ZeroOrMore, Group, Dict, Optional, Combine,\ ParseException, restOfLine, White, alphas, alphanums, nums,\ OneOrMore,quotedString,removeQuotes,dblQuotedString, Regex except: raise ImportError('pyparsing not found', 'http://pyparsing.wikispaces.com/') lbrack = Literal("[").suppress() rbrack = Literal("]").suppress() pound = ("#") comment = pound + Optional(restOfLine) integer = Word(nums + '-').setParseAction(lambda s, l, t: [int(t[0])]) real = Regex(r"[+-]?\d+\.\d*([eE][+-]?\d+)?").setParseAction( lambda s, l, t: [float(t[0])]) dblQuotedString.setParseAction(removeQuotes) key = Word(alphas, alphanums + '_') value_atom = (real | integer | Word(alphanums) | dblQuotedString) value = Forward() # to be defined later with << operator keyvalue = Group(key + value) value << (value_atom | Group(lbrack + ZeroOrMore(keyvalue) + rbrack)) node = Group( Literal("node") + lbrack + Group(OneOrMore(keyvalue)) + rbrack) edge = Group( Literal("edge") + lbrack + Group(OneOrMore(keyvalue)) + rbrack) creator = Group(Literal("Creator") + Optional(restOfLine)) version = Group(Literal("Version") + Optional(restOfLine)) graphkey = Literal("graph").suppress() graph = Dict (Optional(creator)+Optional(version)+\ graphkey + lbrack + ZeroOrMore( (node|edge|keyvalue) ) + rbrack ) graph.ignore(comment) return graph
def __init__(self, debug=False): # Parser definition for headers header = Group( Word(alphas) + Literal(":").suppress() + Optional(CharsNotIn("\r\n"))) headers = Dict(OneOrMore(header)).setResultsName("header") # Parser definition for OFX body aggregate = Forward().setResultsName("OFX") aggregate_open_tag, aggregate_close_tag = self._tag() content_open_tag = self._tag(closed=False) content = Group(content_open_tag + CharsNotIn("<\r\n")) aggregate << Group(aggregate_open_tag \ + Dict(ZeroOrMore(aggregate | content)) \ + aggregate_close_tag) body = Group(aggregate).setResultsName("body") # The parser as a whole self.parser = headers + body if (debug): self.parser.setDebugActions(_ofxStartDebugAction, _ofxSuccessDebugAction, _ofxExceptionDebugAction)
def __init__(self, debug=False): aggregate = Forward().setResultsName("OFC") aggregate_open_tag, aggregate_close_tag = self._tag() content_open_tag = self._tag(closed=False) content = Group(content_open_tag + CharsNotIn("<\r\n")) aggregate << Group(aggregate_open_tag \ + Dict(OneOrMore(aggregate | content)) \ + aggregate_close_tag) self.parser = Group(aggregate).setResultsName("document") if (debug): self.parser.setDebugActions(ofxtools._ofxtoolsStartDebugAction, ofxtools._ofxtoolsSuccessDebugAction, ofxtools._ofxtoolsExceptionDebugAction)
def inifile_BNF(): global inibnf if not inibnf: # punctuation lbrack = Literal("[").suppress() rbrack = Literal("]").suppress() equals = Literal("=").suppress() semi = Literal(";") comment = semi + Optional( restOfLine ) nonrbrack = "".join( [ c for c in printables if c != "]" ] ) + " \t" nonequals = "".join( [ c for c in printables if c != "=" ] ) + " \t" sectionDef = lbrack + Word( nonrbrack ) + rbrack keyDef = ~lbrack + Word( nonequals ) + equals + restOfLine # using Dict will allow retrieval of named data fields as attributes of the parsed results inibnf = Dict( ZeroOrMore( Group( sectionDef + Dict( ZeroOrMore( Group( keyDef ) ) ) ) ) ) inibnf.ignore( comment ) return inibnf
def __defineDictGrammar(self): """Function defines the grammar for parsing a string(mainly) into: 1. Value: Value could be any one of the following 1. Simple types such as: a. numbers: all are floating point b. boolean: [true,false], [yes, no] c. Strings within double quotes d. alphanumerics 2. Dictionary 3. List 2. Dictionary: Set of key value pairs. ':' delimits values from keys. ',' delimites different pairs. '{}' delimits a dictionary. 3. List: Ordered list of values delimited by ',' pyparsing parse actions are used to convert the tokens into pyton native datatype such 'float' for floating point, 'dict' for dictionary and 'list' for list. The parser supports arbitrary nesting of the above tokens. Both the nesting and datastructure type integrity is preserved in the resulting python representation. Application: One of the main use of the grammar is to scrap web pages and extract a combination of JSON and javascript-like HTML attributes into python data structures. Simpler use cases include extracting supported simple data types from say, HTML tables. """ dictDefn = Forward() listDefn = Forward() key = (QuotedString('"') | Word(alphas)) + FollowedBy(Literal(":")) key.setName("key") self.value = MatchFirst([ self.unknown, self.floatNumber, self.boolean, QuotedString('"'), Word(alphanums), dictDefn, listDefn ]) self.value.setName("value") # dict_element = Group(key + self.KDELIM + self.value) dict_element = Group(key + self.KDELIM + self.value) + \ FollowedBy(Or([Literal(","), Literal("}")])) lde = Group(Dict(delimitedList(dict_element))) dictDefn << ((self.quoteit(lde, '{', '}')) | lde) self.dictDefn = dictDefn self.dictDefn.setName("Dictionary") listDefn << self.quoteit(Group(delimitedList(self.value)), '[', ']') self.listDefn = listDefn self.listDefn.setName("List") self.topElement = Or([self.dictDefn, self.listDefn, self.value]) self.parseTypes[WebParser.PSTYPE_DEFAULT] = self.topElement self.parseTypes[WebParser.PSTYPE_DICT] = self.dictDefn return
def fromPrm(fileAsString): """Parses a prm file to a nested python dict """ from pyparsing import Word, Group, Forward, Dict, ZeroOrMore, Optional, alphanums, Suppress # Grammar identifier = Word(alphanums + "_-") value = Word(alphanums + ".e-,<>_[] \n").setParseAction( lambda s, l, t: __convert_value(t[0])) key_value_pair = Group(identifier + Optional(value, default="") + Suppress(";")) block = Forward() block_content = Dict(ZeroOrMore(key_value_pair | block)) block << Group(identifier + Suppress("{") + block_content + Suppress("}")) return block_content.parseString(_remove_comments(fileAsString), parseAll=True).asDict()
def parse_block(self, block_text): """Parses sql block into tokens """ # Valid grammar looks like this: # {sqllinechart: title='Some string' | colors=green, yellow} # make a grammar block_start = Literal("{") sql_start = Keyword(self.TAGNAME, caseless=True) colon = Literal(":") sql_end = Literal("}") separator = Literal("|") block_end = Keyword("{" + self.TAGNAME + "}", caseless=True) # params field_name = Word(alphanums) equal_sign = Suppress(Literal("=")) # whatever value field_value = (CharsNotIn("|}")) # param name and value param_group = Group(field_name + equal_sign + field_value) # list of all params param_list = delimitedList(param_group, '|') # helper param_dict = Dict(param_list) # sql text sql_text = SkipTo(block_end) sqldecl = Forward() sqldecl << (block_start + sql_start + Optional(colon) + Optional(param_dict) + sql_end + sql_text.setResultsName('sqltext') + block_end) block_str = "".join(block_text) tokens = sqldecl.parseString(block_str) return tokens
def __init__(self): FALSE = Keyword("false") NULL = Keyword("null") TRUE = Keyword("true") FALSE.setParseAction(replaceWith(False)) NULL.setParseAction(replaceWith(None)) TRUE.setParseAction(replaceWith(True)) pattern = Forward() label = Word(alphas, alphanums + "_").setResultsName("layer_name") configurable_param = nestedExpr(content=pattern) arg = (NULL ^ FALSE ^ TRUE ^ pyparsing_common.number ^ (Word(alphanums + "*_") + ~Word("=")) ^ configurable_param) args = arg[...].setResultsName("args") args.setParseAction(self.convert_list) options = Dict(Group(Word(alphanums + "_") + Suppress("=") + arg))[...].setResultsName("options") options.setParseAction(self.convert_dict) pattern <<= label + args + options pattern.setParseAction(Pattern) self.pattern = pattern
def PartitionParser(): start_ = Suppress('start') + Suppress('=') + Word(nums) size_ = Suppress('size') + Suppress('=') + Word(nums) id_ = Suppress('Id') + Suppress('=') + Word(nums) device_ = Word(alphas + nums + '/') comment_ = '#' + Optional(restOfLine) warning_ = 'Warning:' + Optional(restOfLine) unit_ = Literal('unit') + Optional( Suppress(':') + Word(alphas + nums) + restOfLine) pinfo = start_ + Suppress(',') pinfo += size_ + Suppress(',') pinfo += id_ + restOfLine partition = Group(device_ + Suppress(':') + pinfo) partition.ignore(comment_) partition.ignore(warning_) partition.ignore(unit_) #partition = ZeroOrMore(partition) return Dict(ZeroOrMore(partition))
def create_grammar(): TRUE = Keyword('true') FALSE = Keyword('false') NULL = Keyword('null') LBRACK, RBRACK, LBRACE, RBRACE, COLON = map(Suppress, '[]{}:') string = dblQuotedString() number = pyparsing_common.number() object_ = Forward() value = Forward() elements = delimitedList(value) array = Group(LBRACK + Optional(elements, []) + RBRACK) value <<= (string | number | Group(object_) | array | TRUE | FALSE | NULL) member = Group(string + COLON + value) members = delimitedList(member) object_ <<= Dict(LBRACE + Optional(members) + RBRACE) return value
def parse_block(self, block_text): # make a grammar block_start = Literal("{") sql_start = Keyword("sqltable", caseless=True) colon = Literal(":") sql_end = Literal("}") separator = Literal("|") block_end = Keyword("{sqltable}", caseless=True) # params field_name = Word(alphanums) equal_sign = Suppress(Literal("=")) # whatever value field_value = (CharsNotIn("|}")) # param name and value param_group = Group(field_name + equal_sign + field_value) # list of all params param_list = delimitedList(param_group, '|') # helper param_dict = Dict(param_list) # sql text sql_text = SkipTo(block_end) sqldecl = Forward() sqldecl << (block_start + sql_start + Optional(colon) + Optional(param_dict) + sql_end + sql_text.setResultsName('sqltext') + block_end) block_str = "".join(block_text) tokens = sqldecl.parseString( block_str ) return tokens
def _define_json(): # https://pyparsing.wikispaces.com/file/view/jsonParser.py TRUE = _make_keyword('true', True) FALSE = _make_keyword('false', False) NULL = _make_keyword('null', None) LBRACK, RBRACK, LBRACE, RBRACE, COLON = map(Suppress, '[]{}:') jsonString = dblQuotedString().setParseAction(removeQuotes) jsonNumber = pyparsing_common.number() jsonObject = Forward() jsonValue = Forward() jsonElements = delimitedList(jsonValue) jsonArray = Group(LBRACK + Optional(jsonElements, []) + RBRACK) jsonValue << (jsonString | jsonNumber | Group(jsonObject) | jsonArray | TRUE | FALSE | NULL) # noqa memberDef = Group(jsonString + COLON + jsonValue) jsonMembers = delimitedList(memberDef) jsonObject << Dict(LBRACE + Optional(jsonMembers) + RBRACE) return jsonValue
def _build(): """Encapsulate so the variables don't leak out.""" # Basic punctuation colon = Literal(':').suppress() hashmark = Literal('#').suppress() comment = (hashmark + restOfLine).suppress() # Enforce Python-style naming conventions command_name = Word(srange("[A-Z]"), srange("[a-zA-Z0-9]")) # StudlyCaps field_name = Word(srange("[a-z_]"), srange("[a-z0-9_]")) # lower_underscore # Put it all together fields = Dict(OneOrMore(Group(field_name + colon + restOfLine))) fields = nestedExpr(opener="{", closer="}", content=fields) command = Group(command_name + fields) # Configure the parser tml_parser = OneOrMore(command) + stringEnd tml_parser.ignore(comment) return tml_parser
def _parse_rec_string(self): ''' Pyparsing parser to the received string ''' #any word word = Word(printables) #recognized tokens from_t = CaselessLiteral(self._FROM) by_t = CaselessLiteral(self._BY) with_t = CaselessLiteral(self._WITH) id_t = CaselessLiteral(self._ID) via_t = CaselessLiteral(self._VIA) for_t = CaselessLiteral(self._FOR) #A group of non tokens phrase = OneOrMore(~from_t + ~by_t + ~with_t + ~id_t + ~for_t + ~via_t + word) #Group phrase with token from_g = Optional(Group(from_t + phrase)) by_g = Optional(Group(by_t + phrase)) with_g = Optional(Group(with_t + phrase)) id_g = Optional(Group(id_t + phrase)) via_g = Optional(Group(via_t + phrase)) for_g = Optional(Group(for_t + phrase)) grouped_data = from_g & by_g & with_g & id_g & via_g & for_g parse_to_dict = Dict(grouped_data) try: rec_dict = parse_to_dict.parseString(self._rec) except ParseException: rec_dict = {} return rec_dict
'next', 'option', 'reserved', 'rewind', 'starts', 'tstp', 'tsfp', 'uid', 'vendor-class-identifier', ] other_statement = oneOf(other_statement_names, caseless=True) + args lease_statement = (hardware | deleted | ends | set_statement | lone_statement | other_statement) + Suppress(';') lease_parser = (lease_or_host("lease_or_host") + ip("ip") + Suppress('{') + Dict(ZeroOrMore(Group(lease_statement))) + Suppress('}')) lease_parser.ignore('#' + restOfLine) def is_lease(entry): """Is `entry` a lease declaration?""" entry_type = entry.lease_or_host.lower() assert entry_type in { 'host', 'lease' }, ("Unknown entry type (not a host or lease): %s" % entry_type) return entry_type == 'lease' def is_host(entry): """Is `entry` a host declaration?""" return not is_lease(entry)
messageDefn = MESSAGE_ - ident("messageId") + LBRACE + messageBody( "body") + RBRACE typespec = oneOf("""double float int32 int64 uint32 uint64 sint32 sint64 fixed32 fixed64 sfixed32 sfixed64 bool string bytes""" ) | ident rvalue = integer | TRUE_ | FALSE_ | ident fieldDirective = LBRACK + Group(ident + EQ + rvalue) + RBRACK fieldDefnPrefix = REQUIRED_ | OPTIONAL_ | REPEATED_ fieldDefn = (Optional(fieldDefnPrefix) + typespec("typespec") + ident("ident") + EQ + integer("fieldint") + ZeroOrMore(fieldDirective) + SEMI) # enumDefn ::= 'enum' ident '{' { ident '=' integer ';' }* '}' enumDefn = ENUM_("typespec") - ident('name') + LBRACE + Dict( ZeroOrMore(Group(ident + EQ + integer + SEMI)))('values') + RBRACE # extensionsDefn ::= 'extensions' integer 'to' integer ';' extensionsDefn = EXTENSIONS_ - integer + TO_ + integer + SEMI # messageExtension ::= 'extend' ident '{' messageBody '}' messageExtension = EXTEND_ - ident + LBRACE + messageBody + RBRACE # messageBody ::= { fieldDefn | enumDefn | messageDefn | extensionsDefn | messageExtension }* messageBody << Group( ZeroOrMore( Group(fieldDefn | enumDefn | messageDefn | extensionsDefn | messageExtension))) # methodDefn ::= 'rpc' ident '(' [ ident ] ')' 'returns' '(' [ ident ] ')' ';' methodDefn = (RPC_ - ident("methodName") + LPAR +
sattrs = ";".join(['%s:%s' % (k, self.attrs[k]) for k in sorted(self.attrs)]) return 'attr(%s)' % sattrs def __repr__(self): rattrs = ",".join(['%r:%r' % (k, self.attrs[k]) for k in sorted(self.attrs)]) return 'Attr({%s})' % rattrs @staticmethod def parse_action(s, loc, tokens): where = W(s, loc) attrs = tokens.get('attrs', None) attrs = {k: v for k, v in attrs.items()} return Attr(attrs, where=where) attr_spec = Dict( delimitedList( Group(Word(alphanums) + Suppress(Literal(':')) + contract_expression('value')), delim=';') )('attrs') attrs_spec = ('(' - attr_spec - ')') attr_contract = Keyword('attr') - attrs_spec attr_contract.setParseAction(Attr.parse_action) add_contract(attr_contract) add_keyword('attr')
VARCHAR = CaselessKeyword("VARCHAR") WHERE = CaselessKeyword("WHERE") query = Forward() whereExpression = Forward() ident = Word(alphanums + "_$").setName("identifier") data_type = Group(VARCHAR + "(" + Word(nums) + ")") | INT | TEXT | DOUBLE | BOOLEAN column_name = (delimitedList(ident, ".", combine=True)("column_name")) column_name_list = Group(delimitedList(column_name)) primary_key = PRIMARY + KEY + "(" + column_name_list("key_columns") + ")" column_definition = Group(ident("def_column_name") + data_type("data_type")) | Group( primary_key("primary_key")) column_definition_list = Dict(delimitedList(column_definition)) table_name = delimitedList(ident, ".", combine=True) table_names = Group(delimitedList(table_name)) E = CaselessLiteral("E") binop = oneOf("= != < > >= <= eq ne lt le gt ge", caseless=True) arithSign = Word("+-", exact=1) realNum = Combine( Optional(arithSign) + (Word(nums) + "." + Optional(Word(nums)) | ("." + Word(nums))) + Optional(E + Optional(arithSign) + Word(nums))) intNum = Combine( Optional(arithSign) + Word(nums) + Optional(E + Optional("+") + Word(nums))) columnRval = realNum | intNum | quotedString | column_name # need to add support for alg expressions
# The name types with their digiteyness not_dig_lower = not_digname.copy().setParseAction(lambda t: t[0].lower()) macro_def = not_dig_lower.copy() macro_ref = not_dig_lower.copy().setParseAction(lambda t: Macro(t[0].lower())) field_name = not_dig_lower.copy() # Spaces in names mean they cannot clash with field names entry_type = not_dig_lower("entry_type") cite_key = any_name("cite_key") # Number has to be before macro name string = number | macro_ref | quoted_string | curly_string # There can be hash concatenation field_value = string + ZeroOrMore(HASH + string) field_def = Group(field_name + EQUALS + field_value) entry_contents = Dict(ZeroOrMore(field_def + COMMA) + Optional(field_def)) # Entry is surrounded either by parentheses or curlies entry = AT + entry_type + bracketed(cite_key + COMMA + entry_contents) # Preamble is a macro-like thing with no name preamble = AT + CaselessLiteral("preamble") + bracketed(field_value) # Macros (aka strings) macro_contents = macro_def + EQUALS + field_value macro = AT + CaselessLiteral("string") + bracketed(macro_contents) # Implicit comments icomment = SkipTo("@").setParseAction(lambda t: t.insert(0, "icomment")) # entries are last in the list (other than the fallback) because they have
from pyparsing import alphas, alphanums, delimitedList, Dict, Forward, \ Group, Literal, Optional, Suppress, Word bracket_open = Suppress(Literal('{')) bracket_close = Suppress(Literal('}')) identifier = Word(alphas, alphanums + "_").setName("identifier") field = Forward() fields = delimitedList(field) obj = Dict(bracket_open + fields + bracket_close) field << Group(identifier + Optional(obj, default=None)) ql = obj class EveryDict(object): def __contains__(self, name): return True def __getitem__(self, name): return self every_dict = EveryDict() def to_dict(d): return {k: to_dict(v) if v != None else every_dict for k, v in d.items()} def parse(s): if not s:
WS = Optional(White(' \t')).suppress() EQ = WS + Suppress('=') + WS FRT = WS + Suppress('->') + WS HSTART = Suppress('[') HEND = Suppress(']') + WS comment = Literal('#') + restOfLine sec_directories = HSTART + CaselessLiteral("directories") + HEND sec_files = HSTART + CaselessLiteral("files") + HEND sec_settings = HSTART + CaselessLiteral("settings") + HEND section = HSTART + Word(printables + " ") + HEND keyvalue = ~HSTART + Dict(Group(Word(alphanums) + EQ + Word(printables))) fromto = Group(~HSTART + SkipTo(FRT).setResultsName("source").setParseAction( lambda x: x[0]) # whyyy + FRT + restOfLine.setResultsName("destination")) part_directories = Dict( Group(sec_directories + Group(ZeroOrMore(fromto)))).setName("part_directories") part_files = Dict(Group(sec_files + Group(ZeroOrMore(fromto)))).setName("part_files") part_settings = Dict(Group(sec_settings + OneOrMore(keyvalue))).setName("part_settings") g = Optional(part_directories) & Optional(part_files) & part_settings g.ignore(comment)