def parse_config(content):
"""
Parse the Bazel project BUILD file content.
:param str content: The Bazel BUILD file content.
:returns: Dictionary of config.
:rtype: dict
"""
quoted = QuotedString(quoteChar='"') | QuotedString(quoteChar='\'')
item_name = pyparsing_common.identifier.setName('id')
item_value = (
Group( # Array values.
Literal('[').suppress() + delimitedList(quoted) +
Literal(']').suppress()) | Group( # Glob case.
Word('glob([').suppress() + delimitedList(quoted) +
Word('])').suppress()) | quoted).setName(
'value') # Quoted string value.
rule_item = Group(item_name + Literal('=').suppress() + item_value +
Optional(',').suppress())
rule_items = Dict(delimitedList(rule_item))
rule_values = nestedExpr(content=rule_items)
rule_taret = item_name
rule = Group(rule_taret + rule_values)
parser = Dict(OneOrMore(rule))
try:
config = parser.parseString(content)
except Exception as e:
logger.warning('No valid Build content')
return {}
return config.asDict()
def inifile_BNF():
global inibnf
if not inibnf:
# punctuation
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
equals = Literal("=").suppress()
semi = Literal(";")
comment = semi + Optional(restOfLine)
nonrbrack = "".join([c for c in printables if c != "]"]) + " \t"
nonequals = "".join([c for c in printables if c != "="]) + " \t"
sectionDef = lbrack + Word(nonrbrack) + rbrack
keyDef = ~lbrack + Word(nonequals) + equals + restOfLine
# using Dict will allow retrieval of named data fields as attributes of the parsed results
inibnf = Dict(
ZeroOrMore(Group(sectionDef + Dict(ZeroOrMore(Group(keyDef))))))
inibnf.ignore(comment)
return inibnf
def inifile_BNF():
global inibnf
if not inibnf:
# punctuation
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
equals = Literal("=").suppress()
semi = Literal(";")
comment = semi + Optional( restOfLine )
nonrbrack = "".join( [ c for c in printables if c != "]" ] ) + " \t"
nonequals = "".join( [ c for c in printables if c != "=" ] ) + " \t"
sectionDef = lbrack + Word( nonrbrack ) + rbrack
keyDef = ~lbrack + Word( nonequals ) + equals + empty + restOfLine
# strip any leading or trailing blanks from key
def stripKey(tokens):
tokens[0] = tokens[0].strip()
keyDef.setParseAction(stripKey)
# using Dict will allow retrieval of named data fields as attributes of the parsed results
inibnf = Dict( ZeroOrMore( Group( sectionDef + Dict( ZeroOrMore( Group( keyDef ) ) ) ) ) )
inibnf.ignore( comment )
return inibnf
def parse(self, data=""):
"""Parse ssh-config data
args:
data (str)
returns:
Parsed config or None (dict)
"""
if data:
self.raw = data
SPACE = White().suppress()
SEP = Suppress(SPACE) | Suppress("=")
HOST = CaselessLiteral("Host").suppress()
MATCH = CaselessLiteral("Match").suppress()
KEY = Word(alphanums)
VALUE = Word(alphanums + ' ~%*?!._-+/,"')
paramValueDef = SkipTo("#" | lineEnd)
indentStack = [1]
HostDecl = (HOST | MATCH) + SEP + VALUE
paramDef = Dict(Group(KEY + SEP + paramValueDef))
block = indentedBlock(paramDef, indentStack)
HostBlock = Dict(Group(HostDecl + block))
try:
return OneOrMore(HostBlock).ignore(pythonStyleComment).parseString(self.raw)
except ParseException as e:
print(e)
return None
def Syntax():
delimitedList = (lambda x: x + ZeroOrMore(Suppress(",") + x) +
Optional(Suppress(",")))
dbl_quoted = Suppress('"') + Optional(CharsNotIn('"')) + Suppress('"')
lelem = Word(alphanums + "-_")
relem = Literal("true") | Literal("false") | Word(nums +
".") | dbl_quoted
dict_ = Forward()
assignment = lelem + Suppress("=") + (relem | (dict_))
dict_ << Suppress("{") + Group(
Dict(delimitedList(Group(assignment)))
| (dbl_quoted + Suppress(",") + Word(nums + "."))
| delimitedList(dict_)) + Suppress("}")
return Dict(delimitedList(Group(assignment)))
def parse_all_configs(configs):
regs = {}
# set up the bnf to be used for each file
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
colon = Literal(":").suppress()
semi = Literal(";")
comment = semi + Optional(restOfLine)
nonrbrack = "".join([c for c in printables if c != "]"]) + " \t"
noncolon = "".join([c for c in printables if c != ":"]) + " \t"
sectionDef = lbrack + Word(nonrbrack) + rbrack
keyDef = ~lbrack + Word(noncolon) + colon + restOfLine
bnf = Dict(ZeroOrMore(Group(sectionDef + ZeroOrMore(Group(keyDef)))))
bnf.ignore(comment)
# bundle into a single var that can be passed to os.path.walk
conf_data = (bnf, regs)
for conf in configs:
if not os.path.exists(conf):
continue
if os.path.isdir(conf):
for dirpath, dirnames, filenames in os.walk(conf):
parse_config_dir(conf_data, dirpath, filenames)
else:
parse_config(bnf, regs, conf)
return regs
def jail_parser(self):
word = Word(alphanums)
token = Word(alphanums + '-_.:')
path = Word(alphanums + '-_.:/')
end = Literal(';').suppress()
jail_name = delimitedList(Word(alphanums + '-.*'),
delim='.',
combine=True)
boolean_parameter = token + end
value = dblQuotedString | sglQuotedString | delimitedList(token) | path
parameter = token + (Literal('=') | Literal('+=')) + value + end
variable = Combine('$' + word) | Combine('$' + '{' + token + '}')
variable_definition = variable + '=' + (dblQuotedString
| sglQuotedString | path) + end
jail_def = (jail_name + Literal('{') + ZeroOrMore(
Group(boolean_parameter | parameter | variable_definition)) +
Literal('}'))
jail = Dict(ZeroOrMore(Group(jail_def | boolean_parameter
| parameter)))
jail.ignore(cppStyleComment)
jail.ignore(pythonStyleComment)
return jail
def parseText(metadata):
try:
header = ("# BEGIN DMI"
"version = " +
OneOrMore(Word(nums + ",.").setResultsName("DMIVer")) +
Optional("width = " + Word(nums).setResultsName("width") +
"height = " + Word(nums).setResultsName("height")))
states = (
Suppress("state = ") + QuotedString("\"").setResultsName("name") +
"dirs = " + Word(nums).setResultsName("dirs") + "frames = " +
Word(nums).setResultsName("frames") +
Optional("delay = " +
OneOrMore(Word(nums + ",.")).setResultsName("delay")) +
Optional("loop =" + Word(nums)) +
Optional("rewind =" + Word(nums)))
full = header + Dict(Optional(OneOrMore(
Group(states)))).setResultsName("states")
clean = re.sub(r"(\n)|(\t)", "", metadata)
print(clean)
return full.parseString(clean).asDict()
except ParseException as e:
e = e
raise Exception("Unable to parse DMI metadata")
def _define_valued_characters_section(self, heading, characters,
parse_action, character_type):
"""Returns a parser object for a section specifying characters
and their valued features.
:param heading: section heading
:type heading: `str`
:param characters: valid characters
:type characters: `list` of `str`
:param parse_action: parse action for a character
:type parse_action: `function`
:param character_type: type of characters being described
:type character_type: `str`
"""
heading = Literal('[{}]'.format(heading))
character = Word(''.join(characters), exact=1).setResultsName(
'character')
character.setParseAction(self._handle_character)
feature = Word(alphas).setResultsName('feature')
feature.setParseAction(parse_action)
value = Literal('+') ^ Literal('-') ^ Literal('\N{MINUS SIGN}')
value.setParseAction(self._handle_feature_value)
feature_value = Group(value + feature)
feature_values = Group(delimitedList(feature_value))
character_definition = Dict(Group(character + Suppress(':') +
feature_values))
character_definitions = Group(OneOrMore(character_definition)).setResultsName(character_type)
section = Suppress(heading) + character_definitions
return section
def parse(self, data=""):
if data:
self.raw = data
SPACE = White().suppress()
HOST = Literal("Host").suppress()
KEY = Word(alphanums + "~*._-/")
VALUE = Word(alphanums + "~*._-/")
paramValueDef = SkipTo("#" | lineEnd)
indentStack = [1]
HostDecl = HOST + SPACE + VALUE
paramDef = Dict(Group(KEY + SPACE + paramValueDef))
block = indentedBlock(paramDef, indentStack)
HostBlock = Dict(Group(HostDecl + block))
try:
return OneOrMore(HostBlock).ignore(pythonStyleComment).parseString(self.raw)
except ParseException as e:
return None
def instance():
lit_e = CaselessLiteral('E')
plusorminus = Literal('+') | Literal('-')
number = Word(nums)
integer = Combine(Optional(plusorminus) +
number).setParseAction(lambda t: int(t[0]))
index = integer.copy().addParseAction(index_check(0))
floatnumber = Combine(integer + Optional(Literal('.') + Optional(number)) +
Optional(lit_e + integer)).setParseAction(
lambda t: float(t[0]))
#comment = Suppress("%") + Word(alphanums + " ")
comment = Regex(r"%.*").setName("comment").suppress()
linend = Or([comment, LineEnd()]).suppress()
section_end = (Literal('#') + LineEnd()).suppress()
vertex = (Group(
OneOrMore(floatnumber('point') + OneOrMore(White()).suppress())) +
linend)('vertex')
vertex_header = (Keyword('VERTEX') + linend).suppress()
vertex_section = (vertex_header + Group(OneOrMore(vertex))('vertices') +
section_end)
simplex = (
Group(OneOrMore(index('index') + OneOrMore(White()).suppress())) +
linend)('simplex')
simplex_header = (Keyword('SIMPLEX') + linend).suppress()
simplex_section = (simplex_header +
Group(OneOrMore(simplex))('simplices') + section_end)
boundarysegment = (Group(
index('id') +
OneOrMore(index('index') + OneOrMore(White()).suppress())) +
linend)('boundarysegment')
boundarysegment_header = (Keyword('BOUNDARYSEGMENTS') + linend).suppress()
boundarysegment_section = (
boundarysegment_header +
Dict(OneOrMore(boundarysegment))('boundarysegments') + section_end)
sections = Each([vertex_section, simplex_section, boundarysegment_section])
dgf_header = (Keyword('DGF') + linend).suppress()
dgf = (dgf_header + Dict(sections) + OneOrMore(section_end))('dgf')
return dgf
def _grammar(self):
ident = Word(alphanums + ".")
semi = Literal(";").suppress()
# lrb = Literal("(").suppress()
# rrb = Literal(")").suppress()
lcb = Literal("{").suppress()
rcb = Literal("}").suppress()
Value = SkipTo(semi)
KeyValue = Dict(Group(ident + Value + semi))
Dictionary = Forward()
Block = lcb + ZeroOrMore(Dictionary | KeyValue) + rcb
Dictionary << Dict(Group(ident + Block))
ParameterFile = ZeroOrMore(Dictionary | KeyValue)
ParameterFile.ignore(cStyleComment)
ParameterFile.ignore(cppStyleComment)
return ParameterFile
def pyparse_gml():
"""A pyparsing tokenizer for GML graph format.
This is not indented to be called directly.
See Also
--------
write_gml, read_gml, parse_gml
Notes
-----
This doesn't implement the complete GML specification for
nested attributes for graphs, edges, and nodes.
"""
global graph
try:
from pyparsing import \
Literal, CaselessLiteral,Word,\
ZeroOrMore, Group, Dict, Optional, Combine,\
ParseException, restOfLine, White, alphanums, nums,\
OneOrMore,quotedString,removeQuotes,dblQuotedString
except ImportError:
raise ImportError, \
"Import Error: not able to import pyparsing: http://pyparsing.wikispaces.com/"
if not graph:
creator = Literal("Creator") + Optional(restOfLine)
graphkey = Literal("graph").suppress()
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
pound = ("#")
comment = pound + Optional(restOfLine)
white = White(" \t\n")
point = Literal(".")
e = CaselessLiteral("E")
integer = Word(nums).setParseAction(lambda s, l, t: [int(t[0])])
real = Combine(
Word("+-" + nums, nums) + Optional(point + Optional(Word(nums))) +
Optional(e + Word("+-" + nums, nums))).setParseAction(
lambda s, l, t: [float(t[0])])
key = Word(alphanums)
value = integer ^ real ^ Word(alphanums) ^ quotedString.setParseAction(
removeQuotes)
keyvalue = Dict(Group(key+OneOrMore(white).suppress()\
+value+OneOrMore(white).suppress()))
node = Group(Literal("node") + lbrack + OneOrMore(keyvalue) + rbrack)
edge = Group(Literal("edge") + lbrack + OneOrMore(keyvalue) + rbrack)
graph = Optional(creator)+\
graphkey + lbrack + ZeroOrMore(edge|node|keyvalue) + rbrack
graph.ignore(comment)
return graph
def pyparse_gml():
"""A pyparsing tokenizer for GML graph format.
This is not intended to be called directly.
See Also
--------
write_gml, read_gml, parse_gml
"""
try:
from pyparsing import \
Literal, CaselessLiteral, Word, Forward,\
ZeroOrMore, Group, Dict, Optional, Combine,\
ParseException, restOfLine, White, alphas, alphanums, nums,\
OneOrMore,quotedString,removeQuotes,dblQuotedString, Regex
except ImportError:
try:
from matplotlib.pyparsing import \
Literal, CaselessLiteral, Word, Forward,\
ZeroOrMore, Group, Dict, Optional, Combine,\
ParseException, restOfLine, White, alphas, alphanums, nums,\
OneOrMore,quotedString,removeQuotes,dblQuotedString, Regex
except:
raise ImportError('pyparsing not found',
'http://pyparsing.wikispaces.com/')
lbrack = Literal("[").suppress()
rbrack = Literal("]").suppress()
pound = ("#")
comment = pound + Optional(restOfLine)
integer = Word(nums + '-').setParseAction(lambda s, l, t: [int(t[0])])
real = Regex(r"[+-]?\d+\.\d*([eE][+-]?\d+)?").setParseAction(
lambda s, l, t: [float(t[0])])
dblQuotedString.setParseAction(removeQuotes)
key = Word(alphas, alphanums + '_')
value_atom = (real | integer | Word(alphanums) | dblQuotedString)
value = Forward() # to be defined later with << operator
keyvalue = Group(key + value)
value << (value_atom | Group(lbrack + ZeroOrMore(keyvalue) + rbrack))
node = Group(
Literal("node") + lbrack + Group(OneOrMore(keyvalue)) + rbrack)
edge = Group(
Literal("edge") + lbrack + Group(OneOrMore(keyvalue)) + rbrack)
creator = Group(Literal("Creator") + Optional(restOfLine))
version = Group(Literal("Version") + Optional(restOfLine))
graphkey = Literal("graph").suppress()
graph = Dict (Optional(creator)+Optional(version)+\
graphkey + lbrack + ZeroOrMore( (node|edge|keyvalue) ) + rbrack )
graph.ignore(comment)
return graph
def __init__(self, debug=False):
aggregate = Forward().setResultsName("OFC")
aggregate_open_tag, aggregate_close_tag = self._tag()
content_open_tag = self._tag(closed=False)
content = Group(content_open_tag + CharsNotIn("<\r\n"))
aggregate << Group(aggregate_open_tag \
+ Dict(OneOrMore(aggregate | content)) \
+ aggregate_close_tag)
self.parser = Group(aggregate).setResultsName("document")
if (debug):
self.parser.setDebugActions(ofxtools._ofxtoolsStartDebugAction,
ofxtools._ofxtoolsSuccessDebugAction,
ofxtools._ofxtoolsExceptionDebugAction)
def __init__(self, debug=False):
# Parser definition for headers
header = Group(
Word(alphas) + Literal(":").suppress() +
Optional(CharsNotIn("\r\n")))
headers = Dict(OneOrMore(header)).setResultsName("header")
# Parser definition for OFX body
aggregate = Forward().setResultsName("OFX")
aggregate_open_tag, aggregate_close_tag = self._tag()
content_open_tag = self._tag(closed=False)
content = Group(content_open_tag + CharsNotIn("<\r\n"))
aggregate << Group(aggregate_open_tag \
+ Dict(ZeroOrMore(aggregate | content)) \
+ aggregate_close_tag)
body = Group(aggregate).setResultsName("body")
# The parser as a whole
self.parser = headers + body
if (debug):
self.parser.setDebugActions(_ofxStartDebugAction,
_ofxSuccessDebugAction,
_ofxExceptionDebugAction)
def __defineDictGrammar(self):
"""Function defines the grammar for parsing a string(mainly) into:
1. Value: Value could be any one of the following
1. Simple types such as:
a. numbers: all are floating point
b. boolean: [true,false], [yes, no]
c. Strings within double quotes
d. alphanumerics
2. Dictionary
3. List
2. Dictionary: Set of key value pairs. ':' delimits values from keys.
',' delimites different pairs. '{}' delimits a dictionary.
3. List: Ordered list of values delimited by ','
pyparsing parse actions are used to convert the tokens into pyton native
datatype such 'float' for floating point, 'dict' for dictionary and
'list' for list. The parser supports arbitrary nesting of the above
tokens. Both the nesting and datastructure type integrity is preserved
in the resulting python representation.
Application:
One of the main use of the grammar is to scrap web pages and extract a
combination of JSON and javascript-like HTML attributes into python
data structures. Simpler use cases include extracting supported simple
data types from say, HTML tables.
"""
dictDefn = Forward()
listDefn = Forward()
key = (QuotedString('"') | Word(alphas)) + FollowedBy(Literal(":"))
key.setName("key")
self.value = MatchFirst([
self.unknown, self.floatNumber, self.boolean,
QuotedString('"'),
Word(alphanums), dictDefn, listDefn
])
self.value.setName("value")
# dict_element = Group(key + self.KDELIM + self.value)
dict_element = Group(key + self.KDELIM + self.value) + \
FollowedBy(Or([Literal(","), Literal("}")]))
lde = Group(Dict(delimitedList(dict_element)))
dictDefn << ((self.quoteit(lde, '{', '}')) | lde)
self.dictDefn = dictDefn
self.dictDefn.setName("Dictionary")
listDefn << self.quoteit(Group(delimitedList(self.value)), '[', ']')
self.listDefn = listDefn
self.listDefn.setName("List")
self.topElement = Or([self.dictDefn, self.listDefn, self.value])
self.parseTypes[WebParser.PSTYPE_DEFAULT] = self.topElement
self.parseTypes[WebParser.PSTYPE_DICT] = self.dictDefn
return
def fromPrm(fileAsString):
"""Parses a prm file to a nested python dict """
from pyparsing import Word, Group, Forward, Dict, ZeroOrMore, Optional, alphanums, Suppress
# Grammar
identifier = Word(alphanums + "_-")
value = Word(alphanums + ".e-,<>_[] \n").setParseAction(
lambda s, l, t: __convert_value(t[0]))
key_value_pair = Group(identifier + Optional(value, default="") +
Suppress(";"))
block = Forward()
block_content = Dict(ZeroOrMore(key_value_pair | block))
block << Group(identifier + Suppress("{") + block_content + Suppress("}"))
return block_content.parseString(_remove_comments(fileAsString),
parseAll=True).asDict()
def parse_block(self, block_text):
"""Parses sql block into tokens
"""
# Valid grammar looks like this:
# {sqllinechart: title='Some string' | colors=green, yellow}
# make a grammar
block_start = Literal("{")
sql_start = Keyword(self.TAGNAME, caseless=True)
colon = Literal(":")
sql_end = Literal("}")
separator = Literal("|")
block_end = Keyword("{" + self.TAGNAME + "}", caseless=True)
# params
field_name = Word(alphanums)
equal_sign = Suppress(Literal("="))
# whatever value
field_value = (CharsNotIn("|}"))
# param name and value
param_group = Group(field_name + equal_sign + field_value)
# list of all params
param_list = delimitedList(param_group, '|')
# helper
param_dict = Dict(param_list)
# sql text
sql_text = SkipTo(block_end)
sqldecl = Forward()
sqldecl << (block_start + sql_start + Optional(colon) +
Optional(param_dict) + sql_end +
sql_text.setResultsName('sqltext') + block_end)
block_str = "".join(block_text)
tokens = sqldecl.parseString(block_str)
return tokens
def create_grammar():
TRUE = Keyword('true')
FALSE = Keyword('false')
NULL = Keyword('null')
LBRACK, RBRACK, LBRACE, RBRACE, COLON = map(Suppress, '[]{}:')
string = dblQuotedString()
number = pyparsing_common.number()
object_ = Forward()
value = Forward()
elements = delimitedList(value)
array = Group(LBRACK + Optional(elements, []) + RBRACK)
value <<= (string | number | Group(object_) | array | TRUE | FALSE | NULL)
member = Group(string + COLON + value)
members = delimitedList(member)
object_ <<= Dict(LBRACE + Optional(members) + RBRACE)
return value
def PartitionParser():
start_ = Suppress('start') + Suppress('=') + Word(nums)
size_ = Suppress('size') + Suppress('=') + Word(nums)
id_ = Suppress('Id') + Suppress('=') + Word(nums)
device_ = Word(alphas + nums + '/')
comment_ = '#' + Optional(restOfLine)
warning_ = 'Warning:' + Optional(restOfLine)
unit_ = Literal('unit') + Optional(
Suppress(':') + Word(alphas + nums) + restOfLine)
pinfo = start_ + Suppress(',')
pinfo += size_ + Suppress(',')
pinfo += id_ + restOfLine
partition = Group(device_ + Suppress(':') + pinfo)
partition.ignore(comment_)
partition.ignore(warning_)
partition.ignore(unit_)
#partition = ZeroOrMore(partition)
return Dict(ZeroOrMore(partition))
def __init__(self):
FALSE = Keyword("false")
NULL = Keyword("null")
TRUE = Keyword("true")
FALSE.setParseAction(replaceWith(False))
NULL.setParseAction(replaceWith(None))
TRUE.setParseAction(replaceWith(True))
pattern = Forward()
label = Word(alphas, alphanums + "_").setResultsName("layer_name")
configurable_param = nestedExpr(content=pattern)
arg = (NULL ^ FALSE ^ TRUE ^ pyparsing_common.number ^
(Word(alphanums + "*_") + ~Word("=")) ^ configurable_param)
args = arg[...].setResultsName("args")
args.setParseAction(self.convert_list)
options = Dict(Group(Word(alphanums + "_") + Suppress("=") +
arg))[...].setResultsName("options")
options.setParseAction(self.convert_dict)
pattern <<= label + args + options
pattern.setParseAction(Pattern)
self.pattern = pattern
def parse_block(self, block_text):
# make a grammar
block_start = Literal("{")
sql_start = Keyword("sqltable", caseless=True)
colon = Literal(":")
sql_end = Literal("}")
separator = Literal("|")
block_end = Keyword("{sqltable}", caseless=True)
# params
field_name = Word(alphanums)
equal_sign = Suppress(Literal("="))
# whatever value
field_value = (CharsNotIn("|}"))
# param name and value
param_group = Group(field_name + equal_sign + field_value)
# list of all params
param_list = delimitedList(param_group, '|')
# helper
param_dict = Dict(param_list)
# sql text
sql_text = SkipTo(block_end)
sqldecl = Forward()
sqldecl << (block_start +
sql_start +
Optional(colon) +
Optional(param_dict) +
sql_end +
sql_text.setResultsName('sqltext') +
block_end)
block_str = "".join(block_text)
tokens = sqldecl.parseString( block_str )
return tokens
def _define_json():
# https://pyparsing.wikispaces.com/file/view/jsonParser.py
TRUE = _make_keyword('true', True)
FALSE = _make_keyword('false', False)
NULL = _make_keyword('null', None)
LBRACK, RBRACK, LBRACE, RBRACE, COLON = map(Suppress, '[]{}:')
jsonString = dblQuotedString().setParseAction(removeQuotes)
jsonNumber = pyparsing_common.number()
jsonObject = Forward()
jsonValue = Forward()
jsonElements = delimitedList(jsonValue)
jsonArray = Group(LBRACK + Optional(jsonElements, []) + RBRACK)
jsonValue << (jsonString | jsonNumber | Group(jsonObject) | jsonArray
| TRUE | FALSE | NULL) # noqa
memberDef = Group(jsonString + COLON + jsonValue)
jsonMembers = delimitedList(memberDef)
jsonObject << Dict(LBRACE + Optional(jsonMembers) + RBRACE)
return jsonValue
def _build():
"""Encapsulate so the variables don't leak out."""
# Basic punctuation
colon = Literal(':').suppress()
hashmark = Literal('#').suppress()
comment = (hashmark + restOfLine).suppress()
# Enforce Python-style naming conventions
command_name = Word(srange("[A-Z]"), srange("[a-zA-Z0-9]")) # StudlyCaps
field_name = Word(srange("[a-z_]"),
srange("[a-z0-9_]")) # lower_underscore
# Put it all together
fields = Dict(OneOrMore(Group(field_name + colon + restOfLine)))
fields = nestedExpr(opener="{", closer="}", content=fields)
command = Group(command_name + fields)
# Configure the parser
tml_parser = OneOrMore(command) + stringEnd
tml_parser.ignore(comment)
return tml_parser
def _parse_rec_string(self):
'''
Pyparsing parser to the received string
'''
#any word
word = Word(printables)
#recognized tokens
from_t = CaselessLiteral(self._FROM)
by_t = CaselessLiteral(self._BY)
with_t = CaselessLiteral(self._WITH)
id_t = CaselessLiteral(self._ID)
via_t = CaselessLiteral(self._VIA)
for_t = CaselessLiteral(self._FOR)
#A group of non tokens
phrase = OneOrMore(~from_t + ~by_t + ~with_t + ~id_t + ~for_t +
~via_t + word)
#Group phrase with token
from_g = Optional(Group(from_t + phrase))
by_g = Optional(Group(by_t + phrase))
with_g = Optional(Group(with_t + phrase))
id_g = Optional(Group(id_t + phrase))
via_g = Optional(Group(via_t + phrase))
for_g = Optional(Group(for_t + phrase))
grouped_data = from_g & by_g & with_g & id_g & via_g & for_g
parse_to_dict = Dict(grouped_data)
try:
rec_dict = parse_to_dict.parseString(self._rec)
except ParseException:
rec_dict = {}
return rec_dict
'next',
'option',
'reserved',
'rewind',
'starts',
'tstp',
'tsfp',
'uid',
'vendor-class-identifier',
]
other_statement = oneOf(other_statement_names, caseless=True) + args
lease_statement = (hardware | deleted | ends | set_statement | lone_statement
| other_statement) + Suppress(';')
lease_parser = (lease_or_host("lease_or_host") + ip("ip") + Suppress('{') +
Dict(ZeroOrMore(Group(lease_statement))) + Suppress('}'))
lease_parser.ignore('#' + restOfLine)
def is_lease(entry):
"""Is `entry` a lease declaration?"""
entry_type = entry.lease_or_host.lower()
assert entry_type in {
'host', 'lease'
}, ("Unknown entry type (not a host or lease): %s" % entry_type)
return entry_type == 'lease'
def is_host(entry):
"""Is `entry` a host declaration?"""
return not is_lease(entry)
messageDefn = MESSAGE_ - ident("messageId") + LBRACE + messageBody(
"body") + RBRACE
typespec = oneOf("""double float int32 int64 uint32 uint64 sint32 sint64
fixed32 fixed64 sfixed32 sfixed64 bool string bytes"""
) | ident
rvalue = integer | TRUE_ | FALSE_ | ident
fieldDirective = LBRACK + Group(ident + EQ + rvalue) + RBRACK
fieldDefnPrefix = REQUIRED_ | OPTIONAL_ | REPEATED_
fieldDefn = (Optional(fieldDefnPrefix) + typespec("typespec") +
ident("ident") + EQ + integer("fieldint") +
ZeroOrMore(fieldDirective) + SEMI)
# enumDefn ::= 'enum' ident '{' { ident '=' integer ';' }* '}'
enumDefn = ENUM_("typespec") - ident('name') + LBRACE + Dict(
ZeroOrMore(Group(ident + EQ + integer + SEMI)))('values') + RBRACE
# extensionsDefn ::= 'extensions' integer 'to' integer ';'
extensionsDefn = EXTENSIONS_ - integer + TO_ + integer + SEMI
# messageExtension ::= 'extend' ident '{' messageBody '}'
messageExtension = EXTEND_ - ident + LBRACE + messageBody + RBRACE
# messageBody ::= { fieldDefn | enumDefn | messageDefn | extensionsDefn | messageExtension }*
messageBody << Group(
ZeroOrMore(
Group(fieldDefn | enumDefn | messageDefn | extensionsDefn
| messageExtension)))
# methodDefn ::= 'rpc' ident '(' [ ident ] ')' 'returns' '(' [ ident ] ')' ';'
methodDefn = (RPC_ - ident("methodName") + LPAR +
sattrs = ";".join(['%s:%s' % (k, self.attrs[k]) for k in sorted(self.attrs)])
return 'attr(%s)' % sattrs
def __repr__(self):
rattrs = ",".join(['%r:%r' % (k, self.attrs[k]) for k in sorted(self.attrs)])
return 'Attr({%s})' % rattrs
@staticmethod
def parse_action(s, loc, tokens):
where = W(s, loc)
attrs = tokens.get('attrs', None)
attrs = {k: v for k, v in attrs.items()}
return Attr(attrs, where=where)
attr_spec = Dict(
delimitedList(
Group(Word(alphanums) +
Suppress(Literal(':')) +
contract_expression('value')),
delim=';')
)('attrs')
attrs_spec = ('(' - attr_spec - ')')
attr_contract = Keyword('attr') - attrs_spec
attr_contract.setParseAction(Attr.parse_action)
add_contract(attr_contract)
add_keyword('attr')
VARCHAR = CaselessKeyword("VARCHAR")
WHERE = CaselessKeyword("WHERE")
query = Forward()
whereExpression = Forward()
ident = Word(alphanums + "_$").setName("identifier")
data_type = Group(VARCHAR + "(" + Word(nums) +
")") | INT | TEXT | DOUBLE | BOOLEAN
column_name = (delimitedList(ident, ".", combine=True)("column_name"))
column_name_list = Group(delimitedList(column_name))
primary_key = PRIMARY + KEY + "(" + column_name_list("key_columns") + ")"
column_definition = Group(ident("def_column_name") +
data_type("data_type")) | Group(
primary_key("primary_key"))
column_definition_list = Dict(delimitedList(column_definition))
table_name = delimitedList(ident, ".", combine=True)
table_names = Group(delimitedList(table_name))
E = CaselessLiteral("E")
binop = oneOf("= != < > >= <= eq ne lt le gt ge", caseless=True)
arithSign = Word("+-", exact=1)
realNum = Combine(
Optional(arithSign) +
(Word(nums) + "." + Optional(Word(nums)) | ("." + Word(nums))) +
Optional(E + Optional(arithSign) + Word(nums)))
intNum = Combine(
Optional(arithSign) + Word(nums) +
Optional(E + Optional("+") + Word(nums)))
columnRval = realNum | intNum | quotedString | column_name # need to add support for alg expressions
