def initBNF(self):
indentStack = [1]
encoding = Literal("<").suppress() + identifier("encoding") + Literal(
">").suppress()
constdecl = Group((const + identifier + assign + value).setParseAction(
self.const_action))
vardecl = Group(
(type_("type_") + Optional(encoding) + identifier("name") +
Optional(assign + value("value") + Optional(identifier)("unit"))
).setParseAction(self.var_action))
insertdecl = Group(
(insert + dblQuotedString + LineEnd().suppress()).setParseAction(
self.insert_action))
procedurecall = Group((identifier + Literal("(").suppress() + Optional(
delimitedList(
(identifier + Optional(assign + identifier)).setParseAction(
self.named_param_action))) +
Literal(")").suppress()).setParseAction(
self.procedurecall_action))
condition = Group(
(identifier("leftidentifier") + comparison("comparison") +
(identifier("identifier")
| value.setParseAction(self.value_action))).setParseAction(
self.condition_action))("condition")
pointer = Literal("*") + identifier
rExp = Forward()
#numexpression = Forward()
opexpression = (identifier("operand") +
(Literal(">>") | Literal("<<") | Literal("+")
| Literal("*") | Literal("/") | Literal("-"))("op") +
Group(rExp)("argument")).setParseAction(
self.opexpression_action)
rExp << (
procedurecall | opexpression | identifier("identifier")
| value.setParseAction(self.value_action) |
#Group( Suppress("(") + rExp + Suppress(")") ) |
#Group( "+" + rExp) |
#Group( "-" + rExp) |
Group(Literal("not") + rExp))
rExpCondition = Group(
(Optional(not_)("not_") + rExp("rExp"))).setParseAction(
self.rExp_condition_action)("condition")
rExp.setParseAction(self.rExp_action)
assignment = ((identifier | pointer)("lval") + assign +
rExp("rval")).setParseAction(self.assignment_action)
addassignment = ((identifier | pointer)("lval") +
(Literal("+=") | Literal("-=") | Literal("*=")
| Literal("&=") | Literal("|=") | Literal(">>=")
| Literal("/=") | Literal("<<="))("op") +
Group(rExp)("rval")).setParseAction(
self.addassignement_action)
statement = Forward()
statementBlock = indentedBlock(statement, indentStack).setParseAction(
self.statementBlock_action)
procedure_statement = Group(
(Keyword("def").suppress() + identifier("funcname") +
Literal("(").suppress() + Literal(")").suppress() +
colon.suppress() + statementBlock).setParseAction(
self.def_action))
while_statement = Group(
(Keyword("while").suppress() +
(condition | rExpCondition)("condition") + colon.suppress() +
statementBlock("statementBlock")).setParseAction(
self.while_action))
if_statement = (Keyword("if") + condition + colon +
statementBlock("ifblock") + Optional(
Keyword("else").suppress() + colon +
statementBlock("elseblock"))).setParseAction(
self.if_action)
statement << (procedure_statement | while_statement | if_statement
| procedurecall | assignment | addassignment)
decl = constdecl | vardecl | insertdecl | Group(statement)
self.program = ZeroOrMore(decl)
self.program.ignore(pythonStyleComment)
def parse_survey():
indent_stack = [1]
stmt = Forward()
identifier = Word(alphas, alphanums + "_")
number = Word(nums)
doublequoted_string = QuotedString('"', escChar="\\")
singlequoted_string = QuotedString("'", escChar="\\")
string = doublequoted_string | singlequoted_string
value = number | string
# TODO Parse expressions properly.
# op = oneOf("+ - * /")
comp_op = oneOf("= != >= > <= <")
comment = Group("#" + restOfLine)
command = Group("@" + identifier + OneOrMore((identifier | value), stopOn=LineEnd()))
variable = Group("${" + identifier + "}")
dot = Literal(".")
expr = Group((variable | dot) + comp_op + value)
if_cond = "if" + (expr | value)
if_body = indentedBlock(pp.ungroup(stmt), indent_stack)
if_block = Group(if_cond + Suppress(":") + if_body)
q_name = identifier
q_type = Group(OneOrMore(identifier))
q_label = string
q_param = ((identifier + identifier) | (identifier + value))
q_params = ZeroOrMore(q_param, stopOn=LineEnd())
question = Group(q_name + q_type + Suppress(":") + Optional(q_label) + q_params)
group_params = Group(ZeroOrMore(q_param, stopOn=LineEnd()) + Optional(if_cond))
group_label = string
group_def = "group" + Group(identifier + Optional(group_label)) + group_params + Optional(if_cond) + Suppress(":")
group_body = indentedBlock(pp.ungroup(stmt), indent_stack)
group_block = Group(group_def + group_body)
repeat_def = "repeat" + Group(identifier + Optional(group_label)) + group_params + Suppress(":")
repeat_block = Group(repeat_def + group_body)
# TODO Add + Suppress(LineEnd())?
stmt <<= (comment | command | if_block | group_block | repeat_block | question)
stmts = OneOrMore(stmt)
return stmts
def parse_choices():
indent_stack = [1]
stmt = Forward()
identifier = Word(alphas, alphanums + "_")
number = Word(nums)
string = QuotedString('"', escChar="\\")
value = number | string
comment = Group("#" + restOfLine)
command = Group("@" + identifier +
OneOrMore((identifier | value), stopOn=LineEnd()))
choice_param = ((identifier + identifier) | (identifier + value))
choice_params = Group(ZeroOrMore(choice_param, stopOn=LineEnd()))
choice_value = value
choice_label = value
choice = Group(choice_value + choice_label + choice_params)
list_name = identifier
list_def = "list" + list_name + choice_params + Suppress(":")
list_body = indentedBlock(ungroup(stmt), indent_stack)
list_block = Group(list_def + list_body)
stmt <<= (comment | command | list_block | choice)
stmts = OneOrMore(stmt)
return stmts
def _build_csv_parser():
separator = pp.Suppress(':')
key = pp.Word(pp.printables, excludeChars=':')
value = pp.Regex(r'[^\n\r]*') + pp.LineEnd().suppress()
block_name = key + separator + pp.LineEnd().suppress()
key_value = key + separator + value
header = (pp.LineStart().suppress() + pp.Word(pp.nums) + pp.ZeroOrMore( pp.White().suppress() + pp.Word(pp.nums)) + pp.LineEnd().suppress())
csv_header = pp.delimitedList(pp.Word(pp.printables, excludeChars=',')) + pp.LineEnd().suppress()
csv_row = pp.delimitedList(pp.Word(pp.nums + ';.+-e_') | pp.Literal('custom')) + pp.LineEnd().suppress()
indent_stack = [1]
block = pp.Forward()
block_body = ( block | key_value)
indented_block = pp.Dict(pp.ungroup(pp.indentedBlock(block_body, indent_stack)))
block << ( block_name + indented_block | key_value)
return pp.Optional(header) + pp.ZeroOrMore(pp.Dict(pp.Group(block))).setResultsName('meta') + \
csv_header.setResultsName('csvHeader') + \
pp.Group(pp.OneOrMore(pp.Group(csv_row))).setResultsName('csvValues')
def DOCSTR_BLOCK(expr, resultsName=None): # pylint: disable=invalid-name
"""Block with an optional docstring followed by one of more `expr`."""
# Copied from pyparsing.indentedBlock
def checkSubIndent(s, l, t): # pylint: disable=invalid-name
curCol = pp.col(l, s) # pylint: disable=invalid-name
if curCol > _indentation_stack[-1]:
_indentation_stack.append(curCol)
else:
raise pp.ParseException(s, l, 'not a subentry')
def checkUnindent(s, l, t): # pylint: disable=invalid-name
del t
if l >= len(s):
return
curCol = pp.col(l, s) # pylint: disable=invalid-name
if not (_indentation_stack and curCol < _indentation_stack[-1]
and curCol <= _indentation_stack[-2]):
raise pp.ParseException(s, l, 'not an unindent')
_indentation_stack.pop()
INDENT = ( # pylint: disable=invalid-name
pp.Empty() +
pp.Empty().setParseAction(checkSubIndent)).setName('INDENT')
UNDENT = pp.Empty().setParseAction(checkUnindent).setName('UNINDENT') # pylint: disable=invalid-name
docstring = Group(
Tag('docstring') + pp.QuotedString('"""', multiline=True))
code = Optional(pp.indentedBlock(expr, _indentation_stack, False), [])
if resultsName:
code = code.setResultsName(resultsName)
body = INDENT + Optional(docstring)('docstring') + code + UNDENT
return S(':') + NEWLINE - body
def _build_ana_rcp_parser():
separator = pp.Suppress(':')
key = pp.Word(pp.printables, excludeChars=':')
value = pp.Regex(r'[^\n\r]*') + pp.LineEnd().suppress()
block_name = key + separator + pp.LineEnd().suppress()
platemap_keylist = pp.Literal(
'platemap_comp4plot_keylist') + separator + pp.delimitedList(
pp.Word(pp.alphas))
run_ids = pp.Literal('run_ids') + separator + pp.delimitedList(
pyparsing_common.integer)
plate_id = (pp.Literal('plate_ids') |
pp.Literal('plate_id')) + separator + pyparsing_common.integer
key_value = (platemap_keylist | run_ids | plate_id
| key + separator + value)
indent_stack = [1]
block = pp.Forward()
block_body = (block | key_value)
indented_block = pp.Dict(
pp.ungroup(pp.indentedBlock(block_body, indent_stack)))
block << (block_name + indented_block | key_value)
return pp.OneOrMore(pp.Dict(pp.Group(block)))
def parse(self, data=""):
"""Parse ssh-config data
args:
data (str)
returns:
Parsed config or None (dict)
"""
if data:
self.raw = data
SPACE = White().suppress()
SEP = Suppress(SPACE) | Suppress("=")
HOST = CaselessLiteral("Host").suppress()
MATCH = CaselessLiteral("Match").suppress()
KEY = Word(alphanums)
VALUE = Word(alphanums + ' ~%*?!._-+/,"')
paramValueDef = SkipTo("#" | lineEnd)
indentStack = [1]
HostDecl = (HOST | MATCH) + SEP + VALUE
paramDef = Dict(Group(KEY + SEP + paramValueDef))
block = indentedBlock(paramDef, indentStack)
HostBlock = Dict(Group(HostDecl + block))
try:
return OneOrMore(HostBlock).ignore(pythonStyleComment).parseString(self.raw)
except ParseException as e:
print(e)
return None
def make_ccss_parser(self):
# CleverCSS (CCSS) grammar
def _parse_name(string, location, tokens):
for t in tokens:
if t[-1] == ';':
t.replace(';', '')
def _parse_property_value(string, location, tokens):
pass
indent_stack = [1]
PROPERTY_NAME = Word(alphanums + '-*')
PROPERTY_VALUE = Word(alphanums + ' (`\'/,%#-."\\);$_').setParseAction(_parse_name)
PROPERTY_LINE_END = ((Optional(Word(';')).suppress() + LineEnd()) |
LineEnd().suppress())
CCSS_PROPERTY = Group(PROPERTY_NAME +
Word(':').suppress() +
PROPERTY_VALUE.setParseAction(_parse_property_value) +
PROPERTY_LINE_END
).setResultsName('property')
CCSS_BREAK = Word('&')
CCSS_CLASS_NAME = Word(alphanums + '.#_-*[]\'="')
CCSS_SINGLE_CLASS = Combine(
Optional(
Combine(CCSS_BREAK + Optional(White()) + Word('>')) |
Combine(CCSS_BREAK + Optional(White()) + Word(':')) |
CCSS_BREAK
) +
Optional(White()) +
(CCSS_CLASS_NAME ^
(CCSS_CLASS_NAME + Word(':') + CCSS_CLASS_NAME))
)
CCSS_SELECTOR = Group(OneOrMore(CCSS_SINGLE_CLASS))
CCSS_SELECTOR_GROUP = Group(delimitedList(CCSS_SELECTOR) +
Word(':').suppress() +
LineEnd().suppress()
).setResultsName('selectors')
CCSS_DEF = Forward()
CCSS_DEF << (Group(
Optional(LineEnd()).suppress() +
CCSS_SELECTOR_GROUP +
indentedBlock(
OneOrMore(CCSS_PROPERTY).setResultsName('properties') |
OneOrMore(CCSS_DEF)
, indent_stack).setResultsName('nodes')
)).setResultsName('content')
CCSS = OneOrMore(CCSS_DEF).ignore(cStyleComment)
return CCSS
def parse(self, data=""):
if data:
self.raw = data
SPACE = White().suppress()
HOST = Literal("Host").suppress()
KEY = Word(alphanums + "~*._-/")
VALUE = Word(alphanums + "~*._-/")
paramValueDef = SkipTo("#" | lineEnd)
indentStack = [1]
HostDecl = HOST + SPACE + VALUE
paramDef = Dict(Group(KEY + SPACE + paramValueDef))
block = indentedBlock(paramDef, indentStack)
HostBlock = Dict(Group(HostDecl + block))
try:
return OneOrMore(HostBlock).ignore(pythonStyleComment).parseString(self.raw)
except ParseException as e:
return None
def _get_block(file_lines, line) -> str:
"""
Return a Python block of code beginning in line.
:param file_lines: Lines of code
:param line: First line of block
"""
frst_ln = file_lines[line - 1]
file_lines = file_lines[line - 1:]
rem_file = "\n".join(file_lines)
indent_stack = [len(frst_ln) - len(frst_ln.lstrip(' ')) + 1]
prs_block = Forward()
block_line = SkipTo(LineEnd())
block_header = SkipTo(LineEnd())
block_body = indentedBlock(prs_block, indent_stack)
block_def = Group(block_header + block_body)
# pylint: disable=pointless-statement
prs_block << (block_def | block_line)
block_list = prs_block.parseString(rem_file).asList()
block_str = (lang.lists_as_string(block_list, '', 0))
return block_str.rstrip()
def parser(self):
integer = Regex('\d+').setParseAction(lambda x, y, z: int(z[0]))
floating_point = Regex('[+-]?(\d*[.])?\d+(e[+-]\d+)?').setParseAction(
lambda x, y, z: float(z[0])
) #Group(Optional(Suppress('-')) + Regex('\d+') + Optional(Literal('.') + Regex('\d+')))
left_bracket = Suppress('[')
right_bracket = Suppress(']')
variable_name = Word(alphas + '0123456789_')
lt = Literal('<')
branch = (integer + Suppress(':') + left_bracket + variable_name + lt +
floating_point + right_bracket +
Group(Literal('yes=') + integer) + Suppress(',') +
Group(Literal('no=') + integer) + Suppress(',') +
Group(Literal('missing=') + integer))
# branch
leaf = integer + Suppress(':') + Suppress('leaf=') + floating_point
stack = [1]
tree = Forward()
tree << (leaf ^ (branch + indentedBlock(tree, stack)))
return OneOrMore(tree)
def buildSyntax(self):
'''WIP'''
dreamScript = pyp.Forward()
# Constants
singlelineString = pyp.QuotedString(
'"', '\\').setResultsName('string').setParseAction(self.makeString)
fileRef = pyp.QuotedString(
"'",
'\\').setResultsName('fileRef').setParseAction(self.makeFileRef)
multilineString = pyp.QuotedString(
quoteChar='{"', endQuoteChar='"}',
multiline=True).setResultsName('string').setParseAction(
self.makeString)
number = pyp.Regex(r'\d+(\.\d*)?([eE]\d+)?').setResultsName(
'number').setParseAction(self.makeNumber)
# Other symbols
listStart = pyp.Suppress('list(')
listEnd = pyp.Suppress(')')
VAR_GLOBAL = pyp.Keyword('global')
VAR_CONST = pyp.Keyword('const')
SLASH = pyp.Literal('/')
EQUAL = pyp.Literal('=')
VAR = pyp.Keyword('var')
#############################
# Grammar
#############################
constant = singlelineString | fileRef | multilineString | number | dreamList
# Lists
listElement = constant | (constant + EQUAL + constant)
listElement = pyp.operatorPrecedence(listElement, [
(
"=",
2,
pyp.opAssoc.LEFT,
),
])
listContents = pyp.delimitedList(listElement)
dreamList << pyp.Group(listStart + listContents + listEnd)
dreamList.setParseAction(self.handleList)
# Paths
relpath = pyp.ident | relpath + SLASH + pyp.ident
abspath = SLASH + relpath
path = (abspath | relpath).setParseAction(self.handlePath)
pathslash = path + SLASH
# Preprocessor stuff
ppStatement = pyp.Forward()
ppDefine = pyp.Keyword('#define') + pyp.ident.setResultsName(
'name') + pyp.restOfLine.setResultsName('value')
ppDefine.setParseAction(self.handlePPDefine)
ppUndef = pyp.Keyword('#undef') + pyp.ident.setResultsName('name')
ppUndef.setParseAction(self.handlePPUndef)
ppIfdef = (pyp.Keyword('#ifdef') +
pyp.ident.setResultsName('name')).setParseAction(
self.handlePPIfdef)
ppIfndef = (pyp.Keyword('#ifndef') +
pyp.ident.setResultsName('name')).setParseAction(
self.handlePPIfndef)
ppElse = (pyp.Keyword('#else') +
pyp.ident.setResultsName('name')).setParseAction(
self.handlePPElse)
ppEndif = pyp.Keyword('#endif').setParseAction(self.handlePPElse)
ppStatement = pyp.lineStart + (ppIfdef | ppIfndef | ppElse | ppEndif)
# Var Declarations
##########################
var_modifiers = pyp.ZeroOrMore(
SLASH + (VAR_GLOBAL | VAR_CONST)).setResultsName('modifiers')
var_assignment = EQUAL + constant
varblock_inner_ref = pyp.ident.setResultsName('name') + pyp.Optional(
var_assignment)
var_argument = VAR + pyp.Optional(
abspath) + SLASH + pyp.ident.setResultsName('name') + pyp.Optional(
var_assignment)
varblock_inner_decl = var_modifiers + pyp.Optional(
abspath) + SLASH + varblock_inner_ref
varblock_element = varblock_inner_decl | varblock_inner_ref
varblock = VAR + pyp.indentedBlock(var_blockinner_decl)
inline_vardecl = VAR + varblock_inner_decl
vardecl = varblock | inline_vardecl
# Proc Declarations
PROC = pyp.Keyword('proc')
proc_args = '(' + pyp.delimitedList(
var_argument | pyp.ident.setResultsName('name')) + ')'
procblock_proc = pyp.ident.setResultsName(
'name') + proc_args + pyp.indentedBlock(proc_instructions)
procblock = PROC + pyp.indentedBlock(procblock_proc, [1])
# Atom blocks
atomdecl = pyp.Forward()
atomdecl << path + pyp.indentedBlock(vardecl | atomdecl | procdecl,
[1])
return dreamScript
SW = lambda kw: S(W(kw))
Group = pp.Group # pylint: disable=invalid-name
Optional = pp.Optional # pylint: disable=invalid-name
LineNum = pp.lineno # pylint: disable=invalid-name
# Grammar for Python Type Declaration (CLIF version)
def fail_block(s, unused_loc, expr, err):
raise pp.ParseFatalException(s, err.loc, 'invalid statement in %s' % expr)
_indentation_stack = [1] # Holds cols to indent from.
BLOCK = lambda el: (
S(':') + NEWLINE - # pylint: disable=g-long-lambda
pp.indentedBlock(el, _indentation_stack).setFailAction(fail_block))
def reset_indentation(): # pylint: disable=invalid-name
_indentation_stack[:] = [1]
K = lambda el: pp.Keyword(el).setParseAction(lambda el, loc, t: [t[0], loc])
# pylint: disable=undefined-variable
E = lambda el: pp.Empty().setParseAction(lambda: el)
Tag = lambda el: pp.Empty().setParseAction(lambda s, loc, t: [el, loc]) # pylint: disable=invalid-name
# pylint: enable=undefined-variable
NEWLINE = pp.lineEnd.setWhitespaceChars(' ').suppress().ignore(
pp.pythonStyleComment)
def __init__(self, fragment_file, sdkconfig):
try:
fragment_file = open(fragment_file, "r")
except TypeError:
pass
path = os.path.realpath(fragment_file.name)
indent_stack = [1]
class parse_ctx:
fragment = None # current fragment
key = "" # current key
keys = list() # list of keys parsed
key_grammar = None # current key grammar
@staticmethod
def reset():
parse_ctx.fragment_instance = None
parse_ctx.key = ""
parse_ctx.keys = list()
parse_ctx.key_grammar = None
def fragment_type_parse_action(toks):
parse_ctx.reset()
parse_ctx.fragment = FRAGMENT_TYPES[
toks[0]]() # create instance of the fragment
return None
def expand_conditionals(toks, stmts):
try:
stmt = toks["value"]
stmts.append(stmt)
except KeyError:
try:
conditions = toks["conditional"]
for condition in conditions:
try:
_toks = condition[1]
_cond = condition[0]
if sdkconfig.evaluate_expression(_cond):
expand_conditionals(_toks, stmts)
break
except IndexError:
expand_conditionals(condition[0], stmts)
except KeyError:
for tok in toks:
expand_conditionals(tok, stmts)
def key_body_parsed(pstr, loc, toks):
stmts = list()
expand_conditionals(toks, stmts)
if parse_ctx.key_grammar.min and len(
stmts) < parse_ctx.key_grammar.min:
raise ParseFatalException(
pstr, loc,
"fragment requires at least %d values for key '%s'" %
(parse_ctx.key_grammar.min, parse_ctx.key))
if parse_ctx.key_grammar.max and len(
stmts) > parse_ctx.key_grammar.max:
raise ParseFatalException(
pstr, loc,
"fragment requires at most %d values for key '%s'" %
(parse_ctx.key_grammar.max, parse_ctx.key))
try:
parse_ctx.fragment.set_key_value(parse_ctx.key, stmts)
except Exception as e:
raise ParseFatalException(
pstr, loc,
"unable to add key '%s'; %s" % (parse_ctx.key, e.message))
return None
key = Word(alphanums + "_") + Suppress(":")
key_stmt = Forward()
condition_block = indentedBlock(key_stmt, indent_stack)
key_stmts = OneOrMore(condition_block)
key_body = Suppress(key) + key_stmts
key_body.setParseAction(key_body_parsed)
condition = originalTextFor(
SDKConfig.get_expression_grammar()).setResultsName("condition")
if_condition = Group(
Suppress("if") + condition + Suppress(":") + condition_block)
elif_condition = Group(
Suppress("elif") + condition + Suppress(":") + condition_block)
else_condition = Group(
Suppress("else") + Suppress(":") + condition_block)
conditional = (if_condition + Optional(OneOrMore(elif_condition)) +
Optional(else_condition)).setResultsName("conditional")
def key_parse_action(pstr, loc, toks):
key = toks[0]
if key in parse_ctx.keys:
raise ParseFatalException(
pstr, loc,
"duplicate key '%s' value definition" % parse_ctx.key)
parse_ctx.key = key
parse_ctx.keys.append(key)
try:
parse_ctx.key_grammar = parse_ctx.fragment.get_key_grammars(
)[key]
key_grammar = parse_ctx.key_grammar.grammar
except KeyError:
raise ParseFatalException(
pstr, loc, "key '%s' is not supported by fragment" % key)
except Exception as e:
raise ParseFatalException(
pstr, loc,
"unable to parse key '%s'; %s" % (key, e.message))
key_stmt << (conditional
| Group(key_grammar).setResultsName("value"))
return None
def name_parse_action(pstr, loc, toks):
parse_ctx.fragment.name = toks[0]
key.setParseAction(key_parse_action)
ftype = Word(alphas).setParseAction(fragment_type_parse_action)
fid = Suppress(":") + Word(alphanums + "_.").setResultsName("name")
fid.setParseAction(name_parse_action)
header = Suppress("[") + ftype + fid + Suppress("]")
def fragment_parse_action(pstr, loc, toks):
key_grammars = parse_ctx.fragment.get_key_grammars()
required_keys = set(
[k for (k, v) in key_grammars.items() if v.required])
present_keys = required_keys.intersection(set(parse_ctx.keys))
if present_keys != required_keys:
raise ParseFatalException(
pstr, loc, "required keys %s for fragment not found" %
list(required_keys - present_keys))
return parse_ctx.fragment
fragment_stmt = Forward()
fragment_block = indentedBlock(fragment_stmt, indent_stack)
fragment_if_condition = Group(
Suppress("if") + condition + Suppress(":") + fragment_block)
fragment_elif_condition = Group(
Suppress("elif") + condition + Suppress(":") + fragment_block)
fragment_else_condition = Group(
Suppress("else") + Suppress(":") + fragment_block)
fragment_conditional = (
fragment_if_condition +
Optional(OneOrMore(fragment_elif_condition)) +
Optional(fragment_else_condition)).setResultsName("conditional")
fragment = (header +
OneOrMore(indentedBlock(key_body, indent_stack,
False))).setResultsName("value")
fragment.setParseAction(fragment_parse_action)
fragment.ignore("#" + restOfLine)
deprecated_mapping = DeprecatedMapping.get_fragment_grammar(
sdkconfig, fragment_file.name).setResultsName("value")
fragment_stmt << (Group(deprecated_mapping) | Group(fragment)
| Group(fragment_conditional))
def fragment_stmt_parsed(pstr, loc, toks):
stmts = list()
expand_conditionals(toks, stmts)
return stmts
parser = ZeroOrMore(fragment_stmt)
parser.setParseAction(fragment_stmt_parsed)
self.fragments = parser.parseFile(fragment_file, parseAll=True)
for fragment in self.fragments:
fragment.path = path
RPAREN = pp.Suppress(')')
LBRACK = pp.Suppress('[')
RBRACK = pp.Suppress(']')
COLON = pp.Suppress(':')
NAME = pp.Word(pp.alphas + '_', pp.alphanums + '_')
string = pp.quotedString().setParseAction(pp.removeQuotes)
number = ppc.number()
comment = '#' + pp.restOfLine
var = NAME.copy().setParseAction(el.Var)
expr = pp.Forward()
stmt = pp.Forward()
stack = [1]
suite = pp.indentedBlock(expr, stack)
params = pp.Optional(pp.delimitedList(expr))
ruledecl = pp.Suppress('rule') + NAME + \
pp.Group(LPAREN + params + RPAREN) + COLON
ruledef = pp.Group(ruledecl + suite).setParseAction(el.Rule)
grouped = (LPAREN + expr + RPAREN).setParseAction(el.Group)
listed = (LBRACK + params + RBRACK).setParseAction(el.List)
call = (NAME + LPAREN + params + RPAREN).setParseAction(el.Call)
sliceops = (opt(expr) + ':' + opt(expr) + opt(':' + expr)) | expr
slice = (LBRACK + sliceops + RBRACK).setParseAction(el.Slice)
compops = pp.oneOf('== != <= < >= >') | (L('is') + L('not')) | \
def get_syntax():
"""
Get the syntax suitable for parsing `pactl list` output
The syntax parses valid text and produces a list of :class:`Record` objects
"""
# The syntax uses indenting so a shared indent stack is required to parse
# output properly. See indentedBlock() documentation for details
indent_stack = [1]
# VALUE
# VALUE = p.Regex(".+").setResultsName("VALUE")
property_value = (
p.indentedBlock(
p.Word(p.alphanums + ".").setResultsName("PROP_NAME")
+ p.Suppress('=')
+ p.QuotedString('"').setResultsName("PROP_VALUE"),
indent_stack
).setResultsName("prop-value")
)
simple_value = p.ungroup(
p.Combine(
p.Regex(r".*")
+ p.Optional(
p.indentedBlock(p.Regex(".+"), indent_stack)
),
joinString='\n',
adjacent=False
)
).setResultsName("simple-value")
value = p.Or(
[property_value, simple_value]
).setResultsName("value")
# NAME
NAME = p.Regex("[^:]+").setResultsName("NAME")
# entry: NAME ':' [VALUE] [indented<VALUE>]
entry = (
NAME + p.Suppress(":") + value
).setResultsName(
"entry"
).setParseAction(
Entry.from_tokens
)
# HEADER
HEADER = p.restOfLine.setResultsName("HEADER")
# record: HEADER '\n' indented<entry>
record = (
HEADER
+ p.Suppress(p.lineEnd)
+ p.indentedBlock(entry, indent_stack).setResultsName("entry-list")
).setResultsName(
"record"
).setParseAction(
Record.from_tokens
)
# record_list: record+
record_list = p.OneOrMore(
record
).setResultsName(
"record-list"
)
syntax = record_list
syntax.enablePackrat()
return syntax
class TypeDocGrammar:
r"""
Grammar to parse boto3 docs syntax.
EOL ::= ["\r"] "\n"
SOL ::= LINE_START
line ::= [^EOL]+ EOL
word ::= alphanums + "_"
indented_block ::= INDENT (line_indented | any_line)
line_indented ::= any_line indented_block
type_definition ::= ":type" [^:]+ ":" [^EOL]+
rtype_definition ::= ":rtype:" [^EOL]+
returns_definition ::= (":returns:" | ":return:") [^EOL]+
param_definition ::= ":param" [^:]+ ":" [^EOL]+ EOL [indented_block]
response_structure ::= "**Response Structure**" line [indented_block]
typed_dict_key_line ::= "-" "**" word "**" "*(" word ")" "--*" [^EOL]+ + EOL
type_line ::= "-" "*(" word ")" "--*" [^EOL]+ + EOL
any_line ::= typed_dict_key_line | type_line | line
"""
indent_stack = [1]
SOL = LineStart().suppress()
EOL = LineEnd().suppress()
word = Word(alphanums + "_")
line = SkipTo(LineEnd()) + EOL
line_indented = Forward()
any_line = Forward()
indented_block = indentedBlock(
line_indented | any_line, indentStack=indent_stack
).setResultsName("indented")
line_indented <<= any_line + indented_block
type_definition = (
SOL
+ Literal(":type")
+ SkipTo(":").setResultsName("name")
+ Literal(":")
+ SkipTo(EOL).setResultsName("type_name")
)
rtype_definition = SOL + Literal(":rtype:") + SkipTo(EOL).setResultsName("type_name")
returns_definition = (
SOL
+ (Literal(":returns:") | Literal(":return:"))
+ SkipTo(EOL).setResultsName("description")
)
param_definition = (
SOL
+ Literal(":param")
+ SkipTo(":").setResultsName("name")
+ Literal(":")
+ SkipTo(EOL).setResultsName("description")
+ EOL
+ Optional(indented_block)
)
response_structure = Literal("**Response Structure**") + line_indented
typed_dict_key_line = (
Literal("-")
+ White(ws=" \t")
+ Literal("**")
+ word.setResultsName("name")
+ Literal("**")
+ White(ws=" \t")
+ Literal("*(")
+ word.setResultsName("type_name")
+ Literal(")")
+ White(ws=" \t")
+ Literal("--*")
+ SkipTo(EOL).setResultsName("description")
+ EOL
)
type_line = (
Literal("-")
+ White(ws=" \t")
+ Literal("*(")
+ word.setResultsName("type_name")
+ Literal(")")
+ White(ws=" \t")
+ Literal("--*")
+ SkipTo(EOL).setResultsName("description")
+ EOL
)
any_line <<= (typed_dict_key_line | type_line | line).setResultsName("line")
@classmethod
def fail_action(
cls, _input_string: str, _chr_index: int, _source: str, error: BaseException
) -> None:
"""
Check for input end.
"""
if "found end of text" not in str(error):
raise error
@classmethod
def reset(cls) -> None:
"""
Reset call stack and packrat.
"""
cls.disable_packrat()
cls.indented_block.setFailAction(cls.fail_action)
cls.indent_stack.clear()
cls.indent_stack.append(1)
@staticmethod
def enable_packrat() -> None:
"""
Enable packrat boost.
"""
ParserElement.enablePackrat(cache_size_limit=128)
@staticmethod
def disable_packrat() -> None:
"""
Disable packrat boost.
"""
ParserElement.enablePackrat(cache_size_limit=None)
conditionals = Forward()
conditional = Group(comparable_expression + comparison + comparable_expression) | \
Group(lpar + conditionals + rpar)
conditionals << Optional(not_logic) + conditional + ZeroOrMore(binary_logic + conditional)
if_keyword = Literal('if')
if_statement = (if_keyword + conditionals).setParseAction(IfStatement)
option = Forward()
response_definition = text + lineEnd.suppress() + Optional(if_statement + Literal('then').suppress())
response = (response_definition + ZeroOrMore(indentedBlock(option, indentStack, True))).setParseAction(Response)
event_send = Literal('->')
event_message_separator = Literal('!').suppress()
event_atom = atom.copy().setParseAction(lambda t: repr(t[0]))
event_message = quotedString | event_atom
event_send_separator = Literal(',').suppress()
event_statement = (event_send + event_atom + event_message_separator + event_message).setParseAction(Event)
options_delimiter = Literal('~')
options_definition = options_delimiter + text + Optional(event_statement + ZeroOrMore(event_send_separator + event_statement))
option << (options_definition + ZeroOrMore(indentedBlock(response, indentStack, True))).setParseAction(Option)
dialog_begin = Literal('begin').suppress() + Group(atom + Optional(Literal('extends').suppress() + atom))
interface_id = Word(alphanums + ":")
#TODO: make function to return thesline_ip_addresse
password = "******" + word_param
enable_password = "******" + word_param
banner_motd = "banner motd " + word_param("type") + restOfLine("path")
line_ip_address = "ip address" + ipAddressWithMask("ip")
line_description = "description" + restOfLine("description")
line_ip_ospf_cost = "ip ospf cost" + integer("cost")
interface_properties = OneOrMore(
line_ip_address("ip address") | line_description("description")
| line_ip_ospf_cost("ospf cost") | comment | hash_comment)
interface_indent = indentedBlock(interface_properties, indentStack,
True)("indent")
interface_id = "interface" + interface_id("id")
interface = (interface_id +
Optional(interface_indent)).setParseAction(fn_interface)
interfaces = OneOrMore(
interface.setResultsName("interface", listAllMatches=True))
#### BGP
#TODO: put each protocol stanza into a function - variable scope, etc
router_bgp = "router bgp" + integer("asn")
#TODO: can router-id be an integer too?
#false_no: returns False if "no" is present, otherwise returns True
false_no = Optional(Literal("no").setParseAction(lambda x: False),
from pyparsing import (indentedBlock, Regex, Suppress, Group, Optional,
OneOrMore, restOfLine, Forward, Literal,
ParserElement, Combine, StringEnd,
ParseResults)
ParserElement.setDefaultWhitespaceChars(' \t')
#COLON = Suppress(Literal(':'))
#label = Suppress(Literal('-')) + Group(OneOrMore(Regex(r'\w+'))) + COLON
label = Group(OneOrMore(Regex(r'\S+')))
nonWhite = Regex(r'\S+')
value = Combine(nonWhite + restOfLine)
item = Forward()
indentStack = [1]
item << (label + Optional(value) +
Optional(indentedBlock(item, indentStack, True)))
parser = OneOrMore(indentedBlock(item, indentStack, False)) + StringEnd()
###
single_top = '''
a
a1
a11
a111?
a112fdsafdsafd fdsa fdsafdsafdsa fdsafdsa fdsa fdsa fsdafdsa
a113
a12
a13
a2
a3
attribute = Word(alphanums + '_-')
value = (Word(alphanums+ '._-') ^ QuotedString('"'))
command = oneOf('Resource Attr Replace Apply SetCache Set Rule Insert Compose Copy Debug Literal Defmac Fetch Script Deliver Prerequisite Modify')
named_command = command + Optional(value)
command_property = Group(attribute + colon + value)
def printCommandName(parse_string, location, matched_tokens_list):
import time
print 'COMMAND (%s):' % str(time.clock()), matched_tokens_list
# This recursive statement form taken from http://w3facility.info/question/simple-demonstration-of-using-pyparsings-indentedblock-recursively/
statement = ((command + newline) ^ (command + arrow) ^ (named_command + arrow)) + ZeroOrMore(command_property)
statement.setParseAction(printCommandName)
statementBlock = Forward()
statementBlock << statement + Optional(indentedBlock(statementBlock, [1]))
if __name__ == '__main__':
from pyparsing import *
import pprint
import sys
#statement.setDebug(True)
FILE = sys.argv[1]
for i, l in enumerate(file(FILE).read().split('\n')):
print i+1, l
parseTree = statementBlock.parseFile(FILE)
print '\n'
pprint.pprint(parseTree.asList())
)
named_command = command + Optional(value)
command_property = Group(attribute + colon + value)
def printCommandName(parse_string, location, matched_tokens_list):
import time
print 'COMMAND (%s):' % str(time.clock()), matched_tokens_list
# This recursive statement form taken from http://w3facility.info/question/simple-demonstration-of-using-pyparsings-indentedblock-recursively/
statement = ((command + newline) ^ (command + arrow) ^
(named_command + arrow)) + ZeroOrMore(command_property)
statement.setParseAction(printCommandName)
statementBlock = Forward()
statementBlock << statement + Optional(indentedBlock(statementBlock, [1]))
if __name__ == '__main__':
from pyparsing import *
import pprint
import sys
#statement.setDebug(True)
FILE = sys.argv[1]
for i, l in enumerate(file(FILE).read().split('\n')):
print i + 1, l
parseTree = statementBlock.parseFile(FILE)
print '\n'
pprint.pprint(parseTree.asList())
def _get_parser():
"""
Create parser for genfile using pyparsing library.
Returns:
Parser capable of turning a genfile into a consumable ParseResults object.
"""
def strip_quotes(orig, loc, toks):
"""Strip quotation marks from the end of a string if present."""
s = toks[0]
if len(s) > 1:
if (s[0] == '"' and s[-1] == '"') or (s[0] == "'" and s[-1] == "'"):
s = s[1:-1]
return s
########## Define important literals ############
pound = p.Literal("#")
point = p.Literal(".")
comma = p.Literal(",")
e = p.CaselessLiteral("E")
colon = p.Literal(":").suppress()
white = p.White(" \t\r")
endl = p.LineEnd().suppress()
plus = p.Literal("+")
minus = p.Literal("-")
nums = p.Word(p.nums)
######### Define important constructs ###########
# Define comment structure
comment = pound + p.restOfLine
# non-quoted string
non_quoted_token = p.Word(p.printables, excludeChars="#,")
non_quoted_token += p.Optional(~endl + white + ~p.FollowedBy(comma|pound|endl))
non_quoted = p.Combine(p.OneOrMore(non_quoted_token))
# quoted string
quote = (p.QuotedString('"', escChar="\\") | p.QuotedString("'", escChar="\\"))
quote.addParseAction(strip_quotes)
# integer
integer = p.Word("+-"+p.nums, p.nums)
integer.setParseAction( lambda s,l,t: int(t[0]) )
# floating point number
fpoint = p.Combine( p.Optional(plus|minus) + p.Optional(nums) +
point + nums + p.Optional( e + integer ) )
fpoint.setParseAction( lambda s,l,t: float(t[0]) )
# Any number
number = (fpoint | integer)
# range specification
range_spec = number + colon + number + p.Optional(colon + number)
range_spec.addParseAction( lambda s,l,t: [tuple(t)] )
######### Define grammar #########################
# Indent stack needed for the pyparsing indentBlock function
indent_stack = [1]
# Statement used for recursive definition of genfile grammar
stmt = p.Forward()
# Variable names must start with a letter and can include
# numbers, dashes, underscores, and periods.
variable = p.Word(p.alphas, p.alphanums + "-_.")
# Comma separated list of values
value = (range_spec | number | quote | non_quoted)
values = p.delimitedList(value)
# Variable domain/scope:
# variable = val1, val2, val3 ...
scope = variable("var") + p.Suppress("=") + p.Group(values)("values")
# Dependencies are variable domains nested beneath another scope
deps = p.indentedBlock(stmt, indent_stack)
# A suite is a variable scope along with all of its dependencies
suite = scope + p.Optional(deps("deps"))
stmt << suite
# A gen schema is one or more suites of variables
genschema = p.OneOrMore(p.Group(suite))
# A genfile starts with an optional schema name then defines a gen schema
genfile = p.Optional(variable("name") + p.FollowedBy(p.lineEnd)) + genschema("schema")
# Ignore comments in genfile
genfile.ignore(comment)
return genfile
def buildSyntax(self):
'''WIP'''
dreamScript = pyp.Forward()
# Constants
singlelineString = pyp.QuotedString('"', '\\').setResultsName('string').setParseAction(self.makeString)
fileRef = pyp.QuotedString("'", '\\').setResultsName('fileRef').setParseAction(self.makeFileRef)
multilineString = pyp.QuotedString(quoteChar='{"', endQuoteChar='"}', multiline=True).setResultsName('string').setParseAction(self.makeString)
number = pyp.Regex(r'\d+(\.\d*)?([eE]\d+)?').setResultsName('number').setParseAction(self.makeNumber)
# Other symbols
listStart = pyp.Suppress('list(')
listEnd = pyp.Suppress(')')
VAR_GLOBAL = pyp.Keyword('global')
VAR_CONST = pyp.Keyword('const')
SLASH = pyp.Literal('/')
EQUAL = pyp.Literal('=')
VAR = pyp.Keyword('var')
#############################
# Grammar
#############################
constant = singlelineString | fileRef | multilineString | number | dreamList
# Lists
listElement = constant | (constant + EQUAL + constant)
listElement = pyp.operatorPrecedence(listElement, [
("=", 2, pyp.opAssoc.LEFT,),
])
listContents = pyp.delimitedList(listElement)
dreamList << pyp.Group(listStart + listContents + listEnd)
dreamList.setParseAction(self.handleList)
# Paths
relpath = pyp.ident | relpath + SLASH + pyp.ident
abspath = SLASH + relpath
path = (abspath | relpath).setParseAction(self.handlePath)
pathslash = path + SLASH
# Preprocessor stuff
ppStatement = pyp.Forward()
ppDefine = pyp.Keyword('#define') + pyp.ident.setResultsName('name') + pyp.restOfLine.setResultsName('value')
ppDefine.setParseAction(self.handlePPDefine)
ppUndef = pyp.Keyword('#undef') + pyp.ident.setResultsName('name')
ppUndef.setParseAction(self.handlePPUndef)
ppIfdef = (pyp.Keyword('#ifdef') + pyp.ident.setResultsName('name')).setParseAction(self.handlePPIfdef)
ppIfndef = (pyp.Keyword('#ifndef') + pyp.ident.setResultsName('name')).setParseAction(self.handlePPIfndef)
ppElse = (pyp.Keyword('#else') + pyp.ident.setResultsName('name')).setParseAction(self.handlePPElse)
ppEndif = pyp.Keyword('#endif').setParseAction(self.handlePPElse)
ppStatement = pyp.lineStart + (ppIfdef | ppIfndef | ppElse | ppEndif)
# Var Declarations
##########################
var_modifiers = pyp.ZeroOrMore(SLASH + (VAR_GLOBAL | VAR_CONST)).setResultsName('modifiers')
var_assignment = EQUAL + constant
varblock_inner_ref = pyp.ident.setResultsName('name') + pyp.Optional(var_assignment)
var_argument = VAR + pyp.Optional(abspath) + SLASH + pyp.ident.setResultsName('name') + pyp.Optional(var_assignment)
varblock_inner_decl = var_modifiers + pyp.Optional(abspath) + SLASH + varblock_inner_ref
varblock_element = varblock_inner_decl | varblock_inner_ref
varblock = VAR + pyp.indentedBlock(var_blockinner_decl)
inline_vardecl = VAR + varblock_inner_decl
vardecl = varblock | inline_vardecl
# Proc Declarations
PROC = pyp.Keyword('proc')
proc_args = '(' + pyp.delimitedList(var_argument | pyp.ident.setResultsName('name')) + ')'
procblock_proc = pyp.ident.setResultsName('name') + proc_args + pyp.indentedBlock(proc_instructions)
procblock = PROC + pyp.indentedBlock(procblock_proc, [1])
# Atom blocks
atomdecl = pyp.Forward()
atomdecl << path + pyp.indentedBlock(vardecl | atomdecl | procdecl , [1])
return dreamScript
import pyparsing as pp
version = pp.__version__
W = pp.Keyword
S = pp.Suppress
SW = lambda kw: S(W(kw))
Group = pp.Group # pylint: disable=invalid-name
Optional = pp.Optional # pylint: disable=invalid-name
LineNum = pp.lineno # pylint: disable=invalid-name
# Grammar for Python Type Declaration (CLIF version)
_indentation_stack = [1] # Holds cols to indent from.
BLOCK = lambda el: S(':') + NEWLINE - pp.indentedBlock(el, _indentation_stack)
def reset_indentation(): # pylint: disable=invalid-name
_indentation_stack[:] = [1]
K = lambda el: pp.Keyword(el).setParseAction(lambda el, loc, t: [t[0], loc])
E = lambda el: pp.Empty().setParseAction(lambda: el)
Tag = lambda el: pp.Empty().setParseAction(lambda s, loc, t: [el, loc]) # pylint: disable=invalid-name
NEWLINE = pp.lineEnd.setWhitespaceChars(' ').suppress().ignore(
pp.pythonStyleComment)
def PARENS(el): # pylint: disable=invalid-name
el.ignore(pp.pythonStyleComment)
return S('(') - el - S(')')
banner_motd = "banner motd " + word_param("type") + restOfLine("path")
line_ip_address = "ip address" + ipAddressWithMask("ip")
line_description = "description" + restOfLine("description")
line_ip_ospf_cost = "ip ospf cost" + integer("cost")
interface_properties = OneOrMore(
line_ip_address("ip address") |
line_description("description") |
line_ip_ospf_cost("ospf cost") |
comment|
hash_comment
)
interface_indent = indentedBlock(interface_properties, indentStack, True)("indent")
interface_id = "interface" + interface_id("id")
interface = ( interface_id + Optional(interface_indent)).setParseAction(fn_interface)
interfaces = OneOrMore(interface.setResultsName("interface", listAllMatches=True))
#### BGP
#TODO: put each protocol stanza into a function - variable scope, etc
router_bgp = "router bgp" + integer("asn")
#TODO: can router-id be an integer too?
#false_no: returns False if "no" is present, otherwise returns True
false_no = Optional(Literal("no").setParseAction(lambda x: False), default = True)
#bgp_synchronization = Optional(false_no, default =True) + "synchronization"
