def build_index_parser():
left = pp.Literal('(').suppress()
right = pp.Literal(')').suppress()
unique = pp.Optional(pp.CaselessKeyword('unique')).setResultsName('unique')
index = pp.CaselessKeyword('index').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
return unique + index + left + pp.delimitedList(attribute_name).setResultsName('attr_list') + right
def __init__(self):
self._added_operator_specs = []
self._added_function_specs = {}
self._base_operators = ("** * / mod × ÷ + - < > <= >= == != ≠ ≤ ≥ between-and within-and"
" in-range-from-to not and ∧ or ∨ ?:").split()
self._base_function_map = {
'sgn': FunctionSpec((lambda x: -1 if x < 0 else 1 if x > 0 else 0), 1),
'abs': FunctionSpec(abs, 1),
'round': FunctionSpec(round, 2),
'trunc': FunctionSpec(math.trunc, 1),
'ceil': FunctionSpec(math.ceil, 1),
'floor': FunctionSpec(math.floor, 1),
'min': FunctionSpec(min, 2),
'max': FunctionSpec(max, 2),
'str': FunctionSpec(lambda x: str(x), 1),
}
# epsilon for computing "close" floating point values - can be updated in customize
self.epsilon = 1e-15
# customize can update or replace with different characters
self.ident_letters = ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyzªº"
"ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ"
+ pp.srange("[Α-Ω]") + pp.srange("[α-ω]"))
# storage for assigned variables
self._variable_map = {}
# customize can add pre-defined constants
self._initial_variables = {}
self.customize()
self._parser = self.make_parser()
def parse_expr(s):
LPAR = pyp.Literal('(').suppress()
RPAR = pyp.Literal(')').suppress()
quote = pyp.Literal('"').suppress()
sp = pyp.OneOrMore(pyp.White()).suppress()
sps = pyp.ZeroOrMore(pyp.White()).suppress()
nums = pyp.Word(pyp.srange("[0-9]"))
num = (nums + pyp.Optional(pyp.Literal('.') + nums))\
.setParseAction(lambda toks: Num(''.join(toks)))
var = (pyp.Word(pyp.alphas.lower(), pyp.srange("[a-zA-Z0-9]")) + pyp.Optional(pyp.Literal("'"))).\
setParseAction(lambda toks: Var(toks[0], toks[1] if len(toks) > 1 else None))
chars = (pyp.QuotedString('"')).setParseAction(lambda toks: Charstr(toks[0]))
boolean = (pyp.oneOf("True False true false")).setParseAction(lambda toks: Bool(toks[0]))
term = pyp.Forward()
pterm = (LPAR + sps + term + sps + RPAR).setParseAction(lambda toks: toks[0])
term << pyp.infixNotation(num | var | pterm | chars | boolean, [
(pyp.oneOf("+ -"), 2, pyp.opAssoc.LEFT, lambda ts: BinOp(ts[0][0], ts[0][1], ts[0][2])),
])
formula = pyp.Forward()
cmpop = pyp.oneOf("== < > <= >= !=")
atom = (sps + term + sps + cmpop + sps + term + sps).\
setParseAction(lambda toks: Cmp(toks[1], toks[0], toks[2]))
patom = (LPAR + sps + atom + sps + RPAR).setParseAction(lambda toks: toks[0])
formula << pyp.infixNotation(patom, [
(pyp.oneOf("&& ||"), 2, pyp.opAssoc.LEFT, lambda ts: BinCon(ts[0][0], ts[0][1], ts[0][2])),
])
res = formula.parseString(s)
r = res[0] if len(res) > 0 else None
return r
def build_index_parser():
left = pp.Literal('(').suppress()
right = pp.Literal(')').suppress()
unique = pp.Optional(pp.CaselessKeyword('unique')).setResultsName('unique')
index = pp.CaselessKeyword('index').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
return unique + index + left + pp.delimitedList(attribute_name).setResultsName('attr_list') + right
def build_attribute_parser():
quoted = pp.Or(pp.QuotedString('"'), pp.QuotedString("'"))
colon = pp.Literal(':').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]')).setResultsName('name')
data_type = pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted)).setResultsName('type')
default = pp.Literal('=').suppress() + pp.SkipTo(colon, ignore=quoted).setResultsName('default')
comment = pp.Literal('#').suppress() + pp.restOfLine.setResultsName('comment')
return attribute_name + pp.Optional(default) + colon + data_type + comment
def build_attribute_parser():
quoted = pp.Or(pp.QuotedString('"'), pp.QuotedString("'"))
colon = pp.Literal(':').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]')).setResultsName('name')
data_type = pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted)).setResultsName('type')
default = pp.Literal('=').suppress() + pp.SkipTo(colon, ignore=quoted).setResultsName('default')
comment = pp.Literal('#').suppress() + pp.restOfLine.setResultsName('comment')
return attribute_name + pp.Optional(default) + colon + data_type + comment
class CTLGrammar:
left = Literal('(').suppress()
right = Literal(')').suppress()
comma = Literal(',').suppress()
_in = Literal('in')
leftCurly = Literal('{').suppress()
rightCurly = Literal('}').suppress()
leftSquare = Literal('[').suppress()
rightSquare = Literal(']').suppress()
neg = Literal('not')
ax = Literal('AX')
ex = Literal('EX')
af = Literal('AF')
ef = Literal('EF')
ag = Literal('AG')
ar = Literal('AR')
eg = Literal('EG')
conj = Literal('and')
disj = Literal('or')
impl = Literal('=>')
au = Literal('AU')
eu = Literal('EU')
true = Literal('true')
false = Literal('false')
word = Word(srange("[A-Za-z0-9_]"))
nums = Word(srange("[0-9]"))
unaryPropositionalOperator = neg
binaryPropositionalOperator = conj | disj | impl
unaryCTLOperator = neg | ax | ex | af | ef | ag | eg
binaryCTLOperator = conj | disj | impl | au | eu | ar
atomic = word | Group(left + word + _in + leftCurly +
Group(OneOrMore(word)) + rightCurly +
right) | Group(left + word + _in + leftSquare +
Group(nums + comma + nums) +
rightSquare + right)
constranit = Forward()
constranit << Or([
true, false, atomic,
Group(left + unaryPropositionalOperator + constranit + right),
Group(left + binaryPropositionalOperator + constranit +
OneOrMore(constranit) + right)
])
ctlFormula = Forward()
ctlFormula << Or([
atomic,
Group(left + unaryCTLOperator + ctlFormula + right),
Group(left + binaryCTLOperator + ctlFormula + ctlFormula + right)
])
req = Group(left + constranit + comma + ctlFormula + right)
def build_foreign_key_parser():
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
new_attributes = pp.Optional(pp.delimitedList(attribute_name)).setResultsName('new_attributes')
arrow = pp.Literal('->').suppress()
ref_table = pp.Word(pp.alphas, pp.alphanums + '._').setResultsName('ref_table')
left = pp.Literal('(').suppress()
right = pp.Literal(')').suppress()
ref_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('ref_attrs')
return new_attributes + arrow + ref_table + ref_attrs
def __init__(self, logger):
self.salt_client = salt.client.LocalClient()
self.etcd = Etcd(logger)
self.logger = logger
# Parse out the username and formation name
# from the ETCD directory string
self.formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
def build_index_parser():
left = pp.Literal("(").suppress()
right = pp.Literal(")").suppress()
unique = pp.Optional(pp.CaselessKeyword("unique")).setResultsName("unique")
index = pp.CaselessKeyword("index").suppress()
attribute_name = pp.Word(pp.srange("[a-z]"), pp.srange("[a-z0-9_]"))
return (unique + index + left +
pp.delimitedList(attribute_name).setResultsName("attr_list") +
right)
def __init__(self, logger):
self.salt_client = salt.client.LocalClient()
self.etcd = Etcd(logger)
self.logger = logger
# Parse out the username and formation name
# from the ETCD directory string
self.formation_parser = Literal('/formations/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('username') + Literal('/') + \
Word(srange("[0-9a-zA-Z_-]")).setResultsName('formation_name')
def handleMacro(toks):
macroChar = toks[0][1]
if macroChar == "d":
return CharSetRandomizer(srange("[0-9]"))
elif macroChar == "w":
return CharSetRandomizer(srange("[A-Za-z0-9_]"))
elif macroChar == "s":
return CharSetRandomizer(" ")
else:
raise ParseFatalException("", 0, "unsupported macro character (" + macroChar + ")")
def handleMacro(toks):
macroChar = toks[0][1]
if macroChar == "d":
return CharSetRandomizer(srange("[0-9]"))
elif macroChar == "w":
return CharSetRandomizer(srange("[A-Za-z0-9_]"))
elif macroChar == "s":
return CharSetRandomizer(" ")
else:
raise ParseFatalException(
"", 0, "unsupported macro character (" + macroChar + ")")
def grammer():
lparen = pp.Suppress("(")
rparen = pp.Suppress(")")
equal = pp.Suppress("=")
nl = pp.Suppress(pp.LineEnd())
reg = pp.Combine("$" + pp.Optional("cr") +
pp.Word(pp.srange("[0-7]"), max=1))
num = pp.Word(pp.srange("[0-9]")).setParseAction(lambda s, l, t: int(t[0]))
val = pp.Word(
pp.srange("[0-9a-fA-F]")).setParseAction(lambda s, l, t: int(t[0], 16))
values = pp.Dict(pp.OneOrMore(pp.Group(reg + equal + val)))
return num + lparen + values + rparen + nl
def build_foreign_key_parser():
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
new_attributes = pp.Optional(
pp.delimitedList(attribute_name)).setResultsName('new_attributes')
arrow = pp.Literal('->').suppress()
ref_table = pp.Word(pp.alphas,
pp.alphanums + '._').setResultsName('ref_table')
left = pp.Literal('(').suppress()
right = pp.Literal(')').suppress()
ref_attrs = pp.Optional(left + pp.delimitedList(attribute_name) +
right).setResultsName('ref_attrs')
return new_attributes + arrow + ref_table + ref_attrs
def compile_attribute(line, in_key=False):
"""
Convert attribute definition from DataJoint format to SQL
:param line: attribution line
:param in_key: set to True if attribute is in primary key set
:returns: (name, sql) -- attribute name and sql code for its declaration
"""
quoted = pp.Or(pp.QuotedString('"'), pp.QuotedString("'"))
colon = pp.Literal(':').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'),
pp.srange('[a-z0-9_]')).setResultsName('name')
data_type = pp.Combine(pp.Word(pp.alphas) +
pp.SkipTo("#", ignore=quoted)).setResultsName(
'type')
default = pp.Literal('=').suppress() + pp.SkipTo(
colon, ignore=quoted).setResultsName('default')
comment = pp.Literal('#').suppress() + pp.restOfLine.setResultsName(
'comment')
attribute_parser = attribute_name + pp.Optional(
default) + colon + data_type + comment
match = attribute_parser.parseString(line + '#', parseAll=True)
match['comment'] = match['comment'].rstrip('#')
if 'default' not in match:
match['default'] = ''
match = {k: v.strip() for k, v in match.items()}
match['nullable'] = match['default'].lower() == 'null'
literals = ['CURRENT_TIMESTAMP'] # not to be enclosed in quotes
if match['nullable']:
if in_key:
raise DataJointError(
'Primary key attributes cannot be nullable in line %s' % line)
match[
'default'] = 'DEFAULT NULL' # nullable attributes default to null
else:
if match['default']:
quote = match['default'].upper(
) not in literals and match['default'][0] not in '"\''
match['default'] = ('NOT NULL DEFAULT ' +
('"%s"' if quote else "%s") % match['default'])
else:
match['default'] = 'NOT NULL'
match['comment'] = match['comment'].replace(
'"', '\\"') # escape double quotes in comment
sql = ('`{name}` {type} {default}' +
(' COMMENT "{comment}"' if match['comment'] else '')).format(
**match)
return match['name'], sql
def build_foreign_key_parser():
left = pp.Literal('(').suppress()
right = pp.Literal(')').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
new_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('new_attrs')
arrow = pp.Literal('->').suppress()
lbracket = pp.Literal('[').suppress()
rbracket = pp.Literal(']').suppress()
option = pp.Word(pp.srange('[a-zA-Z]'))
options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket)
ref_table = pp.Word(pp.alphas, pp.alphanums + '._').setResultsName('ref_table')
ref_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('ref_attrs')
return new_attrs + arrow + options + ref_table + ref_attrs
def grammer():
lparen = pp.Suppress("(")
rparen = pp.Suppress(")")
equal = pp.Suppress("=")
nl = pp.Suppress(pp.LineEnd())
reg = pp.Combine("$" + pp.Optional("cr") + pp.Word(pp.srange("[0-7]"),
max=1))
num = pp.Word(pp.srange("[0-9]")).setParseAction(
lambda s, l, t: int(t[0]))
val = pp.Word(pp.srange("[0-9a-fA-F]")).setParseAction(
lambda s, l, t: int(t[0], 16))
values = pp.Dict(pp.OneOrMore(pp.Group(reg + equal + val)))
return num + lparen + values + rparen + nl
def build_attribute_parser():
quoted = pp.QuotedString('"') ^ pp.QuotedString("'")
colon = pp.Literal(":").suppress()
attribute_name = pp.Word(pp.srange("[a-z]"),
pp.srange("[a-z0-9_]")).setResultsName("name")
data_type = (
pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted))
^ pp.QuotedString("<", endQuoteChar=">",
unquoteResults=False)).setResultsName("type")
default = pp.Literal("=").suppress() + pp.SkipTo(
colon, ignore=quoted).setResultsName("default")
comment = pp.Literal("#").suppress() + pp.restOfLine.setResultsName(
"comment")
return attribute_name + pp.Optional(default) + colon + data_type + comment
def build_foreign_key_parser_old():
# old-style foreign key parser. Superceded by expression-based syntax. See issue #436
# This will be deprecated in a future release.
left = pp.Literal('(').suppress()
right = pp.Literal(')').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
new_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('new_attrs')
arrow = pp.Literal('->').suppress()
lbracket = pp.Literal('[').suppress()
rbracket = pp.Literal(']').suppress()
option = pp.Word(pp.srange('[a-zA-Z]'))
options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket).setResultsName('options')
ref_table = pp.Word(pp.alphas, pp.alphanums + '._').setResultsName('ref_table')
ref_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('ref_attrs')
return new_attrs + arrow + options + ref_table + ref_attrs
def build_foreign_key_parser_old():
# old-style foreign key parser. Superceded by expression-based syntax. See issue #436
# This will be deprecated in a future release.
left = pp.Literal('(').suppress()
right = pp.Literal(')').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
new_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('new_attrs')
arrow = pp.Literal('->').suppress()
lbracket = pp.Literal('[').suppress()
rbracket = pp.Literal(']').suppress()
option = pp.Word(pp.srange('[a-zA-Z]'))
options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket).setResultsName('options')
ref_table = pp.Word(pp.alphas, pp.alphanums + '._').setResultsName('ref_table')
ref_attrs = pp.Optional(left + pp.delimitedList(attribute_name) + right).setResultsName('ref_attrs')
return new_attrs + arrow + options + ref_table + ref_attrs
def build_riscv_panic_output_parser(): # type: () -> typing.Type[ParserElement]
"""Builds a parser for the panic handler output using pyparsing"""
# We don't match the first line, since "Guru Meditation" will not be printed in case of an abort:
# Guru Meditation Error: Core 0 panic'ed (Store access fault). Exception was unhandled.
# Core 0 register dump:
reg_dump_header = Group(Literal('Core') +
Word(nums)('core_id') +
Literal('register dump:'))('reg_dump_header')
# MEPC : 0x4200232c RA : 0x42009694 SP : 0x3fc93a80 GP : 0x3fc8b320
reg_name = Word(srange('[A-Z_0-9/-]'))('name')
hexnumber_with_0x = Combine(Literal('0x') + Word(hexnumber))
reg_value = hexnumber_with_0x('value')
reg_dump_one_reg = Group(reg_name + Literal(':') + reg_value) # not named because there will be OneOrMore
reg_dump_all_regs = Group(OneOrMore(reg_dump_one_reg))('regs')
reg_dump = Group(reg_dump_header + reg_dump_all_regs) # not named because there will be OneOrMore
reg_dumps = Group(OneOrMore(reg_dump))('reg_dumps')
# Stack memory:
# 3fc93a80: 0x00000030 0x00000021 0x3fc8aedc 0x4200232a 0xa5a5a5a5 0xa5a5a5a5 0x3fc8aedc 0x420099b0
stack_line = Group(Word(hexnumber)('base') + Literal(':') +
Group(OneOrMore(hexnumber_with_0x))('data'))
stack_dump = Group(Literal('Stack memory:') +
Group(OneOrMore(stack_line))('lines'))('stack_dump')
# Parser for the complete panic output:
panic_output = reg_dumps + stack_dump
return panic_output
def parse_commanders(x):
# Grammar
LBRAK, RBRAK, COMMA = (pp.Literal(x).suppress() for x in "[],")
AND = pp.Literal("and").suppress()
and_ = pp.Optional(COMMA) + AND
RANK = pp.Or(pp.Literal(x) for x in RANKS_LOOKUP.keys())("rank")
NAME = pp.Word(pp.srange("[A-Z]"), pp.alphas + pp.alphas8bit, min=2)
ABBR = pp.Regex(r"([A-Z]\.)+")
SUFFIX = pp.Literal("Jr.")
NOCMDR = (pp.Literal("None"))("none")
COMBATANT = pp.oneOf(' '.join(('US', 'CS', 'I')))("combatant")
nametoken = ABBR | pp.quotedString | NAME
name = (pp.OneOrMore(nametoken) + pp.Optional(pp.Literal(",") + SUFFIX))("fullname")
name.addParseAction(lambda s,l,t: ' '.join(t))
cmdrname = RANK + name
commander = pp.Group(cmdrname | NOCMDR)
commander_list = pp.Group(pp.delimitedList(commander, ",") +
pp.Optional(and_ + commander))("commanders")
milforce = pp.Group(commander_list + LBRAK + COMBATANT + RBRAK)
grammar = pp.delimitedList(milforce, ";")
toks = grammar.parseString(x)
# A smarter grammar could probably have avoided this
res = {}
for _force in toks:
k = _force['combatant']
res[k] = [x.asDict() for x in _force['commanders']
if "none" not in x.asDict()]
return res
def build_foreign_key_parser():
left = pp.Literal('(').suppress()
right = pp.Literal(')').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
new_attrs = pp.Optional(left + pp.delimitedList(attribute_name) +
right).setResultsName('new_attrs')
arrow = pp.Literal('->').suppress()
lbracket = pp.Literal('[').suppress()
rbracket = pp.Literal(']').suppress()
option = pp.Word(pp.srange('[a-zA-Z]'))
options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket)
ref_table = pp.Word(pp.alphas,
pp.alphanums + '._').setResultsName('ref_table')
ref_attrs = pp.Optional(left + pp.delimitedList(attribute_name) +
right).setResultsName('ref_attrs')
return new_attrs + arrow + options + ref_table + ref_attrs
def build_grammar(self):
forbidden_keywords = ~pp.Literal("ONBUILD") + ~pp.Literal(
"MAINTAINER") + ~pp.Literal("FROM")
keyword = (forbidden_keywords + pp.Word(
pp.srange("[A-Z]")).setResultsName("onbuild").setName("keyword"))
grammar = keyword + pp.OneOrMore(
pp.White()) + pp.restOfLine.setResultsName("onbuild_argument")
try:
result = grammar.parseString(self.argument, parseAll=True)
onbuild_command = result.get("onbuild", None)
onbuild_argument = result.get("onbuild_argument", "")
if onbuild_command:
instruction_class = None
try:
instruction_class = get_class_from_keyword(onbuild_command)
except KeyError:
self.append_syntax_error(self.argument,
hint="Instruction of type " +
onbuild_command +
" does not exist.")
if instruction_class:
instruction = instruction_class(onbuild_argument,
self.physical_line_number)
foreign_grammar = instruction.build_grammar()
grammar = keyword + foreign_grammar
except pp.ParseException:
# This will be reraised again by the instruction base anyway
pass
return grammar
def grammar():
"""Define the query grammar.
Backus-Naur form (BNF) of the grammar::
<grammar> ::= "*" | <items>
<items> ::= <item> | <item> <whitespace> <items>
<item> ::= <key>:<value>
Given that the pyparsing library defines the grammar in a BNF-like style, for the details of the tokens not
specified above check directly the source code.
Returns:
pyparsing.ParserElement: the grammar parser.
"""
quoted_string = pp.quotedString.copy().addParseAction(
pp.removeQuotes) # Both single and double quotes are allowed
# Key-value tokens: key:value
# Lowercase key, all printable characters except the parentheses that are part of the global grammar for the value
key = pp.Word(pp.srange('[a-z0-9-_.]"'), min=2)('key')
all_but_par = ''.join(
[c for c in pp.printables if c not in ('(', ')', '{', '}')])
value = (quoted_string | pp.Word(all_but_par))('value')
item = pp.Combine(key + ':' + value)
# Final grammar, see the docstring for its BNF based on the tokens defined above
# Groups are used to split the parsed results for an easy access
return pp.Group(pp.Literal('*')('all')) | pp.OneOrMore(pp.Group(item))
def grammar():
dneg = pp.Literal('not ')
name = pp.Combine(
pp.Literal('-') * (0, 1) + pp.Word(pp.srange('[a-z_]')) +
pp.Word(pp.alphanums + '_') * (0, 1))
lpar = pp.Suppress('(')
rpar = pp.Suppress(')')
sepr = pp.Suppress(';') | pp.Suppress(',')
num = pp.Word(pp.nums)
# arg_body = pp.nestedExpr(opener='(', closer=')', content=pp.Word(pp.alphanums)).setResultsName('arg_body')
arg_body = pp.Forward()
literal = pp.Forward()
func = pp.Forward()
term = pp.Forward()
term <<= (func | num)
arg_body <<= pp.Group(lpar + ((term + (sepr + term) *
(0, None)) | pp.empty()) + rpar)
func <<= pp.Group(name + arg_body * (0, 1))
literal <<= (dneg * (0, 1) + func)
#Semantic Actions
func.setParseAction(PA_Functionc)
num.setParseAction(PA_Num)
literal.setParseAction(PA_Literal)
return literal
def _construct_grammar():
logical_operator = get_logical_operator()
logical_expression = get_logical_expression()
facets_expression = get_facet_expression()
highlight_expression = get_highlight_expression()
sort_expression = get_sort_expression()
aggs_expression = get_aggregations_expression()
nested_expression = get_nested_expression()
# The below line describes how the type expression should be.
type_expression = Word('type')\
+ Word(':').suppress()\
+ Word(srange("[a-zA-Z0-9_]"))\
+ Optional(CaselessLiteral('AND')).suppress()
type_expression.setParseAction(parse_type_expression)
base_expression = Optional(highlight_expression)\
+ Optional(sort_expression)\
+ Optional(type_expression)\
+ ZeroOrMore(
(facets_expression
| aggs_expression
| nested_expression
| logical_expression)
+ Optional(logical_operator)
).setParseAction(parse_one_or_more_logical_expressions)
base_expression.setParseAction(parse_type_logical_facets_expression)
return base_expression
def __init__(self):
lpar = pp.Literal("(").suppress()
rpar = pp.Literal(")").suppress()
element = pp.Combine(
pp.Word(pp.srange("[A-Z]"), exact=1) +
pp.Optional(pp.Word(pp.srange("[a-z]"), max=1)))
integer = pp.Word(pp.nums)
point = pp.Literal(".")
fnumber = pp.Combine(integer + pp.Optional(
point + pp.Optional(integer))) | pp.Combine(point + integer)
self.formula = pp.Forward()
atom = element | pp.Group(lpar + self.formula + rpar)
self.formula << pp.OneOrMore(
pp.Group(atom + pp.Optional(fnumber, default="1")))
self.elements = {}
def build_foreign_key_parser():
arrow = pp.Literal('->').suppress()
lbracket = pp.Literal('[').suppress()
rbracket = pp.Literal(']').suppress()
option = pp.Word(pp.srange('[a-zA-Z]'))
options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket).setResultsName('options')
ref_table = pp.restOfLine.setResultsName('ref_table')
return arrow + options + ref_table
def build_foreign_key_parser():
arrow = pp.Literal('->').suppress()
lbracket = pp.Literal('[').suppress()
rbracket = pp.Literal(']').suppress()
option = pp.Word(pp.srange('[a-zA-Z]'))
options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket).setResultsName('options')
ref_table = pp.restOfLine.setResultsName('ref_table')
return arrow + options + ref_table
def Syntax():
op = oneOf('+ - / *')
lpar = Literal('(').suppress()
rpar = Literal(')').suppress()
num = Word(srange('[1-9]'), nums)
expr = Forward()
atom = num | Group(lpar + expr + rpar)
expr << atom + ZeroOrMore(op + expr)
return expr
def Syntax():
op = oneOf( '+ - / *')
lpar = Literal('(') .suppress()
rpar = Literal( ')' ).suppress()
num = Word(srange('[1-9]'),nums)
expr = Forward()
atom = num | Group( lpar + expr + rpar )
expr << atom + ZeroOrMore( op + expr )
return expr
def build_foreign_key_parser():
arrow = pp.Literal("->").suppress()
lbracket = pp.Literal("[").suppress()
rbracket = pp.Literal("]").suppress()
option = pp.Word(pp.srange("[a-zA-Z]"))
options = pp.Optional(lbracket + pp.delimitedList(option) +
rbracket).setResultsName("options")
ref_table = pp.restOfLine.setResultsName("ref_table")
return arrow + options + ref_table
def __infer_precedence(user_query: str) -> str:
# Put square brackets that indicate precedence of operations.
extra_chars = pp.srange(r"[\0x80-\0x7FF]") # Support Cyrillic letters.
tag = pp.Word(pp.alphas + '_' + extra_chars)
parser = pp.operatorPrecedence(tag, [("NOT", 1, pp.opAssoc.RIGHT),
("AND", 2, pp.opAssoc.LEFT),
("OR", 2, pp.opAssoc.LEFT)])
parsed_expression = parser.parseString(user_query)[0]
return str(parsed_expression)
def _query(self):
## Utility
ident = pp.Word(pp.srange("[a-zA-Z0-9_.+-]"))
## Terms
ident_term = pp.Word(
pp.srange("[a-zA-Z0-9_.+-]")).setParseAction(make_IdentTerm)
quoted_term = ('"' + pp.OneOrMore(ident) +
'"').setParseAction(make_QuotedTerm)
not_term = ('-' +
(ident_term | quoted_term)).setParseAction(make_NotTerm)
disj_term = pp.Forward()
disj_term = (
ident +
pp.OneOrMore(pp.CaselessKeyword('or') +
(disj_term | ident))).setParseAction(make_DisjTerm)
term = quoted_term | disj_term | not_term | ident_term
## Selector base
basic_selector = pp.Forward()
selector_start = ident + ':'
## Selector bodies only
selector_body = pp.Forward()
list_selector_body = '[' + pp.delimitedList(selector_body, ',') + ']'
source_selector_body = ident + pp.Optional('/' + ident)
generic_selector_body = ident
selector_body << (list_selector_body
| source_selector_body
| generic_selector_body)
## Full selectors
list_selector = selector_start + list_selector_body
source_selector = pp.CaselessKeyword(
'source') + ':' + source_selector_body
generic_selector = selector_start + generic_selector_body
basic_selector << (
list_selector.setParseAction(make_ListSelector) \
| source_selector.setParseAction(make_source) \
| generic_selector.setParseAction(make_GenericSelector))
not_selector = ('!' + basic_selector).setParseAction(make_NotSelector)
selector = not_selector | basic_selector
query = pp.ZeroOrMore(selector | term).setParseAction(make_Query)
return query.parseString(self.query)
def get_highlight_expression():
field_expression = Word(srange("[a-zA-Z0-9_.*]"))
field_expression.setParseAction(parse_highlight_field_expression)
fields_expression = OneOrMore(
field_expression + Optional(',').suppress())
fields_expression.setParseAction(parse_highlight_expression)
highlight_expression = Word('highlight:').suppress() \
+ Word('[').suppress() \
+ fields_expression + Word(']').suppress()
return highlight_expression
def handle_macro(toks):
macroChar = toks[0][1]
if macroChar == "d":
return CharacterRangeEmitter("0123456789")
elif macroChar == "w":
return CharacterRangeEmitter(srange("[A-Za-z0-9_]"))
elif macroChar == "s":
return LiteralEmitter(" ")
else:
raise ParseFatalException("", 0, "unsupported macro character (" + macroChar + ")")
def build_foreign_key_parser_old():
# old-style foreign key parser. Superseded by expression-based syntax. See issue #436
# This will be deprecated in a future release.
left = pp.Literal("(").suppress()
right = pp.Literal(")").suppress()
attribute_name = pp.Word(pp.srange("[a-z]"), pp.srange("[a-z0-9_]"))
new_attrs = pp.Optional(left + pp.delimitedList(attribute_name) +
right).setResultsName("new_attrs")
arrow = pp.Literal("->").suppress()
lbracket = pp.Literal("[").suppress()
rbracket = pp.Literal("]").suppress()
option = pp.Word(pp.srange("[a-zA-Z]"))
options = pp.Optional(lbracket + pp.delimitedList(option) +
rbracket).setResultsName("options")
ref_table = pp.Word(pp.alphas,
pp.alphanums + "._").setResultsName("ref_table")
ref_attrs = pp.Optional(left + pp.delimitedList(attribute_name) +
right).setResultsName("ref_attrs")
return new_attrs + arrow + options + ref_table + ref_attrs
def handleMacro(toks):
macroChar = toks[0][1]
if macroChar == "d":
return CharacterRangeEmitter("0123456789")
elif macroChar == "w":
return CharacterRangeEmitter(srange("[A-Za-z0-9_]"))
elif macroChar == "s":
return LiteralEmitter(" ")
else:
raise ParseFatalException("",0,"unsupported macro character (" + macroChar + ")")
def _query(self):
## Utility
ident = pp.Word(pp.srange("[a-zA-Z0-9_.+-]"))
## Terms
ident_term = pp.Word(pp.srange("[a-zA-Z0-9_.+-]")).setParseAction(
make_IdentTerm)
quoted_term = ('"' + pp.OneOrMore(ident) + '"').setParseAction(
make_QuotedTerm)
not_term = ('-' + (ident_term | quoted_term)).setParseAction(
make_NotTerm)
disj_term = pp.Forward()
disj_term = (ident + pp.OneOrMore(pp.CaselessKeyword('or') + (disj_term | ident))).setParseAction(make_DisjTerm)
term = quoted_term | disj_term | not_term | ident_term
## Selector base
basic_selector = pp.Forward()
selector_start = ident + ':'
## Selector bodies only
selector_body = pp.Forward()
list_selector_body = '[' + pp.delimitedList(selector_body, ',') + ']'
source_selector_body = ident + pp.Optional('/' + ident)
generic_selector_body = ident
selector_body << (
list_selector_body
| source_selector_body
| generic_selector_body)
## Full selectors
list_selector = selector_start + list_selector_body
source_selector = pp.CaselessKeyword('source') + ':' + source_selector_body
generic_selector = selector_start + generic_selector_body
basic_selector << (
list_selector.setParseAction(make_ListSelector) \
| source_selector.setParseAction(make_source) \
| generic_selector.setParseAction(make_GenericSelector))
not_selector = ('!' + basic_selector).setParseAction(make_NotSelector)
selector = not_selector | basic_selector
query = pp.ZeroOrMore(selector | term).setParseAction(
make_Query)
return query.parseString(self.query)
def _build():
"""Encapsulate so the variables don't leak out."""
# Basic punctuation
colon = Literal(':').suppress()
hashmark = Literal('#').suppress()
comment = (hashmark + restOfLine).suppress()
# Enforce Python-style naming conventions
command_name = Word(srange("[A-Z]"), srange("[a-zA-Z0-9]")) # StudlyCaps
field_name = Word(srange("[a-z_]"), srange("[a-z0-9_]")) # lower_underscore
# Put it all together
fields = Dict(OneOrMore(Group(field_name + colon + restOfLine)))
fields = nestedExpr(opener="{", closer="}", content=fields)
command = Group(command_name + fields)
# Configure the parser
tml_parser = OneOrMore(command) + stringEnd
tml_parser.ignore(comment)
return tml_parser
def check_if_function(code):
return_type = Word(alphanums + '_[]') # Bad style to have "_" but syntactically valid
function_name = Word(alphanums + '_:')
args = Word(alphanums + ',_[]&* ')
function_open = Literal("{")
function_close = Literal("}")
function_declaration = Optional(srange("[a-z]")) + return_type + function_name + "(" + Optional(args) + ")"
grammar = function_declaration + Optional(function_open)
if len(grammar.searchString(code)):
return True
return False
def _build():
"""Encapsulate so the variables don't leak out."""
# Basic punctuation
colon = Literal(':').suppress()
hashmark = Literal('#').suppress()
comment = (hashmark + restOfLine).suppress()
# Enforce Python-style naming conventions
command_name = Word(srange("[A-Z]"), srange("[a-zA-Z0-9]")) # StudlyCaps
field_name = Word(srange("[a-z_]"),
srange("[a-z0-9_]")) # lower_underscore
# Put it all together
fields = Dict(OneOrMore(Group(field_name + colon + restOfLine)))
fields = nestedExpr(opener="{", closer="}", content=fields)
command = Group(command_name + fields)
# Configure the parser
tml_parser = OneOrMore(command) + stringEnd
tml_parser.ignore(comment)
return tml_parser
def _create_tokenizer(self):
alphabet_lower = 'абвгдеёжзийклмнопрстуфхцчшщъыьэюя'
alphabet = alphabet_lower + alphabet_lower.upper()
assignment = Literal(':=')
delimiter = Literal(';')
add = oneOf('+ -')
multy = oneOf('* /')
comp = oneOf('< <= = >= > <>')
lpar, rpar = map(Suppress, '()')
# Функции
f_sum = Keyword('sum')
# Имя корневой переменной, латиница
var_name = Word(srange('[a-zA-Z]') + '_' + nums)
node_val = Word(alphabet + '/@_' + nums)
xpath_expr = Word(alphabet + srange('[a-zA-Z]') + '_"[]@/<=>.' + nums)
xpath_expr_white = Word(alphabet + srange('[a-zA-Z]') + '_"[]@/<=>. ' +
nums)
var_assignment = (var_name + assignment + node_val).setParseAction(
self._push_var)
func_args = xpath_expr_white.setParseAction(self._push_xpath)
funcs = (f_sum + Group(lpar + func_args + rpar)).setParseAction(
self._push)
term = funcs | xpath_expr
multi_expr = term + ZeroOrMore(
(multy + term).setParseAction(self._push))
add_expr = multi_expr + ZeroOrMore(
(add + multi_expr).setParseAction(self._push))
expression = var_assignment | (add_expr + Optional(
(comp + add_expr).setParseAction(self._push)))
return expression
def ImportFromString(self, text):
#alphabet + number + underbar
nameG = Word( srange("[a-zA-Z_]"), srange("[a-zA-Z0-9_]") )
stringDoubleG = Literal('"') + SkipTo('"') + Literal('"')
stringSingleG = Literal("'") + SkipTo("'") + Literal("'")
stringG = stringDoubleG | stringSingleG
#comment
comment = (Literal('/*').suppress() + SkipTo('*/').suppress() + Literal('*/').suppress())
digraphNameG = Word( srange("[a-zA-Z_]"), srange("[a-zA-Z0-9_]") )
digraphNameG.setParseAction( self.DigraphNameAction )
nodeNameG = Word( srange("[a-zA-Z_]"), srange("[a-zA-Z0-9_]") )
nodeNameG.setParseAction( self.NodeNameAction )
startG = Literal('digraph').suppress() + digraphNameG.suppress() + Literal('{').suppress()
endG = Literal('}').suppress()
attrBaseG = nameG+Literal('=').suppress()+(nameG ^ stringG)
attrBaseG.setParseAction( self.AttributeAction )
attrRepeatG = attrBaseG + ZeroOrMore( Literal(',').suppress() + attrBaseG )
attrG = ZeroOrMore( Literal('[').suppress() + ZeroOrMore( attrRepeatG ) + Literal(']').suppress() )
nodeG = nodeNameG + attrG
edgeNameG = nameG + Literal('->').suppress() + nameG
edgeNameG.setParseAction( self.EdgeNameAction )
edgeG = edgeNameG + attrG
sizeG = Literal('size') + Literal('=') + (quotedString | dblQuotedString)
baseG = (sizeG | (nodeG ^ edgeG)) + Literal(';').suppress()
baseG = baseG | comment
baseG = ZeroOrMore(baseG)
grammerG = startG + baseG + endG
grammerG.parseString(text)
def compile_attribute(line, in_key=False):
"""
Convert attribute definition from DataJoint format to SQL
:param line: attribution line
:param in_key: set to True if attribute is in primary key set
:returns: (name, sql) -- attribute name and sql code for its declaration
"""
quoted = pp.Or(pp.QuotedString('"'), pp.QuotedString("'"))
colon = pp.Literal(':').suppress()
attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]')).setResultsName('name')
data_type = pp.Combine(pp.Word(pp.alphas)+pp.SkipTo("#", ignore=quoted)).setResultsName('type')
default = pp.Literal('=').suppress() + pp.SkipTo(colon, ignore=quoted).setResultsName('default')
comment = pp.Literal('#').suppress() + pp.restOfLine.setResultsName('comment')
attribute_parser = attribute_name + pp.Optional(default) + colon + data_type + comment
match = attribute_parser.parseString(line+'#', parseAll=True)
match['comment'] = match['comment'].rstrip('#')
if 'default' not in match:
match['default'] = ''
match = {k: v.strip() for k, v in match.items()}
match['nullable'] = match['default'].lower() == 'null'
literals = ['CURRENT_TIMESTAMP'] # not to be enclosed in quotes
if match['nullable']:
if in_key:
raise DataJointError('Primary key attributes cannot be nullable in line %s' % line)
match['default'] = 'DEFAULT NULL' # nullable attributes default to null
else:
if match['default']:
quote = match['default'].upper() not in literals and match['default'][0] not in '"\''
match['default'] = ('NOT NULL DEFAULT ' +
('"%s"' if quote else "%s") % match['default'])
else:
match['default'] = 'NOT NULL'
match['comment'] = match['comment'].replace('"', '\\"') # escape double quotes in comment
sql = ('`{name}` {type} {default}' + (' COMMENT "{comment}"' if match['comment'] else '')).format(**match)
return match['name'], sql
def get_sort_expression():
value_expression = Word(srange("[a-zA-Z0-9_.*]"))
value_expression.setParseAction(lambda tokens: tokens[0])
quoted_value_expression = Word('"').suppress() +\
value_expression + Word('"').suppress()
option_value = value_expression | quoted_value_expression
option_value.setParseAction(lambda tokens: tokens[0])
simple_option = Word(srange("[a-zA-Z0-9_.*]")) +\
Word(':').suppress() + option_value
simple_option.setParseAction(lambda tokens: (tokens[0], tokens[1]))
option = Forward()
option << (simple_option |
(Word(srange("[a-zA-Z0-9_.*]")) +
Word(':').suppress() +
nestedExpr(content=option)))
option.setParseAction(
lambda tokens: parse_sort_field_option(tokens.asList())
)
exp = option + ZeroOrMore(Word(',').suppress() + option)
field_expression = Optional('-') + Word(
srange("[a-zA-Z0-9_.*]")
) + Optional(nestedExpr(content=exp))
field_expression.setParseAction(parse_sort_field_expression)
fields_expression = field_expression + ZeroOrMore(
Word(',').suppress() + field_expression)
fields_expression.setParseAction(parse_sort_expression)
sort_expression = Word('sort:').suppress() \
+ Word('[').suppress() \
+ fields_expression + Word(']').suppress()
return sort_expression
def parse_list_vms(stdout, stderr):
"""
"""
id_vm_name = dblQuotedString(alphas).setResultsName('name')
id_vm_uuid = Word(srange("[a-zA-Z0-9_\-]")).setResultsName('uuid')
left_brace = Suppress("{")
right_brace = Suppress("}")
vm_group = Group(id_vm_name + left_brace + id_vm_uuid + right_brace)
vm_list = OneOrMore(vm_group)
token_lists = vm_list.parseString(stdout, parseAll=True)
return [{'name': token_list.name.replace('\"', ''),
'uuid': token_list.uuid} for token_list in token_lists]
def get_facet_expression():
facet_logical_expression = get_nested_logical_expression()
single_facet_expression = Word(
srange("[a-zA-Z0-9_.]")) +\
Optional(
Word('(').suppress() +
OneOrMore(facet_logical_expression).setParseAction(parse_one_or_more_facets_expression) +
Word(')').suppress())
single_facet_expression.setParseAction(parse_single_facet_expression)
base_facets_expression = OneOrMore(single_facet_expression
+ Optional(',').suppress())
base_facets_expression.setParseAction(parse_base_facets_expression)
facets_expression = Word('facets:').suppress() \
+ Word('[').suppress() \
+ base_facets_expression + Word(']').suppress()
return facets_expression
def parse_createvm(stdout, stderr):
"""
"""
single_quote = Suppress(Literal('\''))
name_prefix = Suppress(Word('Virtual machine'))
id_name = Word(alphanums).setResultsName('name')
name_postfix = Suppress(Word('is created and registered.'))
uuid_prefix = Suppress(Word('UUID:'))
id_vm_uuid = Word(srange("[a-zA-Z0-9_\-]")).setResultsName('uuid')
file_prefix = Suppress(Word('Settings file:'))
id_file_path = Word(alphanums + " /.").setResultsName('file_path')
vm_info = Group(name_prefix + single_quote + id_name +
single_quote + name_postfix + EOL + uuid_prefix + id_vm_uuid + EOL +
file_prefix + single_quote + id_file_path + single_quote + EOL)
out = vm_info.parseString(stdout)[0]
return {'name': out.name, 'uuid': out.uuid, 'file_path': out.file_path}
def __init__(self):
# self.SPACE = pp.Literal(' ').suppress()
self.COLON = pp.Literal(':').suppress()
self.args =pp.Group(pp.ZeroOrMore(pp.Regex(r"[^ :]+")))
self.trail = pp.Word(pp.alphanums)
self.cmd = pp.Word(pp.srange("[A-Z]"), min = 3, max = 6)
self.login = pp.Word(pp.alphanums, min = 3, max = 10)
self.user = pp.Or(self.login | "*").setName('user')
self.line = (self.user.setResultsName('user').setName('USER') +
# self.SPACE +
self.cmd.setResultsName('cmd').setName('CMD') +
self.args.setResultsName('args').setName('ARGS') +
pp.Optional(self.COLON +
self.trail.setResultsName('trail').setName('trail')
)
).setName('line')
def get_aggregations_expression():
aggs_logical_expression = get_nested_logical_expression()
single_aggs_expression = Word(
srange("[a-zA-Z0-9_.]")) +\
Optional(
Word('(').suppress() +
OneOrMore(aggs_logical_expression).setParseAction(
parse_one_or_more_aggs_expression) +
Word(')').suppress())
single_aggs_expression.setParseAction(parse_single_aggs_expression)
base_aggs_expression = OneOrMore(single_aggs_expression
+ Optional(',').suppress())
base_aggs_expression.setParseAction(parse_base_aggs_expression)
aggs_expression = Word('aggregations:').suppress() \
+ Word('[').suppress() \
+ base_aggs_expression + Word(']').suppress()
return aggs_expression
def _int_expression():
from pyparsing import Word, alphanums, nums, Forward, ZeroOrMore, Combine, CaselessLiteral, srange
entry_name = Word(alphanums + ' _+:.-')
integer = Word(nums).addParseAction(lambda s,l,t: [Constant(int(t[0]))])
hex = Combine(CaselessLiteral("0x") + Word(srange("[0-9a-fA-F]"))).addParseAction(lambda s,l,t:[Constant(int(t[0][2:], 16))])
named_reference = ('${' + entry_name + '}').addParseAction(lambda s,l,t:ValueResult(t[1]))
length_reference = ('len{' + entry_name + '}').addParseAction(lambda s,l,t:LengthResult(t[1]))
expression = Forward()
factor = hex | integer | named_reference | length_reference | ('(' + expression + ')').addParseAction(lambda s,l,t:t[1])
entry = factor
for ops in _operators:
op_parse = reduce(operator.or_,
[(character + entry).addParseAction(_half(op)) for character, op in ops])
entry = (entry + ZeroOrMore(op_parse)).addParseAction(_collapse)
expression << entry
return expression
def parse_preservation_priority(x):
"""Parse preservation_priority
Parse cwsac:preservation_priority into cwsac:military_importance and
cwsac:preservation_class.
"""
MISSING = pp.Literal("N/D") | pp.Literal("N/A")
MISSING.addParseAction(lambda s,l,t: [None])
PRIORITY = pp.Word(pp.alphas) + "." + pp.Word(pp.nums)
PRIORITY.addParseAction(lambda s,l,t: [''.join(t)])
CLASS = pp.Literal("Class").suppress()
OP = pp.Literal("(").suppress()
CP = pp.Literal("(").suppress()
priority = (PRIORITY | MISSING)
milimport = pp.Word(pp.srange("[A-D]"))
grammar = priority + OP + CLASS + milimport + pp.Optional(CP)
toks = grammar.parseString(x).asList()
return toks
def __init__(self, filename):
# BNF
start = Literal(";")
text = QuotedString(quoteChar="[",
escChar="\\",
multiline=True,
unquoteResults=True,
endQuoteChar="]")
prop_id = Word(srange("[A-Za-z]"), min=1, max=10)
prop = prop_id + Group(OneOrMore(text))
node = ZeroOrMore(start) + OneOrMore(prop)
sequence = OneOrMore(node)
branch = Forward()
branch << "(" + sequence + ZeroOrMore(branch) + ")"
self.game = OneOrMore(branch)
self.sgf_content = filename
self.moves = None
self.__parse()
self.current = 0
def _defPattern(self):
import pyparsing as pp
pattern = None
#defining sub-variables for tokens
pi = pp.CaselessLiteral( "$PI" )
e = pp.CaselessLiteral( "$E" )
plus = pp.Literal( "+" )
minus = pp.Literal( "-" )
mult = pp.Literal( "*" )
div = pp.Literal( "/" )
lpar = pp.Literal( "(" ).suppress()
rpar = pp.Literal( ")" ).suppress()
point = pp.Literal( "." )
#defining variables
fnumber = pp.Combine( pp.Word( "+-"+pp.nums, pp.nums ) +
pp.Optional( point + pp.Optional( pp.Word( pp.nums ) ) ) +
pp.Optional( e + pp.Word( "+-"+pp.nums, pp.nums ) ) )
#ident = pp.Word(pp.alphas, pp.alphas+pp.nums+"_$")
funct = pp.oneOf("sin cos tan log exp log sqrt",caseless=True)
varia = pp.Word( pp.srange("[a-zA-Z_]"),max=1)
addop = plus | minus
mulop = mult | div
expop = pp.Literal( "**" )
#defining pattern itself
expr = pp.Forward()
atom = (pp.Optional("-") + ( pi | e | varia.addParseAction(self._addVar2Stack) |
fnumber | funct + lpar + expr + rpar ) | ( lpar + expr.suppress() + rpar ))
# by defining exponentiation as "atom [ ^ factor ]..." instead of "atom [ ^ atom ]...",
# we get right-to-left exponents, instead of left-to-righ
# that is, 2^3^2 = 2^(3^2), not (2^3)^2.
factor = pp.Forward()
factor << atom + pp.ZeroOrMore( ( expop + factor ))
term = factor + pp.ZeroOrMore( ( mulop + factor ))
expr << term + pp.ZeroOrMore( ( addop + term ))
pattern = expr
return pattern
def parse_commanders(x):
# Grammar
LBRAK, RBRAK, COMMA = (pp.Literal(x).suppress() for x in "[],")
AND = pp.Literal("and").suppress()
and_ = pp.Optional(COMMA) + AND
NAME = pp.Word(pp.srange("[A-Z]"), pp.alphas + pp.alphas8bit, min=2)
RANK = pp.Or(pp.Literal(rnk) for rnk in RANKS_LOOKUP.keys())("rank")
# ('Lt. Gen.', 'Brig. Gen.', 'Maj. Gen.', 'Lt. Col.',
# 'Lt. Cdr.', 'Lt. Comdr.', '1st Lt.', 'Rear Adm.', 'Bvt. Col.', 'BCol.',
# 'Gen.', 'Maj.', 'Col.', 'Capt.', 'Cdr.', 'Lt.', 'Adm.',
# 'Chief', 'Flag-Officer', 'Acting Master'))("rank")
ABBR = pp.Regex(r"([A-Z]\.)+")
SUFFIX = pp.Literal("Jr.")
NOCMDR = pp.Literal("No Union commander")("none")
USN = pp.Literal("U.S.N.")("usn")
COMBATANT = pp.oneOf(' '.join(('US', 'CS', 'I')))("combatant")
nametoken = ABBR | pp.quotedString | NAME
name = (pp.OneOrMore(nametoken) + pp.Optional(COMMA + SUFFIX))("fullname")
name.addParseAction(lambda s,l,t: ' '.join(t))
cmdrname = (RANK + name + pp.Optional(COMMA + USN))
commander = pp.Group(cmdrname | NOCMDR)
commander_list = pp.Group(pp.delimitedList(commander, ",") +
pp.Optional(and_ + commander))("commanders")
milforce = pp.Group(commander_list + LBRAK + COMBATANT + RBRAK)
grammar = pp.delimitedList(milforce, ";")
toks = grammar.parseString(x)
# A smarter grammar could probably have avoided this
res = {}
for _force in toks:
k = _force['combatant']
res[k] = [x.asDict() for x in _force['commanders']
if "none" not in x.asDict()]
return res