def define_number(self):
"""
Return the syntax definition for a number in Arabic Numerals.
Override this method to support numeral systems other than Arabic
Numerals (0-9).
Do not override this method just to change the character used to
separate thousands and decimals: Use :attr:`T_THOUSANDS_SEPARATOR`
and :attr:`T_DECIMAL_SEPARATOR`, respectively.
"""
# Defining the basic tokens:
to_dot = lambda t: "."
to_plus = lambda t: "+"
to_minus = lambda t: "-"
positive_sign = Literal(self._grammar.get_token("positive_sign"))
positive_sign.setParseAction(to_plus)
negative_sign = Literal(self._grammar.get_token("negative_sign"))
negative_sign.setParseAction(to_minus)
decimal_sep = Literal(self._grammar.get_token("decimal_separator"))
decimal_sep.setParseAction(to_dot)
thousands_sep = Suppress(
self._grammar.get_token("thousands_separator"))
digits = Word(nums)
# Building the integers and decimals:
sign = positive_sign | negative_sign
thousands = Word(nums, max=3) + \
OneOrMore(thousands_sep + Word(nums, exact=3))
integers = thousands | digits
decimals = decimal_sep + digits
number = Combine(Optional(sign) + integers + Optional(decimals))
number.setParseAction(self.make_number)
number.setName("number")
return number
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack, rbrack, lbrace, rbrace, lparen, rparen = map(Literal, "[]{}()")
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~ reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t"
reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack)
reLiteral = (escapedChar | oneOf(list(reLiteralChar)))
reDot = Literal(".")
repetition = (
(lbrace + Word(nums).setResultsName("count") + rbrace) |
(lbrace + Word(nums).setResultsName("minCount") + "," + Word(nums).setResultsName("maxCount") + rbrace) |
oneOf(list("*+?"))
)
reRange.setParseAction(handle_range)
reLiteral.setParseAction(handle_literal)
reMacro.setParseAction(handle_macro)
reDot.setParseAction(handle_dot)
reTerm = (reLiteral | reRange | reMacro | reDot)
reExpr = operatorPrecedence(reTerm, [
(repetition, 1, opAssoc.LEFT, handle_repetition),
(None, 2, opAssoc.LEFT, handle_sequence),
(Suppress('|'), 2, opAssoc.LEFT, handle_alternative),
])
_parser = reExpr
return _parser
def integer_value():
'''
Grammar for integer numbers
'''
integer_t = Combine(Optional(MINUS_OP) + Word(nums))
integer_t.setParseAction(lambda t: int(t[0]))
return integer_t
def define_number(self):
"""
Return the syntax definition for a number in Arabic Numerals.
Override this method to support numeral systems other than Arabic
Numerals (0-9).
Do not override this method just to change the character used to
separate thousands and decimals: Use :attr:`T_THOUSANDS_SEPARATOR`
and :attr:`T_DECIMAL_SEPARATOR`, respectively.
"""
# Defining the basic tokens:
to_dot = lambda t: "."
to_plus = lambda t: "+"
to_minus = lambda t: "-"
positive_sign = Literal(self._grammar.get_token("positive_sign"))
positive_sign.setParseAction(to_plus)
negative_sign = Literal(self._grammar.get_token("negative_sign"))
negative_sign.setParseAction(to_minus)
decimal_sep = Literal(self._grammar.get_token("decimal_separator"))
decimal_sep.setParseAction(to_dot)
thousands_sep = Suppress(self._grammar.get_token("thousands_separator"))
digits = Word(nums)
# Building the integers and decimals:
sign = positive_sign | negative_sign
thousands = Word(nums, max=3) + \
OneOrMore(thousands_sep + Word(nums, exact=3))
integers = thousands | digits
decimals = decimal_sep + digits
number = Combine(Optional(sign) + integers + Optional(decimals))
number.setParseAction(self.make_number)
number.setName("number")
return number
def _grammar():
from pyparsing import alphas, alphanums, nums
from pyparsing import oneOf, Suppress, Optional, Group, ZeroOrMore, NotAny
from pyparsing import Forward, operatorPrecedence, opAssoc, Word, White
from pyparsing import delimitedList, Combine, Literal, OneOrMore
expression = Forward()
LPAR, RPAR, DOT, LBRAC, RBRAC = map(Suppress, "().{}")
nw = NotAny(White())
identifier = Word(alphas + "_", alphanums + "_")
integer = Word(nums)
integer.setParseAction(IntegerNode)
fractional = Combine(Word('+' + '-' + nums, nums) + '.' + Word(nums))
fractional.setParseAction(FloatNode)
literal = fractional | integer
arglist = delimitedList(expression)
seqrange = LBRAC + expression + Suppress('..') + expression + RBRAC
seqrange.setParseAction(lambda t: SequenceNode(start=t[0], stop=t[1]))
seqexplicit = LBRAC + Optional(arglist) + RBRAC
seqexplicit.setParseAction(lambda t: SequenceNode(lst=t))
sequence = seqrange | seqexplicit
rollmod = nw + Group(oneOf("d k r e x") + Optional(integer))
numdice = Optional(integer, default=1)
roll = numdice + nw + Suppress("d") + nw + (integer | sequence)
roll += Group(ZeroOrMore(rollmod))
roll.setParseAction(DieRollNode)
call = LPAR + Group(Optional(arglist)) + RPAR
function = identifier + call
function.setParseAction(FunctionNode)
seqexpr = ((roll | sequence | function) +
Group(OneOrMore(DOT + identifier + call)))
seqexpr.setParseAction(SeqMethodNode)
variable = Word(alphas + "_", alphanums + "_ ")
variable.setParseAction(VariableNode)
atom = seqexpr | roll | literal | sequence | function | variable
expoop = Literal('^')
signop = oneOf("+ -")
multop = oneOf("* /")
plusop = oneOf("+ -")
# noinspection PyUnresolvedReferences
expression << operatorPrecedence(atom, [
(expoop, 2, opAssoc.LEFT, BinaryOpNode),
(signop, 1, opAssoc.RIGHT, UnaryOpNode),
(multop, 2, opAssoc.LEFT, BinaryOpNode),
(plusop, 2, opAssoc.LEFT, BinaryOpNode),
])
return expression
def func_tokens(dictionary, parse_action):
func_name = Word(alphas+'_', alphanums+'_')
func_ident = Combine('$' + func_name.copy()('funcname'))
func_tok = func_ident + originalTextFor(nestedExpr())('args')
func_tok.leaveWhitespace()
func_tok.setParseAction(parse_action)
func_tok.enablePackrat()
rx_tok = Combine(Literal('$').suppress() + Word(nums)('num'))
def replace_token(tokens):
index = int(tokens.num)
return dictionary.get(index, u'')
rx_tok.setParseAction(replace_token)
strip = lambda s, l, tok: tok[0].strip()
text_tok = CharsNotIn(u',').setParseAction(strip)
quote_tok = QuotedString('"')
if dictionary:
arglist = Optional(delimitedList(quote_tok | rx_tok | text_tok))
else:
arglist = Optional(delimitedList(quote_tok | text_tok))
return func_tok, arglist, rx_tok
def func_tokens(dictionary, parse_action):
func_name = Word(alphas + '_', alphanums + '_')
func_ident = Combine('$' + func_name.copy()('funcname'))
func_tok = func_ident + originalTextFor(nestedExpr())('args')
func_tok.leaveWhitespace()
func_tok.setParseAction(parse_action)
func_tok.enablePackrat()
rx_tok = Combine(Literal('$').suppress() + Word(nums)('num'))
def replace_token(tokens):
index = int(tokens.num)
return dictionary.get(index, '')
rx_tok.setParseAction(replace_token)
strip = lambda s, l, tok: tok[0].strip()
text_tok = CharsNotIn(',').setParseAction(strip)
quote_tok = QuotedString('"')
if dictionary:
arglist = Optional(delimitedList(quote_tok | rx_tok | text_tok))
else:
arglist = Optional(delimitedList(quote_tok | text_tok))
return func_tok, arglist, rx_tok
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack, rbrack, lbrace, rbrace, lparen, rparen, colon, qmark = map(
Literal, "[]{}():?")
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(
c for c in printables if c not in r"\[]{}().*?+|") + " \t"
reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack)
reLiteral = (escapedChar | oneOf(list(reLiteralChar)))
reNonCaptureGroup = Suppress("?:")
reDot = Literal(".")
repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) |
(lbrace + Word(nums).setResultsName("minCount") + "," +
Word(nums).setResultsName("maxCount") + rbrace)
| oneOf(list("*+?")))
reRange.setParseAction(handleRange)
reLiteral.setParseAction(handleLiteral)
reMacro.setParseAction(handleMacro)
reDot.setParseAction(handleDot)
reTerm = (reLiteral | reRange | reMacro | reDot | reNonCaptureGroup)
reExpr = infixNotation(reTerm, [
(repetition, 1, opAssoc.LEFT, handleRepetition),
(None, 2, opAssoc.LEFT, handleSequence),
(Suppress('|'), 2, opAssoc.LEFT, handleAlternative),
])
_parser = reExpr
return _parser
def __init__(self):
# speed up infixNotation considerably at the price of some cache memory
ParserElement.enablePackrat()
boolean = Keyword('True') | Keyword('False')
none = Keyword('None')
integer = Word(nums)
real = Combine(Word(nums) + "." + Word(nums))
string = (QuotedString('"', escChar='\\')
| QuotedString("'", escChar='\\'))
regex = QuotedString('/', escChar='\\')
identifier = Word(alphas, alphanums + '_')
dereference = infixNotation(identifier, [
(Literal('.'), 2, opAssoc.LEFT, EvalArith),
])
result = (Keyword('bad') | Keyword('fail') | Keyword('good')
| Keyword('ignore') | Keyword('unknown'))
rval = boolean | none | real | integer | string | regex | result | dereference
rvallist = Group(
Suppress('[') + Optional(delimitedList(rval)) + Suppress(']'))
rvalset = Group(
Suppress('{') + Optional(delimitedList(rval)) + Suppress('}'))
operand = rval | rvallist | rvalset
# parse actions replace the parsed tokens with an instantiated object
# which we can later call into for evaluation of its content
boolean.setParseAction(EvalBoolean)
none.setParseAction(EvalNone)
integer.setParseAction(EvalInteger)
real.setParseAction(EvalReal)
string.setParseAction(EvalString)
regex.setParseAction(EvalRegex)
identifier.setParseAction(EvalIdentifier)
result.setParseAction(EvalResult)
rvallist.setParseAction(EvalList)
rvalset.setParseAction(EvalSet)
identity_test = Keyword('is') + ~Keyword('not') | Combine(
Keyword('is') + Keyword('not'), adjacent=False, joinString=' ')
membership_test = Keyword('in') | Combine(
Keyword('not') + Keyword('in'), adjacent=False, joinString=' ')
comparison_op = oneOf('< <= > >= != == isdisjoint')
comparison = identity_test | membership_test | comparison_op
self.parser = infixNotation(operand, [
(Literal('**'), 2, opAssoc.LEFT, EvalPower),
(oneOf('+ - ~'), 1, opAssoc.RIGHT, EvalModifier),
(oneOf('* / // %'), 2, opAssoc.LEFT, EvalArith),
(oneOf('+ -'), 2, opAssoc.LEFT, EvalArith),
(oneOf('<< >>'), 2, opAssoc.LEFT, EvalArith),
(Literal('&'), 2, opAssoc.LEFT, EvalArith),
(Literal('^'), 2, opAssoc.LEFT, EvalArith),
(Literal('|'), 2, opAssoc.LEFT, EvalArith),
(comparison, 2, opAssoc.LEFT, EvalLogic),
(Keyword('not'), 1, opAssoc.RIGHT, EvalModifier),
(Keyword('and'), 2, opAssoc.LEFT, EvalLogic),
(Keyword('or'), 2, opAssoc.LEFT, EvalLogic),
(Keyword('->'), 2, opAssoc.LEFT, EvalArith),
])
def float_value():
'''
Grammar for float values
'''
from grammar.symbols import DOT
float_t = Combine(Optional(MINUS_OP) + Word(nums) + DOT + Word(nums))
float_t.setParseAction(lambda t: float(t[0]))
return float_t
def _build_filter_parser(field_names):
field = _build_field_expr(field_names)
negation = CaselessKeyword('not')
negation.setParseAction(lambda x: Negation(x[0]))
comparison_operator = Group(
Keyword('=') ^ Keyword('!=') ^ Keyword('>=') ^ Keyword('<=')
^ Keyword('<') ^ Keyword('>') ^ Keyword('lte') # match before lt
^ Keyword('lt') ^ Keyword('gte') # match before gt
^ Keyword('gt')
^ (Optional(negation) +
(CaselessKeyword('contains') ^ CaselessKeyword('icontains')
^ CaselessKeyword('startswith') ^ CaselessKeyword('istartswith')
^ CaselessKeyword('endswith') ^ CaselessKeyword('iendswith')
^ CaselessKeyword('eq'))))
comparison_operator.setParseAction(lambda x: Operator(x))
single_value_operator = Group(
CaselessKeyword('isnull')
^ (Optional(negation) + CaselessKeyword('isnull')))
single_value_operator.setParseAction(lambda x: Operator(x))
plusorminus = Literal('+') | Literal('-')
num_integer = Combine(Optional(plusorminus) + Word(nums))
num_integer.setParseAction(lambda x: Integer(x[0]))
num_float = Combine(Optional(plusorminus) + Word(nums) + '.' + Word(nums))
num_float.setParseAction(lambda x: Float(x[0]))
quoted_string = (QuotedString("'") ^ QuotedString('"'))
quoted_string.setParseAction(lambda x: String(x[0]))
boolean = Or([
CaselessKeyword(v) for v in BOOLEAN_TRUE_VALUES + BOOLEAN_FALSE_VALUES
])
boolean.setParseAction(lambda x: Boolean(x[0]))
value = (quoted_string ^ num_integer ^ boolean)
comparison = Group((field + comparison_operator + value)
^ (value + comparison_operator + field)
^ (field + comparison_operator + field)
^ (field + single_value_operator))
comparison.setParseAction(lambda x: Comparison(x))
invalid_comparison = Group(
(value + comparison_operator + value).setParseAction(lambda x: fail(
"Value may not be compared with values: {0}".format(' '.join(x)))))
logical_op = Group(CaselessKeyword("and") | CaselessKeyword("or"))
logical_op.setParseAction(lambda x: LogicalOp(x[0][0]))
statement = Optional(comparison | invalid_comparison) + ZeroOrMore(
logical_op + (comparison | invalid_comparison))
return statement
def _BNF(self):
base16 = Literal("$")
hex = Combine(base16 + Word(hexnums + "_"))
base4 = Literal("%%")
quaternary = Combine(base4 + Word("0123_"))
base2 = Literal("%")
binary = Combine(base2 + Word("01_"))
plusminus = Literal("+") | Literal("-")
integer = Combine(Optional(plusminus) + Word(nums+"_"))
name_token = Combine(Optional(Literal(":") | Literal("@")) + Word("_" + alphas, "_" + alphanums))
name_token.setParseAction(self._mark_name_token)
lparens = Literal("(").suppress()
rparens = Literal(")").suppress()
# op0 = Literal("@")
op1 = (Literal("^^") | Literal("||") | Literal("|<") | Literal(">|") | Literal("!")).setParseAction(self._mark_unary)
op2 = Literal("->") | Literal("<-") | Literal(">>") | Literal("<<") | Literal("~>") | Literal("><")
op3 = Literal("&")
op4 = Literal("|") | Literal("^")
op5 = Literal("**") | Literal("*") | Literal("//") | Literal("/")
op6 = Literal("+") | Literal("-")
op7 = Literal("#>") | Literal("<#")
op8 = Literal("<") | Literal(">") | Literal("<>") | Literal("==") | Literal("=<") | Literal("=>")
op9 = Literal("NOT").setParseAction(self._mark_unary)
op10 = Literal("AND")
op11 = Literal("OR")
op12 = Literal(",")
expr = Forward()
atom = name_token | hex | quaternary | binary | integer | quotedString
atom.setParseAction(self._push)
atom = atom | (lparens + expr.suppress() + rparens)
# term0 = atom + ZeroOrMore((op0 + atom) .setParseAction(self._push))
# term1 = term0 + ZeroOrMore((op1 + term0) .setParseAction(self._push))
term1 = atom + ZeroOrMore((op1 + atom) .setParseAction(self._push))
term2 = term1 + ZeroOrMore((op2 + term1) .setParseAction(self._push))
term3 = term2 + ZeroOrMore((op3 + term2) .setParseAction(self._push))
term4 = term3 + ZeroOrMore((op4 + term3) .setParseAction(self._push))
term5 = term4 + ZeroOrMore((op5 + term4) .setParseAction(self._push))
term6 = term5 + ZeroOrMore((op6 + term5) .setParseAction(self._push))
term7 = term6 + ZeroOrMore((op7 + term6) .setParseAction(self._push))
term8 = term7 + ZeroOrMore((op8 + term7) .setParseAction(self._push))
term9 = term8 + ZeroOrMore((op9 + term8) .setParseAction(self._push))
term10 = term9 + ZeroOrMore((op10 + term9) .setParseAction(self._push))
term11 = term10 + ZeroOrMore((op11 + term10).setParseAction(self._push))
expr << term11 + ZeroOrMore((op12 + term11).setParseAction(self._push))
return expr
def parseString(self):
"""
Parse string.
Initial attempt at grammar, needs restructuring.
ANDop :: 'AND'
ORop :: 'OR'
lpar :: '('
rpar :: ')'
keyword :: alphas
result :: alphanums
word :: alphanums
hashtag :: COMBINE('#' word)
cashtag :: COMBINE ('$' word)
atom :: COMBINE('-' atom) | word | COMBINE(keyword ':' result) | lpar expr rpar | hashtag | cashtag
term :: atom [ ORop atom ]*
expr :: term [ ANDop term ]*
"""
if self.bnf is None:
ANDop = Keyword('AND')
ORop = Keyword('OR')
colon=Literal(':')
word = ~colon + ~ORop+ ~ANDop + Word(alphanums,min=1)
keyword = ~colon + ~ORop + ~ANDop + Word(alphas,min=1)
quoted = quotedString
hashtag = Combine(Literal('#')+Word(alphanums))
cashtag = Combine(Literal('$')+Word(alphanums))
minus = Literal('-')
lpar, rpar = map(Suppress, "()")
expr = Forward()
atom = Forward()
atom <<= (minus[...].setParseAction(self.push_first)+
(Group(lpar + expr + rpar)
|
Group(keyword.setResultsName("keyword") + colon + word.setResultsName("value")).setParseAction(self.push_first)
| hashtag.setParseAction(self.push_first)
| cashtag.setParseAction(self.push_first)
| Group(quoted.setResultsName('quote')).setParseAction(self.push_first)
|
word.setParseAction(self.push_first).setResultsName('word')[1]
)
).setParseAction(self.push_unary_minus)
# AND is optional word1 word2 implicitly is word1 AND word2
term = Forward()
term <<= atom + (ORop + atom).setParseAction(self.push_first)[...]
expr <<= term + (Optional(ANDop) + term).setParseAction(self.push_and)[...]
self.bnf = expr
self.parsed_str=self.bnf.parseString(self.query_str, parseAll=True)
def getkw_bnf(self):
sect_begin = Literal("{").suppress()
sect_end = Literal("}").suppress()
array_begin = Literal("[").suppress()
array_end = Literal("]").suppress()
tag_begin = Literal("<").suppress()
tag_end = Literal(">").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_data=Literal('$end').suppress()
prtable = alphanums+r'!$%&*+-./<>?@^_|~'
ival=Regex('[-]?\d+')
dval=Regex('-?\d+\.\d*([eE]?[+-]?\d+)?')
lval=Regex('([Yy]es|[Nn]o|[Tt]rue|[Ff]alse|[Oo]n|[Oo]ff)')
# Helper definitions
kstr= quotedString.setParseAction(removeQuotes) ^ \
dval ^ ival ^ lval ^ Word(prtable)
name = Word(alphas+"_",alphanums+"_")
vec=array_begin+delimitedList(dval ^ ival ^ lval ^ Word(prtable) ^ \
Literal("\n").suppress() ^ \
quotedString.setParseAction(removeQuotes))+array_end
sect=name+sect_begin
tag_sect=name+Group(tag_begin+name+tag_end)+sect_begin
# Grammar
keyword = name + eql + kstr
vector = name + eql + vec
data=Combine(dmark+name)+SkipTo(end_data)+end_data
section=Forward()
sect_def=(sect | tag_sect ) #| vec_sect)
input=section | data | vector | keyword
section << sect_def+ZeroOrMore(input) + sect_end
# Parsing actions
ival.setParseAction(self.conv_ival)
dval.setParseAction(self.conv_dval)
lval.setParseAction(self.conv_lval)
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
data.setParseAction(self.store_data)
sect.setParseAction(self.add_sect)
tag_sect.setParseAction(self.add_sect)
sect_end.setParseAction(self.pop_sect)
bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error)
bnf.ignore(pythonStyleComment)
return bnf
def getkw_bnf(self):
sect_begin = Literal("{").suppress()
sect_end = Literal("}").suppress()
array_begin = Literal("[").suppress()
array_end = Literal("]").suppress()
tag_begin = Literal("<").suppress()
tag_end = Literal(">").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_data=Literal('$end').suppress()
prtable = alphanums+r'!$%&*+-./<>?@^_|~'
ival=Regex('[-]?\d+')
dval=Regex('-?\d+\.\d*([eE]?[+-]?\d+)?')
lval=Regex('([Yy]es|[Nn]o|[Tt]rue|[Ff]alse|[Oo]n|[Oo]ff)')
# Helper definitions
kstr= quotedString.setParseAction(removeQuotes) ^ \
dval ^ ival ^ lval ^ Word(prtable)
name = Word(alphas+"_",alphanums+"_")
vec=array_begin+delimitedList(dval ^ ival ^ lval ^ Word(prtable) ^ \
Literal("\n").suppress() ^ \
quotedString.setParseAction(removeQuotes))+array_end
sect=name+sect_begin
tag_sect=name+Group(tag_begin+name+tag_end)+sect_begin
# Grammar
keyword = name + eql + kstr
vector = name + eql + vec
data=Combine(dmark+name)+SkipTo(end_data)+end_data
section=Forward()
sect_def=(sect | tag_sect ) #| vec_sect)
input=section | data | vector | keyword
section << sect_def+ZeroOrMore(input) + sect_end
# Parsing actions
ival.setParseAction(self.conv_ival)
dval.setParseAction(self.conv_dval)
lval.setParseAction(self.conv_lval)
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
data.setParseAction(self.store_data)
sect.setParseAction(self.add_sect)
tag_sect.setParseAction(self.add_sect)
sect_end.setParseAction(self.pop_sect)
bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error)
bnf.ignore(pythonStyleComment)
return bnf
def rhs_value_p():
Ipv4Address = Combine(Word(nums) +
('.' + Word(nums)) * 3).setResultsName('ipv4')
Ipv4Address = Ipv4Address.setParseAction(lambda s, l, toks: toks[0])
Int = Word(nums)
Int = Int.setParseAction(lambda s, l, toks: int(toks[0]))
Float = Combine(Word(nums) + '.' + Word(nums)).setResultsName('float')
Float = Float.setParseAction(lambda s, l, toks: float(toks[0]))
String = quotedString.copy().addParseAction(pyparsing.removeQuotes)
rhs = pyparsing.Or([String, Int, Float, Ipv4Address])
return rhs
def grammar(self):
""" Define the parser grammar.
"""
# Ignore TeX commands between delimiters $$, \(, \)
tex_eq = (Literal(r'\(') | Literal(r'$$') | Literal(r'\[')) + ... + (
Literal(r'\)') | Literal(r'$$') | Literal(r'\]'))
# Define elemtary stuff
leftAc = Literal('{').suppress()
rightAc = Literal('}').suppress()
lpar = Literal('(')
rpar = Literal(')')
integer = Word(nums) # simple unsigned integer
real = Regex(r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?")
real.setParseAction(self.real_hook)
number = real | integer
# Define function
fnname = Word(alphas, alphanums + "_")('name')
# Require expr to finalize the def
function = Forward()
function.setParseAction(self.function_hook)
# What are the namming rule for the jocker? Need to start by a letter,
# may contain almost everything
variable = Combine(leftAc + Word(alphas, alphanums + "_") +
rightAc)('name')
variable.setParseAction(self.variable_hook)
variable.ignore(tex_eq)
# arithmetic operators
minus = Literal('-')
arithOp = oneOf("+ * /") | minus
equal = Literal('{=').suppress()
# Require atom to finalize the def
expr = Forward()
# Define atom
atom = number | (0, None) * minus + (Group(lpar + expr + rpar)
| variable | function)
atom.setParseAction(self.atom_hook)
# Finalize postponed elements...
expr << atom + ZeroOrMore(arithOp + atom)
# Need to group arguments for swapping them
function << fnname + Group(lpar + Group(ZeroOrMore(expr)) +
Optional(Literal(',') + Group(expr)) + rpar)
# Define equation
equation = equal + expr + rightAc
equation.setParseAction(self.equation_hook)
return equation, variable
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack = Literal("[")
rbrack = Literal("]")
lbrace = Literal("{")
rbrace = Literal("}")
lparen = Literal("(")
rparen = Literal(")")
reMacro = Suppress("\\") + oneOf(list("dwsZ"))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in string.printable
if c not in r"\[]{}().*?+|")
reRange = Combine(lbrack.suppress() +
SkipTo(rbrack, ignore=escapedChar) +
rbrack.suppress())
reLiteral = (escapedChar | oneOf(list(reLiteralChar)))
reDot = Literal(".")
repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) |
(lbrace + Word(nums).setResultsName("minCount") + "," +
Word(nums).setResultsName("maxCount") + rbrace)
| oneOf(list("*+?")))
reExpr = Forward()
reGroup = (lparen.suppress() +
Optional(Literal("?").suppress() +
oneOf(list(":P"))).setResultsName("option") +
reExpr.setResultsName("expr") + rparen.suppress())
reTerm = (reLiteral | reRange | reMacro | reDot | reGroup)
reExpr << operatorPrecedence(reTerm, [
(repetition, 1, opAssoc.LEFT, create(Repetition)),
(None, 2, opAssoc.LEFT, create(Sequence)),
(Suppress('|'), 2, opAssoc.LEFT, create(Alternation)),
])
reGroup.setParseAction(create(Group))
reRange.setParseAction(create(Range))
reLiteral.setParseAction(create(Character))
reMacro.setParseAction(create(Macro))
reDot.setParseAction(create(Dot))
_parser = reExpr
return _parser
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack = Literal("[")
rbrack = Literal("]")
lbrace = Literal("{")
rbrace = Literal("}")
lparen = Literal("(")
rparen = Literal(")")
reMacro = Suppress("\\") + oneOf(list("dwsZ"))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in string.printable if c not in r"\[]{}().*?+|")
reRange = Combine(lbrack.suppress() + SkipTo(rbrack,ignore=escapedChar) + rbrack.suppress())
reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) )
reDot = Literal(".")
repetition = (
( lbrace + Word(nums).setResultsName("count") + rbrace ) |
( lbrace + Word(nums).setResultsName("minCount")+","+ Word(nums).setResultsName("maxCount") + rbrace ) |
oneOf(list("*+?"))
)
reExpr = Forward()
reGroup = (lparen.suppress() +
Optional(Literal("?").suppress() + oneOf(list(":P"))).setResultsName("option") +
reExpr.setResultsName("expr") +
rparen.suppress())
reTerm = ( reLiteral | reRange | reMacro | reDot | reGroup )
reExpr << operatorPrecedence( reTerm,
[
(repetition, 1, opAssoc.LEFT, create(Repetition)),
(None, 2, opAssoc.LEFT, create(Sequence)),
(Suppress('|'), 2, opAssoc.LEFT, create(Alternation)),
]
)
reGroup.setParseAction(create(Group))
reRange.setParseAction(create(Range))
reLiteral.setParseAction(create(Character))
reMacro.setParseAction(create(Macro))
reDot.setParseAction(create(Dot))
_parser = reExpr
return _parser
def getkw_bnf(self):
lcb = Literal("{").suppress()
rcb = Literal("}").suppress()
lsb = Literal("[").suppress()
rsb = Literal("]").suppress()
lps = Literal("(").suppress()
rps = Literal(")").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_sect = rcb
end_data = Literal('$end').suppress()
prtable = srange("[0-9a-zA-Z]") + '!$%&*+-./<>?@^_|~:'
kstr = Word(prtable) ^ quotedString.setParseAction(removeQuotes)
name = Word(alphas + "_", alphanums + "_")
vec=lsb+delimitedList(Word(prtable) ^ Literal("\n").suppress() ^\
quotedString.setParseAction(removeQuotes))+rsb
key = kstr ^ vec
keyword = name + eql + kstr
vector = name + eql + vec
data = Combine(dmark + name) + SkipTo(end_data) + end_data
data.setParseAction(self.store_data)
sect = name + lcb
sect.setParseAction(self.add_sect)
key_sect = name + Group(lps + kstr + rps) + lcb
key_sect.setParseAction(self.add_sect)
vec_sect = name + Group(lps + vec + rps) + lcb
vec_sect.setParseAction(self.add_vecsect)
end_sect.setParseAction(self.pop_sect)
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
section = Forward()
input = section ^ data ^ keyword ^ vector
sectdef = sect ^ key_sect ^ vec_sect
section << sectdef + ZeroOrMore(input) + rcb
bnf = ZeroOrMore(input)
bnf.ignore(pythonStyleComment)
return bnf
def grammar(self, tabilify=True):
number = Combine(Word(nums) + Optional("." + OneOrMore(Word(nums))))
table = Combine(Word(alphas) + OneOrMore(Word("_"+alphanums)))
if tabilify:
number.setParseAction(self._number_parse_action)
table.setParseAction(self._table_parse_action)
signop = oneOf('+ -')
multop = oneOf('* /')
plusop = oneOf('+ -')
operand = number | table
return operatorPrecedence(operand, [
(signop, 1, opAssoc.RIGHT),
(multop, 2, opAssoc.LEFT),
(plusop, 2, opAssoc.LEFT)
])
def grammar(self):
number = Combine(Word(nums) + Optional("." + OneOrMore(Word(nums))))
table = Combine(Word(alphas) + OneOrMore(Word("_"+alphanums)))
number.setParseAction(self._number_parse_action)
table.setParseAction(self._table_parse_action)
signop = oneOf('+ -')
multop = oneOf('* /')
plusop = oneOf('+ -')
operand = number | table
return operatorPrecedence(operand, [
(signop, 1, opAssoc.RIGHT),
(multop, 2, opAssoc.LEFT),
(plusop, 2, opAssoc.LEFT)
])
def getkw_bnf(self):
sect_begin = Literal("{").suppress()
sect_end = Literal("}").suppress()
array_begin = Literal("[").suppress()
array_end = Literal("]").suppress()
arg_begin = Literal("(").suppress()
arg_end = Literal(")").suppress()
eql = Literal("=").suppress()
dmark = Literal('$').suppress()
end_data=Literal('$end').suppress()
prtable = alphanums+r'!$%&*+-./<>?@^_|~'
# Helper definitions
kstr=Word(prtable) ^ quotedString.setParseAction(removeQuotes)
name = Word(alphas+"_",alphanums+"_")
vec=array_begin+delimitedList(Word(prtable) ^ \
Literal("\n").suppress() ^ \
quotedString.setParseAction(removeQuotes))+array_end
sect=name+sect_begin
key_sect=name+Group(arg_begin+kstr+arg_end)+sect_begin
vec_sect=name+Group(arg_begin+vec+ arg_end)+sect_begin
# Grammar
keyword = name + eql + kstr
vector = name + eql + vec
data=Combine(dmark+name)+SkipTo(end_data)+end_data
section=Forward()
sect_def=(sect | key_sect | vec_sect)
input=section | data | vector | keyword
section << sect_def+ZeroOrMore(input) + sect_end
# Parsing actions
keyword.setParseAction(self.store_key)
vector.setParseAction(self.store_vector)
data.setParseAction(self.store_data)
sect.setParseAction(self.add_sect)
key_sect.setParseAction(self.add_sect)
vec_sect.setParseAction(self.add_vecsect)
sect_end.setParseAction(self.pop_sect)
bnf=ZeroOrMore(input) + StringEnd().setFailAction(parse_error)
bnf.ignore(pythonStyleComment)
return bnf
def _build_grammar(self):
expr = Forward()
float_lit = Combine(Word(nums) + '.' + Word(nums))
float_lit.setName('float')
float_lit.setParseAction(lambda x: \
self.to_literal(float(x[0])))
int_lit = Word(nums)
int_lit.setName('int')
int_lit.setParseAction(lambda x: \
self.to_literal(int(x[0])))
num = (float_lit | int_lit)
num.setParseAction(lambda x: x[0])
tag_name = Word(alphas + "_", alphanums + "_")
tag_name.setName('tag_name')
tag_name.setParseAction(lambda t: tag_reference.TagReference(t[0]))
quoted_string = QuotedString("'")
quoted_string.setParseAction(lambda s: self.to_literal(s[0]))
oper = oneOf('+ * / -')
oper.setParseAction(lambda o: o[0])
lpar = Literal("(").suppress()
rpar = Literal(")").suppress()
arith = Group(lpar + expr + oper + expr + rpar)
arith.setParseAction(lambda t: \
self.to_arith(t[0][0], t[0][1], t[0][2]))
assign = tag_name + '=' + expr
assign.setName('assign')
assign.setParseAction(lambda x: self.to_assign(x[0],x[2]))
print_tags = Literal('?')
print_tags.setParseAction(lambda x: self.to_print_tags())
expr <<(arith|assign|tag_name|num|quoted_string|print_tags)
expr.setParseAction(lambda x: x[0])
return expr
def create_bnf():
cvt_int = lambda toks: int(toks[0])
cvt_real = lambda toks: float(toks[0])
cvt_tuple = lambda toks : tuple(toks.asList())
cvt_dict = lambda toks: dict(toks.asList())
# define punctuation as suppressed literals
(lparen, rparen, lbrack, rbrack,
lbrace, rbrace, colon) = map(Suppress,"()[]{}:")
integer = Combine(Optional(oneOf("+ -")) + Word(nums)).setName("integer")
integer.setParseAction(cvt_int)
real = Combine(Optional(oneOf("+ -"))+ Word(nums)
+ "." + Optional(Word(nums))
+ Optional("e" + Optional(oneOf("+ -"))
+ Word(nums))).setName("real")
real.setParseAction(cvt_real)
tuple_str = Forward()
list_str = Forward()
dict_str = Forward()
list_item = (real | integer | Group(list_str) | tuple_str | dict_str
| quotedString.setParseAction(removeQuotes)
| Word(alphas8bit + alphas, alphas8bit + alphanums + "_"))
list_item2 = list_item | Empty().setParseAction(lambda: [None])
tuple_str << (Suppress("(") + Optional(delimitedList(list_item)) +
Optional(Suppress(",")) + Suppress(")"))
tuple_str.setParseAction(cvt_tuple)
list_str << (lbrack + Optional(delimitedList(list_item) +
Optional(Suppress(","))) + rbrack)
dict_entry = Group(list_item + colon + list_item2)
dict_inner = delimitedList(dict_entry) + Optional(Suppress(","))
dict_inner.setParseAction(cvt_dict)
dict_str << (lbrace + Optional(dict_inner) + rbrace)
return dict_inner
def numeric_literal() -> Token:
numeral = Combine(
Word(nums) + ZeroOrMore(Optional(Word("_")) + Word(nums)))
numeral.setParseAction(lambda t: t[0].replace("_", ""))
decimal_literal = Group(numeral)
decimal_literal.setParseAction(lambda t: (int(t[0][0]), 0))
extended_digit = Word(nums + "ABCDEF")
based_numeral = Combine(extended_digit +
ZeroOrMore(Optional("_") + extended_digit))
based_numeral.setParseAction(lambda t: t[0].replace("_", ""))
based_literal = numeral + Literal("#") - based_numeral - Literal("#")
based_literal.setParseAction(lambda t: (int(t[2], int(t[0])), int(t[0])))
num_literal = based_literal | decimal_literal
num_literal.setName("Number")
return locatedExpr(num_literal).setParseAction(lambda s, l, t: Number(
t[0][1][0], t[0][1][1], parser_location(t[0][0], t[0][2], s)))
def num_parser():
"""A floating point number parser
"""
T_DOT = Literal(".")
T_MIN = Literal("-")
T_PLU = Literal("+")
T_EXP = Literal("e")
num = Combine(
Optional(T_MIN) + Word(nums) + Optional(T_DOT + Word(nums)) +
Optional(T_EXP + Optional(T_MIN ^ T_PLU) + Word(nums)))
num = num.setParseAction(lambda t: float(t[0]))
return num
def pattern():
"""pyparsing pattern
"""
def attachLocation(s, loc, tocs):
"""pyparsing callback. Saves path position in the original string
"""
return [(loc, tocs[0])]
path = CharsNotIn(" \t")("path")
path.setParseAction(attachLocation)
longPath = CharsNotIn(" \t", min=2)("path")
longPath.setParseAction(attachLocation)
slashPath = Combine(Literal('/') + Optional(CharsNotIn(" \t")))("path")
slashPath.setParseAction(attachLocation)
pat = ((Literal('f ') + Optional(White()) + Optional(path)) ^ longPath ^ slashPath) + \
Optional(White() + Word(nums)("line"))
pat.leaveWhitespace()
pat.setParseAction(CommandOpen.create)
return pat
def pattern():
"""pyparsing pattern
"""
def attachLocation(s, loc, tocs):
"""pyparsing callback. Saves path position in the original string
"""
return [(loc, tocs[0])]
from pyparsing import CharsNotIn, Combine, Literal, Optional, White, Word, nums # delayed import, performance optimization
path = CharsNotIn(" \t")("path")
path.setParseAction(attachLocation)
longPath = CharsNotIn(" \t", min=2)("path")
longPath.setParseAction(attachLocation)
slashPath = Combine(Literal('/') + Optional(CharsNotIn(" \t")))("path")
slashPath.setParseAction(attachLocation)
pat = ((Literal('f ') + Optional(White()) + Optional(path)) ^ longPath ^ slashPath) + \
Optional(White() + Word(nums)("line"))
pat.leaveWhitespace()
pat.setParseAction(CommandOpen.create)
return pat
def expr(self) -> ParserElement:
MENTION = Combine(
"[" + Optional(
SkipTo("|", failOn="]") + Suppress("|"),
default="",
) + "~" + Optional(CaselessLiteral("accountid:")) +
Word(alphanums + ":-").setResultsName("accountid") + "]", )
return ((StringStart()
| Optional(PrecededBy(White(), retreat=1), default=" ")) +
MENTION.setParseAction(self.action) +
(StringEnd() | Optional(FollowedBy(
White() | Char(punctuation, excludeChars="[") | MENTION),
default=" ")))
def expr(self) -> ParserElement:
NON_ALPHANUMS = Regex(r"\W", flags=re.UNICODE)
TOKEN = Suppress(self.TOKEN)
IGNORE = White() + TOKEN | self.get_ignore_expr()
ELEMENT = Combine(
TOKEN + (~White() & ~Char(self.TOKEN)) +
SkipTo(TOKEN, ignore=IGNORE, failOn="\n") + TOKEN +
FollowedBy(NON_ALPHANUMS | StringEnd()), )
return (StringStart()
| PrecededBy(NON_ALPHANUMS, retreat=1)) + Combine(
ELEMENT.setParseAction(self.action) +
Optional(~ELEMENT, default=" "), )
def define_math(self):
digits = Word(nums)
variable = Word(alphas +
self._grammar.get_token("namespace_separator"))
to_dot = lambda t: "."
decimal_sep = Literal(self._grammar.get_token("decimal_separator"))
decimal_sep.setParseAction(to_dot)
thousands_sep = Suppress(
self._grammar.get_token("thousands_separator"))
thousands = Word(
nums, max=3) + OneOrMore(thousands_sep + Word(nums, exact=3))
integers = thousands | digits
decimals = decimal_sep + digits
expop = Literal('^')
signop = oneOf('+ -')
multop = oneOf('* /')
plusop = oneOf('+ -')
factop = Literal('!')
modop = Literal('%')
operand = Combine((integers + Optional(decimals)) | variable)
expr = operatorPrecedence(operand, [
(Literal("["), 1, opAssoc.RIGHT),
(Literal("]"), 1, opAssoc.LEFT),
("!", 1, opAssoc.LEFT),
(expop, 2, opAssoc.RIGHT),
(signop, 1, opAssoc.RIGHT),
(multop, 2, opAssoc.LEFT),
(modop, 2, opAssoc.LEFT),
(plusop, 2, opAssoc.LEFT),
])
expr = Combine(expr)
expr.setParseAction(self.make_arithmetic)
expr.setName("arithmetic")
return expr
def expr(self) -> ParserElement:
INTENSITY = Word(nums)
ALPHA = Word(nums + ".")
SEP = "," + Optional(White())
RGBA = (CaselessLiteral("rgba(") + INTENSITY.setResultsName("red") +
SEP + INTENSITY.setResultsName("green") + SEP +
INTENSITY.setResultsName("blue") + SEP + ALPHA + ")")
COLOR = Word("#", hexnums) ^ Word(alphas) ^ RGBA
expr = Combine(
"{color:" + COLOR.setResultsName("color") + "}" +
SkipTo("{color}").setResultsName("text") + "{color}", )
return expr.setParseAction(self.action)
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack,rbrack,lbrace,rbrace,lparen,rparen,colon,qmark = map(Literal,"[]{}():?")
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t"
reRange = Combine(lbrack + SkipTo(rbrack,ignore=escapedChar) + rbrack)
reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) )
reNonCaptureGroup = Suppress("?:")
reDot = Literal(".")
repetition = (
( lbrace + Word(nums)("count") + rbrace ) |
( lbrace + Word(nums)("minCount")+","+ Word(nums)("maxCount") + rbrace ) |
oneOf(list("*+?"))
)
reRange.setParseAction(handleRange)
reLiteral.setParseAction(handleLiteral)
reMacro.setParseAction(handleMacro)
reDot.setParseAction(handleDot)
reTerm = ( reLiteral | reRange | reMacro | reDot | reNonCaptureGroup)
reExpr = infixNotation( reTerm,
[
(repetition, 1, opAssoc.LEFT, handleRepetition),
(None, 2, opAssoc.LEFT, handleSequence),
(Suppress('|'), 2, opAssoc.LEFT, handleAlternative),
]
)
_parser = reExpr
return _parser
def _simple_number():
s = Combine(optional_sign + usn)
s.setParseAction(lambda s, l, tok: SimpleNumber(tok[0]))
return s
raise ParseException(loc, "invalid float format %s"%toks[0])
exponent = CaselessLiteral("e")+Optional(sign)+Word(nums)
#note that almost all these fields are optional,
#and this can match almost anything. We rely on Pythons built-in
#float() function to clear out invalid values - loosely matching like this
#speeds up parsing quite a lot
floatingPointConstant = Combine(
Optional(sign) +
Optional(Word(nums)) +
Optional(Literal(".") + Optional(Word(nums)))+
Optional(exponent)
)
floatingPointConstant.setParseAction(convertToFloat)
number = floatingPointConstant
#same as FP constant but don't allow a - sign
nonnegativeNumber = Combine(
Optional(Word(nums)) +
Optional(Literal(".") + Optional(Word(nums)))+
Optional(exponent)
)
nonnegativeNumber.setParseAction(convertToFloat)
coordinate = number
#comma or whitespace can seperate values all over the place in SVG
maybeComma = Optional(Literal(',')).suppress()
#!/usr/bin/env python3
from pyparsing import Word, alphas, nums, Combine, oneOf
# Identifier
ident = Combine(oneOf("me you") + '.' + Word(alphas))
# ident.setParseAction(lambda s,l,t: [ eval(t[0]) ])
# literal values
val = Combine(Word(nums) + '.' + Word(nums)) |\
Word(nums)
val.setParseAction(lambda s, l, t: [float(t[0])])
val.setName("value")
# Arithmetic expression
expr = val + '+' + val |\
val + '-' + val |\
val + '*' + val |\
val + '/' + val |\
val
# Combinations of arithmetic expressions
exprs = expr + '+' + expr |\
expr + '-' + expr |\
expr + '*' + expr |\
expr + '/' + expr |\
expr
# Conditional expressions
c_expr = ident + '&' + ident |\
ident + '|' + ident |\
return query
NO_BRTS = printables.replace('(', '').replace(')', '')
SINGLE = Word(NO_BRTS.replace('*', ''))
WILDCARDS = Optional('*') + SINGLE + Optional('*') + WordEnd(wordChars=NO_BRTS)
QUOTED = quotedString.setParseAction(removeQuotes)
OPER_AND = CaselessLiteral('and')
OPER_OR = CaselessLiteral('or')
OPER_NOT = '-'
TERM = Combine(Optional(Word(alphas).setResultsName('meta') + ':') +
(QUOTED.setResultsName('query') |
WILDCARDS.setResultsName('query')))
TERM.setParseAction(createQ)
EXPRESSION = operatorPrecedence(TERM, [
(OPER_NOT, 1, opAssoc.RIGHT),
(OPER_OR, 2, opAssoc.LEFT),
(Optional(OPER_AND, default='and'), 2, opAssoc.LEFT)])
EXPRESSION.setParseAction(unionQ)
QUERY = OneOrMore(EXPRESSION) + StringEnd()
QUERY.setParseAction(unionQ)
def advanced_search(pattern):
"""Parse the grammar of a pattern
and build a queryset with it"""
query_parsed = QUERY.parseString(pattern)
def transform_human(text, variables=None):
"""Transform user input with given context.
Args:
text (str): User input.
variables (dict): Variables for purposes of substitution.
Returns:
A 2-tuple of: (A human-readable script that Script can parse,
A list of contextual information for tooltips, etc.)
"""
if variables is None:
variables = {} # No mutable default value.
# these are parseActions for pyparsing.
def str_literal_to_hex(s, loc, toks):
for i, t in enumerate(toks):
toks[i] = ''.join(['0x', t.encode('hex')])
return toks
def var_name_to_value(s, loc, toks):
for i, t in enumerate(toks):
val = variables.get(t[1:])
if val:
toks[i] = val
return toks
def implicit_opcode_to_explicit(s, loc, toks):
"""Add "OP_" prefix to an opcode."""
for i, t in enumerate(toks):
toks[i] = '_'.join(['OP', t])
return toks
def hex_to_formatted_hex(s, loc, toks):
"""Add "0x" prefix and ensure even length."""
for i, t in enumerate(toks):
new_tok = t
# Add '0x' prefix
if not t.startswith('0x'):
if t.startswith('x'):
new_tok = ''.join(['0', t])
else:
new_tok = ''.join(['0x', t])
# Even-length string
if len(new_tok) % 2 != 0:
new_tok = ''.join([new_tok[0:2], '0', new_tok[2:]])
toks[i] = new_tok
return toks
# ^ parseActions for pyparsing end here.
str_literal = QuotedString('"')
str_literal.setParseAction(str_literal_to_hex)
var_name = Combine(Word('$') + Word(pyparsing.alphas))
var_name.setParseAction(var_name_to_value)
# Here we populate the list of contextual tips.
# Explicit opcode names
op_names = [str(i) for i in OPCODE_NAMES.keys()]
op_names_explicit = ' '.join(op_names)
def is_small_int(op):
"""True if op is one of OP_1, OP_2, ...OP_16"""
try:
i = int(op[3:])
return True
except ValueError:
return False
op_names_implicit = ' '.join([i[3:] for i in op_names if not is_small_int(i)])
# Hex, implicit (e.g. 'a') and explicit (e.g. '0x0a')
explicit_hex = Combine(Word('0x') + Word(pyparsing.hexnums) + pyparsing.WordEnd())
implicit_hex = Combine(pyparsing.WordStart() + OneOrMore(Word(pyparsing.hexnums)) + pyparsing.WordEnd())
explicit_hex.setParseAction(hex_to_formatted_hex)
implicit_hex.setParseAction(hex_to_formatted_hex)
# Opcodes, implicit (e.g. 'ADD') and explicit (e.g. 'OP_ADD')
explicit_op = pyparsing.oneOf(op_names_explicit)
implicit_op = Combine(pyparsing.WordStart() + pyparsing.oneOf(op_names_implicit))
implicit_op.setParseAction(implicit_opcode_to_explicit)
contexts = pyparsing.Optional(var_name('Variable') |
str_literal('String literal') |
explicit_op('Opcode') |
implicit_op('Opcode') |
explicit_hex('Hex') |
implicit_hex('Hex'))
matches = [(i[0].asDict(), i[1], i[2]) for i in contexts.scanString(text)]
context_tips = []
for i in matches:
d = i[0]
if len(d.items()) == 0: continue
match_type, value = d.items()[0]
start = i[1]
end = i[2]
context_tips.append( (start, end, value, match_type) )
# Now we do the actual transformation.
s = text
s = var_name.transformString(s)
s = str_literal.transformString(s)
s = implicit_op.transformString(s)
s = implicit_hex.transformString(s)
s = explicit_hex.transformString(s)
return s, context_tips
def __init__(self):
self.ae = False
self.local_dict = None
self.f = None
self.user_functions = None
self.expr_stack = []
self.texpr_stack = []
# Define constants
self.constants = {}
# Define Operators
self.opn = {"+": operator.add,
"-": operator.sub,
"*": operator.mul,
"/": operator.truediv,
">": operator.gt,
">=": operator.ge,
"<": operator.lt,
"<=": operator.le,
"==": operator.eq,
"!=": operator.ne,
"|": operator.or_,
"&": operator.and_,
"!": operator.inv}
# Define xarray DataArray operators with 1 input parameter
self.xfn1 = {"angle": xr.ufuncs.angle,
"arccos": xr.ufuncs.arccos,
"arccosh": xr.ufuncs.arccosh,
"arcsin": xr.ufuncs.arcsin,
"arcsinh": xr.ufuncs.arcsinh,
"arctan": xr.ufuncs.arctan,
"arctanh": xr.ufuncs.arctanh,
"ceil": xr.ufuncs.ceil,
"conj": xr.ufuncs.conj,
"cos": xr.ufuncs.cos,
"cosh": xr.ufuncs.cosh,
"deg2rad": xr.ufuncs.deg2rad,
"degrees": xr.ufuncs.degrees,
"exp": xr.ufuncs.exp,
"expm1": xr.ufuncs.expm1,
"fabs": xr.ufuncs.fabs,
"fix": xr.ufuncs.fix,
"floor": xr.ufuncs.floor,
"frexp": xr.ufuncs.frexp,
"imag": xr.ufuncs.imag,
"iscomplex": xr.ufuncs.iscomplex,
"isfinite": xr.ufuncs.isfinite,
"isinf": xr.ufuncs.isinf,
"isnan": xr.ufuncs.isnan,
"isreal": xr.ufuncs.isreal,
"log": xr.ufuncs.log,
"log10": xr.ufuncs.log10,
"log1p": xr.ufuncs.log1p,
"log2": xr.ufuncs.log2,
"rad2deg": xr.ufuncs.rad2deg,
"radians": xr.ufuncs.radians,
"real": xr.ufuncs.real,
"rint": xr.ufuncs.rint,
"sign": xr.ufuncs.sign,
"signbit": xr.ufuncs.signbit,
"sin": xr.ufuncs.sin,
"sinh": xr.ufuncs.sinh,
"sqrt": xr.ufuncs.sqrt,
"square": xr.ufuncs.square,
"tan": xr.ufuncs.tan,
"tanh": xr.ufuncs.tanh,
"trunc": xr.ufuncs.trunc}
# Define xarray DataArray operators with 2 input parameter
self.xfn2 = {"arctan2": xr.ufuncs.arctan2,
"copysign": xr.ufuncs.copysign,
"fmax": xr.ufuncs.fmax,
"fmin": xr.ufuncs.fmin,
"fmod": xr.ufuncs.fmod,
"hypot": xr.ufuncs.hypot,
"ldexp": xr.ufuncs.ldexp,
"logaddexp": xr.ufuncs.logaddexp,
"logaddexp2": xr.ufuncs.logaddexp2,
"logicaland": xr.ufuncs.logical_and,
"logicalnot": xr.ufuncs.logical_not,
"logicalor": xr.ufuncs.logical_or,
"logicalxor": xr.ufuncs.logical_xor,
"maximum": xr.ufuncs.maximum,
"minimum": xr.ufuncs.minimum,
"nextafter": xr.ufuncs.nextafter}
# Define non-xarray DataArray operators with 2 input parameter
self.fn2 = {"percentile": np.percentile}
# Define xarray DataArray reduction operators
self.xrfn = {"all": xr.DataArray.all,
"any": xr.DataArray.any,
"argmax": xr.DataArray.argmax,
"argmin": xr.DataArray.argmin,
"max": xr.DataArray.max,
"mean": xr.DataArray.mean,
"median": xr.DataArray.median,
"min": xr.DataArray.min,
"prod": xr.DataArray.prod,
"sum": xr.DataArray.sum,
"std": xr.DataArray.std,
"var": xr.DataArray.var}
# Define non-xarray DataArray operators with 2 input parameter
self.xcond = {"<": np.percentile}
# Define Grammar
point = Literal(".")
e = CaselessLiteral("E")
fnumber = Combine(Word("+-"+nums, nums) +
Optional(point + Optional(Word(nums))) +
Optional(e + Word("+-"+nums, nums)))
variable = Word(alphas, alphas+nums+"_$")
seq = Literal("=")
b_not = Literal("~")
plus = Literal("+")
minus = Literal("-")
mult = Literal("*")
div = Literal("/")
gt = Literal(">")
gte = Literal(">=")
lt = Literal("<")
lte = Literal("<=")
eq = Literal("==")
neq = Literal("!=")
b_or = Literal("|")
b_and = Literal("&")
l_not = Literal("!")
lpar = Literal("(").suppress()
rpar = Literal(")").suppress()
comma = Literal(",")
colon = Literal(":")
lbrac = Literal("[")
rbrac = Literal("]")
lcurl = Literal("{")
rcurl = Literal("}")
qmark = Literal("?")
scolon = Literal(";")
addop = plus | minus
multop = mult | div
sliceop = colon
compop = gte | lte | gt | lt
eqop = eq | neq
bitcompop = b_or | b_and
bitnotop = b_not
logicalnotop = l_not
assignop = seq
expop = Literal("^")
expr = Forward()
indexexpr = Forward()
atom = (Optional("-") +
(variable + seq + expr).setParseAction(self.push_assign) |
indexexpr.setParseAction(self.push_index) |
(lpar + expr + qmark.setParseAction(self.push_ternary1) + expr +
scolon.setParseAction(self.push_ternary2) + expr +
rpar).setParseAction(self.push_ternary) |
(lpar + expr + qmark + expr + scolon + expr +
rpar).setParseAction(self.push_ternary) |
(logicalnotop + expr).setParseAction(self.push_ulnot) |
(bitnotop + expr).setParseAction(self.push_unot) |
(minus + expr).setParseAction(self.push_uminus) |
(variable + lcurl + expr +
rcurl).setParseAction(self.push_mask) |
(variable + lpar + expr + (comma + expr)*3 +
rpar).setParseAction(self.push_expr4) |
(variable + lpar + expr + (comma + expr)*2 +
rpar).setParseAction(self.push_expr3) |
(variable + lpar + expr + comma + expr +
rpar).setParseAction(self.push_expr2) |
(variable + lpar + expr + rpar |
variable).setParseAction(self.push_expr1) |
fnumber.setParseAction(self.push_expr) |
(lpar + expr + ZeroOrMore(comma + expr).setParseAction(self.get_tuple) +
rpar).setParseAction(self.push_tuple) |
(lpar + expr.suppress() +
rpar).setParseAction(self.push_uminus))
# Define order of operations for operators
factor = Forward()
factor << atom + ZeroOrMore((expop + factor).setParseAction(self.push_op))
term = factor + ZeroOrMore((multop + factor).setParseAction(self.push_op))
term2 = term + ZeroOrMore((addop + term).setParseAction(self.push_op))
term3 = term2 + ZeroOrMore((sliceop + term2).setParseAction(self.push_op))
term4 = term3 + ZeroOrMore((compop + term3).setParseAction(self.push_op))
term5 = term4 + ZeroOrMore((eqop + term4).setParseAction(self.push_op))
term6 = term5 + ZeroOrMore((bitcompop + term5).setParseAction(self.push_op))
expr << term6 + ZeroOrMore((assignop + term6).setParseAction(self.push_op))
# Define index operators
colon_expr = (colon + FollowedBy(comma) ^ colon +
FollowedBy(rbrac)).setParseAction(self.push_colon)
range_expr = colon_expr | expr | colon
indexexpr << (variable + lbrac + delimitedList(range_expr, delim=',') +
rbrac).setParseAction(self.push_expr)
self.parser = expr
def _grammar():
from pyparsing import alphas, alphanums, nums
from pyparsing import oneOf, Suppress, Optional, Group, ZeroOrMore, NotAny
from pyparsing import Forward, operatorPrecedence, opAssoc, Word, White
from pyparsing import delimitedList, Combine, Literal, OneOrMore
expression = Forward()
LPAR, RPAR, DOT, LBRAC, RBRAC = map(Suppress, "().{}")
nw = NotAny(White())
identifier = Word(alphas + "_", alphanums + "_")
integer = Word(nums)
integer.setParseAction(IntegerNode)
fractional = Combine(Word('+' + '-' + nums, nums) + '.' + Word(nums))
fractional.setParseAction(FloatNode)
literal = fractional | integer
arglist = delimitedList(expression)
seqrange = LBRAC + expression + Suppress('..') + expression + RBRAC
seqrange.setParseAction(lambda t: SequenceNode(start=t[0], stop=t[1]))
seqexplicit = LBRAC + Optional(arglist) + RBRAC
seqexplicit.setParseAction(lambda t: SequenceNode(lst=t))
sequence = seqrange | seqexplicit
rollmod = nw + Group(oneOf("d k r e x") + Optional(integer))
numdice = Optional(integer, default=1)
roll = numdice + nw + Suppress("d") + nw + (integer | sequence)
roll += Group(ZeroOrMore(rollmod))
roll.setParseAction(DieRollNode)
call = LPAR + Group(Optional(arglist)) + RPAR
function = identifier + call
function.setParseAction(FunctionNode)
seqexpr = ((roll | sequence | function)
+ Group(OneOrMore(DOT + identifier + call)))
seqexpr.setParseAction(SeqMethodNode)
variable = Word(alphas + "_", alphanums + "_ ")
variable.setParseAction(VariableNode)
atom = seqexpr | roll | literal | sequence | function | variable
expoop = Literal('^')
signop = oneOf("+ -")
multop = oneOf("* /")
plusop = oneOf("+ -")
# noinspection PyUnresolvedReferences
expression << operatorPrecedence(
atom,
[
(expoop, 2, opAssoc.LEFT, BinaryOpNode),
(signop, 1, opAssoc.RIGHT, UnaryOpNode),
(multop, 2, opAssoc.LEFT, BinaryOpNode),
(plusop, 2, opAssoc.LEFT, BinaryOpNode),
]
)
return expression
sizing the data with the NUMBER similar to Dan Bernstein's netstrings
setup.
SPACE White space is basically ignored. This is interesting because since
Stackish is serialized consistently this means you can use \n as the
separation character and perform reasonable diffs on two structures.
"""
from pyparsing import Suppress,Word,nums,alphas,alphanums,Combine,oneOf,\
Optional,QuotedString,Forward,Group,ZeroOrMore,printables,srange
MARK,UNMARK,AT,COLON,QUOTE = map(Suppress,"[]@:'")
NUMBER = Word(nums)
NUMBER.setParseAction(lambda t:int(t[0]))
FLOAT = Combine(oneOf("+ -") + Word(nums) + "." + Optional(Word(nums)))
FLOAT.setParseAction(lambda t:float(t[0]))
STRING = QuotedString('"', multiline=True)
WORD = Word(alphas,alphanums+"_:")
ATTRIBUTE = Combine(AT + WORD)
strBody = Forward()
def setBodyLength(tokens):
strBody << Word(srange(r'[\0x00-\0xffff]'), exact=int(tokens[0]))
return ""
BLOB = Combine(QUOTE + Word(nums).setParseAction(setBodyLength) +
COLON + strBody + QUOTE)
item = Forward()
def assignUsing(s):
def assignPA(tokens):
if s in tokens:
def _unsigned_integer():
s = Combine(Optional("+") + Word(nums))
s.setParseAction(lambda s, l, tok: SimpleInteger(tok[0]))
return s
model_reference = Regex(r'([\w\.]*#[\w\.]+)')
variable = Regex(r'([a-zA-Z0-9\._]+)')
string = QuotedString('"', escChar="\\") | QuotedString('\'', escChar="\\")
operand = model_reference | real | integer | constant | string | variable
plusop = oneOf('+ -')
multop = oneOf('* / // %')
groupop = Literal(',')
expr = Forward()
modifier = Combine(Word(alphas + nums) + ':')
integer.setParseAction(EvalInteger)
real.setParseAction(EvalReal)
string.setParseAction(EvalString)
constant.setParseAction(EvalConstant)
variable.setParseAction(EvalVariable)
model_reference.setParseAction(EvalModelReference)
comparisonop = (oneOf("< <= > >= != == ~= ^= $=") |
(Literal('not in') + WordEnd()) |
(oneOf("in lt lte gt gte matches contains icontains like") + WordEnd()))
logicopOR = Literal('or') + WordEnd()
logicopAND = Literal('and') + WordEnd()
expr << operatorPrecedence(operand, [
(modifier, 1, opAssoc.RIGHT, EvalModifierOp),
'''
ParsingTmp.keywords.append(x)
W = Where
O = Optional
S = Suppress
number = Word(nums)
point = Literal('.')
e = CaselessLiteral('E')
plusorminus = Literal('+') | Literal('-')
integer = Combine(O(plusorminus) + number)
floatnumber = Combine(integer + (point + O(number)) ^ (e + integer))
integer.setParseAction(lambda tokens: SimpleRValue(int(tokens[0])))
floatnumber.setParseAction(lambda tokens: SimpleRValue(float(tokens[0])))
pi = Keyword('pi').setParseAction(lambda tokens: SimpleRValue(math.pi, 'pi')) #@UnusedVariable
isnumber = lambda x: isinstance(x, Number)
rvalue = Forward()
rvalue.setName('rvalue')
contract_expression = Forward()
contract_expression.setName('contract')
simple_contract = Forward()
simple_contract.setName('simple_contract')
# Import all expressions -- they will call add_contract()
from .library import (EqualTo, Unary, Binary, composite_contract,
identifier_contract, misc_variables_contract,
int_variables_contract, int_variables_ref,
def create_bnf(stack):
point = Literal(".")
comma = Literal(",")
e = CaselessLiteral("E")
inumber = Word(nums)
fnumber = Combine(Word("+-"+nums, nums) +
Optional(point + Optional(Word(nums))) +
Optional(e + Word("+-"+nums, nums)))
_of = Literal('of')
_in = Literal('in')
_by = Literal('by')
_copy = Literal('copy')
_mv = Literal('-v').setParseAction(replace('OA_SubV'))
_me = Literal('-e').setParseAction(replace('OA_SubE'))
_mf = Literal('-f').setParseAction(replace('OA_SubF'))
_mc = Literal('-c').setParseAction(replace('OA_SubC'))
_ms = Literal('-s').setParseAction(replace('OA_SubS'))
_pv = Literal('+v').setParseAction(replace('OA_AddV'))
_pe = Literal('+e').setParseAction(replace('OA_AddE'))
_pf = Literal('+f').setParseAction(replace('OA_AddF'))
_pc = Literal('+c').setParseAction(replace('OA_AddC'))
_ps = Literal('+s').setParseAction(replace('OA_AddS'))
_inv = Literal('*v').setParseAction(replace('OA_IntersectV'))
_ine = Literal('*e').setParseAction(replace('OA_IntersectE'))
_inf = Literal('*f').setParseAction(replace('OA_IntersectF'))
_inc = Literal('*c').setParseAction(replace('OA_IntersectC'))
_ins = Literal('*s').setParseAction(replace('OA_IntersectS'))
regop = (_mv | _me | _mf | _mc | _ms |
_pv | _pe | _pf | _pc | _ps |
_inv | _ine | _inf | _inc | _ins)
lpar = Literal("(").suppress()
rpar = Literal(")").suppress()
_all = Literal('all').setParseAction(replace('KW_All'))
vertex = Literal('vertex')
vertices = Literal('vertices')
cell = Literal('cell')
cells = Literal('cells')
group = Literal('group')
_set = Literal('set')
surface = Literal('surface')
ident = Word(alphas + '_.', alphanums + '_.')
set_name = Word(nums) | ident
function = Word(alphas + '_', alphanums + '_')
function = Group(function).setParseAction(join_tokens)
region = Combine(Literal('r.') + Word(alphas + '_',
'_' + alphas + nums + '.'))
region = Group(Optional(_copy, default='nocopy') + region)
region.setParseAction(replace('KW_Region', keep=True))
coor = oneOf('x y z')
boolop = oneOf('& |')
relop = oneOf('< > <= >= != ==')
bool_term = (ZeroOrMore('(') + (coor | fnumber) + relop + (coor | fnumber)
+ ZeroOrMore(')'))
relation = Forward()
relation << (ZeroOrMore('(')
+ bool_term + ZeroOrMore(boolop + relation)
+ ZeroOrMore(')'))
relation = Group(relation).setParseAction(join_tokens)
nos = Group(vertices + _of + surface).setParseAction(replace('E_VOS'))
nir = Group(vertices + _in + relation).setParseAction(
replace('E_VIR', keep=True))
nbf = Group(vertices + _by + function).setParseAction(
replace('E_VBF', keep=True))
ebf = Group(cells + _by + function).setParseAction(
replace('E_CBF', keep=True))
eog = Group(cells + _of + group + Word(nums)).setParseAction(
replace('E_COG', keep=True))
nog = Group(vertices + _of + group + Word(nums)).setParseAction(
replace('E_VOG', keep=True))
onir = Group(vertex + _in + region).setParseAction(
replace_with_region('E_OVIR', 2))
ni = Group(vertex + delimitedList(inumber)).setParseAction(
replace('E_VI', keep=True))
ei1 = Group(cell + delimitedList(inumber)).setParseAction(
replace('E_CI1', keep=True))
etuple = (lpar.suppress() + inumber + comma.suppress()
+ inumber + rpar.suppress())
ei2 = Group(cell + delimitedList(etuple)).setParseAction(
replace('E_CI2', keep=True))
noset = Group(vertices + _of + _set + set_name).setParseAction(
replace('E_VOSET', keep=True))
eoset = Group(cells + _of + _set + set_name).setParseAction(
replace('E_COSET', keep=True))
region_expression = Forward()
atom1 = (_all | region | ni | onir | nos | nir | nbf
| ei1 | ei2 | ebf | eog | nog | noset | eoset)
atom1.setParseAction(to_stack(stack))
atom2 = (lpar + region_expression.suppress() + rpar)
atom = (atom1 | atom2)
aux = (regop + region_expression)
aux.setParseAction(to_stack(stack))
region_expression << atom + ZeroOrMore(aux)
region_expression = StringStart() + region_expression + StringEnd()
return region_expression
return _handle_id(tokens[0])
elif tokens[1] == ".":
return Dot(_handle_id(tokens[0]), _handle_ci(tokens[2:]))
else: # tokens[1] == "["
return _handle_ci([ArrayAccess(_handle_id(tokens[0]), tokens[2])] + tokens[4:])
complex_identifier.setParseAction(lambda s, l, t: _handle_ci(t))
_define_identifier = complex_identifier
_variable_identifier = complex_identifier
# Integer numbers
_integer_number = Combine(Optional("-") + PWord(nums))
_integer_number.setParseAction(lambda s, l, t: int(t[0]))
# Constants
_boolean_constant = oneOf("TRUE FALSE")
_boolean_constant.setParseAction(lambda s, l, t: Falseexp() if t[0] == "FALSE" else Trueexp())
_integer_constant = _integer_number
_symbolic_constant = identifier
_range_constant = _integer_number + Suppress("..") + _integer_number
_range_constant.setParseAction(lambda s, l, t: RangeConst(t[0], t[1]))
_word_sign_specifier = oneOf("u s")
_word_base = oneOf("b B o O d D h H")
_word_width = PWord(nums)
_word_value = PWord("0123456789abcdefABCDEF", "0123456789abcdefABCDEF_")
_word_constant = Combine(
res += other
if DEBUG:
print "CollectionOut", res
return [res]
# SPARQL Grammar from http://www.w3.org/TR/sparql11-query/#grammar
# ------ TERMINALS --------------
# [139] IRIREF ::= '<' ([^<>"{}|^`\]-[#x00-#x20])* '>'
IRIREF = Combine(
Suppress('<') + Regex(r'[^<>"{}|^`\\%s]*' % ''.join('\\x%02X' % i
for i in range(33))) +
Suppress('>'))
IRIREF.setParseAction(lambda x: rdflib.URIRef(x[0]))
# [164] P_CHARS_BASE ::= [A-Z] | [a-z] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x02FF] | [#x0370-#x037D] | [#x037F-#x1FFF] | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF]
if sys.maxunicode == 0xffff:
# this is narrow python build (default on windows/osx)
# this means that unicode code points over 0xffff are stored
# as several characters, which in turn means that regex character
# ranges with these characters do not work.
# See
# * http://bugs.python.org/issue12729
# * http://bugs.python.org/issue12749
# * http://bugs.python.org/issue3665
#
# Here we simple skip the [#x10000-#xEFFFF] part
# this means that some SPARQL queries will not parse :(
def _BNF(self):
base16 = Literal("$")
hex = Combine(base16 + Word(hexnums + "_"))
base4 = Literal("%%")
quaternary = Combine(base4 + Word("0123_"))
base2 = Literal("%")
binary = Combine(base2 + Word("01_"))
plusminus = Literal("+") | Literal("-")
integer = Combine(Optional(plusminus) + Word(nums + "_"))
name_token = Combine(
Optional(Literal(":") | Literal("@")) +
Word("_" + alphas, "_" + alphanums))
name_token.setParseAction(self._mark_name_token)
lparens = Literal("(").suppress()
rparens = Literal(")").suppress()
# op0 = Literal("@")
op1 = (Literal("^^") | Literal("||") | Literal("|<") | Literal(">|")
| Literal("!")).setParseAction(self._mark_unary)
op2 = Literal("->") | Literal("<-") | Literal(">>") | Literal(
"<<") | Literal("~>") | Literal("><")
op3 = Literal("&")
op4 = Literal("|") | Literal("^")
op5 = Literal("**") | Literal("*") | Literal("//") | Literal("/")
op6 = Literal("+") | Literal("-")
op7 = Literal("#>") | Literal("<#")
op8 = Literal("<") | Literal(">") | Literal("<>") | Literal(
"==") | Literal("=<") | Literal("=>")
op9 = Literal("NOT").setParseAction(self._mark_unary)
op10 = Literal("AND")
op11 = Literal("OR")
op12 = Literal(",")
expr = Forward()
atom = name_token | hex | quaternary | binary | integer | quotedString
atom.setParseAction(self._push)
atom = atom | (lparens + expr.suppress() + rparens)
# term0 = atom + ZeroOrMore((op0 + atom) .setParseAction(self._push))
# term1 = term0 + ZeroOrMore((op1 + term0) .setParseAction(self._push))
term1 = atom + ZeroOrMore((op1 + atom).setParseAction(self._push))
term2 = term1 + ZeroOrMore((op2 + term1).setParseAction(self._push))
term3 = term2 + ZeroOrMore((op3 + term2).setParseAction(self._push))
term4 = term3 + ZeroOrMore((op4 + term3).setParseAction(self._push))
term5 = term4 + ZeroOrMore((op5 + term4).setParseAction(self._push))
term6 = term5 + ZeroOrMore((op6 + term5).setParseAction(self._push))
term7 = term6 + ZeroOrMore((op7 + term6).setParseAction(self._push))
term8 = term7 + ZeroOrMore((op8 + term7).setParseAction(self._push))
term9 = term8 + ZeroOrMore((op9 + term8).setParseAction(self._push))
term10 = term9 + ZeroOrMore((op10 + term9).setParseAction(self._push))
term11 = term10 + ZeroOrMore(
(op11 + term10).setParseAction(self._push))
expr << term11 + ZeroOrMore((op12 + term11).setParseAction(self._push))
return expr
if res:
res += [res[-3], rdflib.RDF.rest, b, b, rdflib.RDF.first, x]
else:
res += [b, rdflib.RDF.first, x]
res += [b, rdflib.RDF.rest, rdflib.RDF.nil]
if DEBUG:
print "CollectionOut", res
return [res]
# SPARQL Grammar from http://www.w3.org/TR/sparql11-query/#grammar
# ------ TERMINALS --------------
# [139] IRIREF ::= '<' ([^<>"{}|^`\]-[#x00-#x20])* '>'
IRIREF = Combine(Suppress("<") + Regex(r'[^<>"{}|^`\\%s]*' % "".join("\\x%02X" % i for i in range(33))) + Suppress(">"))
IRIREF.setParseAction(lambda x: rdflib.URIRef(x[0]))
# [164] P_CHARS_BASE ::= [A-Z] | [a-z] | [#x00C0-#x00D6] | [#x00D8-#x00F6] | [#x00F8-#x02FF] | [#x0370-#x037D] | [#x037F-#x1FFF] | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF]
if sys.maxunicode == 0xFFFF:
# this is narrow python build (default on windows/osx)
# this means that unicode code points over 0xffff are stored
# as several characters, which in turn means that regex character
# ranges with these characters do not work.
# See
# * http://bugs.python.org/issue12729
# * http://bugs.python.org/issue12749
# * http://bugs.python.org/issue3665
#
# Here we simple skip the [#x10000-#xEFFFF] part
# this means that some SPARQL queries will not parse :(
Create pyparsing grammar for problem configuration and options.
"""
from pyparsing import (Word, Group, Suppress, Combine, Optional,
Forward, Empty, quotedString, oneOf, removeQuotes,
delimitedList, nums, alphas, alphanums,
Keyword, CaselessLiteral)
word_free = Word(alphas + '][@_-/.+**' + alphanums)
word_strict = Word(alphas, alphas + alphanums + '_' )
(lparen, rparen, lbrack, rbrack,
lbrace, rbrace, colon, equal_sign) = map(Suppress, '()[]{}:=')
integer = Combine(Optional(oneOf('+ -')) + Word(nums)).setName('integer')
cvt_int = lambda toks: int(toks[0])
integer.setParseAction(cvt_int)
boolean_true = Keyword('True', caseless=True)
boolean_true.setParseAction(lambda x: True)
boolean_false = Keyword('False', caseless=True)
boolean_false.setParseAction(lambda x: False)
boolean = boolean_true | boolean_false
none = Keyword('None', caseless=True)
cvt_none = lambda toks: [None]
none.setParseAction(cvt_none)
e = CaselessLiteral("e")
real = (Combine(Optional(oneOf('+ -')) + Word(nums)
sizing the data with the NUMBER similar to Dan Bernstein's netstrings
setup.
SPACE White space is basically ignored. This is interesting because since
Stackish is serialized consistently this means you can use \n as the
separation character and perform reasonable diffs on two structures.
"""
from pyparsing import Suppress,Word,nums,alphas,alphanums,Combine,oneOf,\
Optional,QuotedString,Forward,Group,ZeroOrMore,printables,srange
MARK, UNMARK, AT, COLON, QUOTE = map(Suppress, "[]@:'")
NUMBER = Word(nums)
NUMBER.setParseAction(lambda t: int(t[0]))
FLOAT = Combine(oneOf("+ -") + Word(nums) + "." + Optional(Word(nums)))
FLOAT.setParseAction(lambda t: float(t[0]))
STRING = QuotedString('"', multiline=True)
WORD = Word(alphas, alphanums + "_:")
ATTRIBUTE = Combine(AT + WORD)
strBody = Forward()
def setBodyLength(tokens):
strBody << Word(srange(r'[\0x00-\0xffff]'), exact=int(tokens[0]))
return ""
BLOB = Combine(QUOTE + Word(nums).setParseAction(setBodyLength) + COLON +
strBody + QUOTE)
def parsebinop(opexpr):
"parse action for binary operators"
left, opstr, right = opexpr
for opclass in binopclasses:
if opstr in opclass.literals:
return opclass(left, right)
binop = oneOf(binopstr)
arithSign = Word("+-", exact=1)
realNum = Combine(Optional(arithSign) +
(Word(nums) + "." + Optional(Word(nums)) | ("." + Word(nums))) # noqa
+ Optional(E + Optional(arithSign) + Word(nums)))
realNum.setParseAction(lambda x: expression.NumericLiteral(float(x[0])))
intNum = Combine(Optional(arithSign) + Word(nums) +
Optional(E + Optional("+") + Word(nums)))
intNum.setParseAction(lambda x: expression.NumericLiteral(int(x[0])))
number = realNum | intNum
variable = ident.copy()
variable.setParseAction(lambda x: model.Var(x[0]))
quotedString.setParseAction(lambda x: expression.StringLiteral(x[0][1:-1]))
literal = quotedString | number
valueref = variable | literal
Optional( '.' + Word(nums)) +
Optional( Word('eE',exact=1) + Word(nums+'+-',nums)))
str_sexp = QuotedString('"', escChar='\\', unquoteResults=False)
bool_sexp = (TRUE | FALSE)
atom = (num_sexp | str_sexp | bool_sexp |
NONE)
extra_allowed_ident_chars = '+-=*/!?%'
var_sexp = Word(initChars=alphas + extra_allowed_ident_chars, bodyChars=alphanums + extra_allowed_ident_chars)
sexp << (atom | var_sexp | list_sexp).ignore(comment)
multi_sexp = OneOrMore(sexp)
def convert_num(s, l, toks):
n = toks[0]
try:
return int(n)
except ValueError:
return float(n)
num_sexp.setParseAction(convert_num)
def get_syntax_tree(fname):
infile = open(fname, 'r')
try:
file_text = infile.read()
return get_syntax_tree_from_str(file_text).asList()
finally:
infile.close()
def get_syntax_tree_from_str(str):
return multi_sexp.parseString(str, parseAll=True)
def create_bnf( stack ):
point = Literal( "." )
comma = Literal( "," )
e = CaselessLiteral( "E" )
inumber = Word( nums )
fnumber = Combine( Word( "+-"+nums, nums ) +
Optional( point + Optional( Word( nums ) ) ) +
Optional( e + Word( "+-"+nums, nums ) ) )
_of = Literal( 'of' )
_in = Literal( 'in' )
_by = Literal( 'by' )
_copy = Literal( 'copy' )
_mn = Literal( '-n' ).setParseAction( replace( 'OA_SubN' ) )
_me = Literal( '-e' ).setParseAction( replace( 'OA_SubE' ) )
_pn = Literal( '+n' ).setParseAction( replace( 'OA_AddN' ) )
_pe = Literal( '+e' ).setParseAction( replace( 'OA_AddE' ) )
_inn = Literal( '*n' ).setParseAction( replace( 'OA_IntersectN' ) )
_ine = Literal( '*e' ).setParseAction( replace( 'OA_IntersectE' ) )
regop = (_mn | _me | _pn | _pe | _inn | _ine)
lpar = Literal( "(" ).suppress()
rpar = Literal( ")" ).suppress()
_all = Literal( 'all' ).setParseAction( replace( 'KW_All' ) )
node = Literal( 'node' )
nodes = Literal( 'nodes' )
element = Literal( 'element' )
elements = Literal( 'elements' )
group = Literal( 'group' )
surface = Literal( 'surface' )
variable = Word( 'xyz', max = 1 ) | Literal( 'domain' )
any_var = Word( alphas + '_', alphanums + '_' ) | fnumber
function = Word( alphas, alphanums + '_' )
function = Group( function ).setParseAction( join_tokens )
region = Combine( Literal( 'r.' ) + Word( alphas, '_' + alphas + nums ) )
region = Group( Optional( _copy, default = 'nocopy' ) + region )
region.setParseAction( replace( 'KW_Region', keep = True ) )
coor = oneOf( 'x y z' )
boolop = oneOf( '& |' )
relop = oneOf( '< > <= >= != ==' )
bool_term = ZeroOrMore( '(' ) + (coor | fnumber ) + relop + (coor | fnumber)\
+ ZeroOrMore( ')' )
relation = Forward()
relation << ZeroOrMore( '(' )\
+ bool_term + ZeroOrMore( boolop + relation )\
+ ZeroOrMore( ')' )
relation = Group( relation ).setParseAction( join_tokens )
nos = Group( nodes + _of + surface ).setParseAction( replace( 'E_NOS' ) )
nir = Group( nodes + _in + relation ).setParseAction( \
replace( 'E_NIR', keep = True ) )
nbf = Group( nodes + _by + function ).setParseAction( \
replace( 'E_NBF', keep = True ) )
ebf = Group( elements + _by + function ).setParseAction( \
replace( 'E_EBF', keep = True ) )
eog = Group( elements + _of + group + Word( nums ) ).setParseAction( \
replace( 'E_EOG', keep = True ) )
nog = Group( nodes + _of + group + Word( nums ) ).setParseAction( \
replace( 'E_NOG', keep = True ) )
onir = Group( node + _in + region ).setParseAction( \
replace_with_region( 'E_ONIR', 2 ) )
ni = Group( node + delimitedList( inumber ) ).setParseAction( \
replace( 'E_NI', keep = True ) )
ei1 = Group( element + delimitedList( inumber ) ).setParseAction( \
replace( 'E_EI1', keep = True ) )
etuple = lpar.suppress() + inumber + comma.suppress() \
+ inumber + rpar.suppress()
ei2 = Group( element + delimitedList( etuple ) ).setParseAction( \
replace( 'E_EI2', keep = True ) )
region_expression = Forward()
atom1 = (_all | region | ni | onir | nos | nir | nbf
| ei1 | ei2 | ebf | eog | nog)
atom1.setParseAction( to_stack( stack ) )
atom2 = (lpar + region_expression.suppress() + rpar)
atom = (atom1 | atom2)
aux = (regop + region_expression)
aux.setParseAction( to_stack( stack ) )
region_expression << atom + ZeroOrMore( aux )
region_expression = StringStart() + region_expression + StringEnd()
# region.set_debug()
# relation.set_debug()
# region_expression.set_debug()
return region_expression
and the discussion of the "-" operator in the docs.
"""
ParsingTmp.keywords.append(x)
W = Where
O = Optional
S = Suppress
basenumber = Word(nums)
point = Literal('.')
e = CaselessLiteral('E')
plusorminus = Literal('+') | Literal('-')
integer = Combine(O(plusorminus) + basenumber)
integer.setParseAction(lambda tokens: SimpleRValue(int(tokens[0])))
floatnumber = Combine(
O(plusorminus) + integer + (point + O(basenumber)) ^ (e + integer))
floatnumber.setParseAction(lambda tokens: SimpleRValue(float(tokens[0])))
pi = Keyword('pi').setParseAction(
lambda tokens: SimpleRValue(math.pi, 'pi')) # @UnusedVariable
try:
import numpy
except ImportError:
numpy = None
def isnumber(x):
# These are scalar quantities that we can compare (=,>,>=, etc.)
if isinstance(x, Number):
equal_op = Literal( "=" )
not_equal_op = Literal( "!=" )
greater_op = Combine( Literal( ">" ) + ~Literal( "=" ) )
greater_equal_op = Combine( Literal( ">" ) + Literal( "=" ) )
less_op = Combine( Literal( "<" ) + ~Literal( "=" ) )
less_equal_op = Combine( Literal( "<" ) + Literal( "=" ) )
addop = plus | minus
multop = mult | div
compop = less_op | greater_op | less_equal_op | greater_equal_op | not_equal_op | equal_op
expop = Literal( "^" )
#assign = Literal( "=" )
band = Literal( "@" )
expr = Forward()
atom = ( ( e | floatnumber | integer | ident.setParseAction( assignVar ) | fn + lpar + expr + rpar | fn + lpar + expr + colon + expr + rpar | fn + lpar + expr + colon + expr + colon + expr + rpar ).setParseAction(pushFirst) |
( lpar + expr.suppress() + rpar )
)
factor = Forward()
factor << atom + ( ( band + factor ).setParseAction( pushFirst ) | ZeroOrMore( ( expop + factor ).setParseAction( pushFirst ) ) )
term = factor + ZeroOrMore( ( multop + factor ).setParseAction( pushFirst ) )
addterm = term + ZeroOrMore( ( addop + term ).setParseAction( pushFirst ) )
expr << addterm + ZeroOrMore( ( compop + addterm ).setParseAction( pushFirst ) )
bnf = expr
pattern = bnf + StringEnd()
# map operator symbols to corresponding arithmetic operations
opn = { "+" : ( lambda a,b: numpy.add( a, b ) ),
