def _getPattern(self):
arith_expr = Forward()
comp_expr = Forward()
logic_expr = Forward()
LPAR, RPAR, SEMI = map(Suppress, "();")
identifier = Word(alphas+"_", alphanums+"_")
multop = oneOf('* /')
plusop = oneOf('+ -')
expop = Literal( "^" )
compop = oneOf('> < >= <= != ==')
andop = Literal("AND")
orop = Literal("OR")
current_value = Literal( "." )
assign = Literal( "=" )
# notop = Literal('NOT')
function = oneOf(' '.join(self.FUNCTIONS))
function_call = Group(function.setResultsName('fn') + LPAR + Optional(delimitedList(arith_expr)) + RPAR)
aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}')
single_column = QuotedString(quoteChar='[', endQuoteChar=']')
integer = Regex(r"-?\d+")
real = Regex(r"-?\d+\.\d*")
# quotedString enables strings without quotes to pass
operand = \
function_call.setParseAction(self.__evalFunction) | \
aggregate_column.setParseAction(self.__evalAggregateColumn) | \
single_column.setParseAction(self.__evalSingleColumn) | \
((real | integer).setParseAction(self.__evalConstant)) | \
quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \
current_value.setParseAction(self.__evalCurrentValue) | \
identifier.setParseAction(self.__evalString)
arith_expr << operatorPrecedence(operand,
[
(expop, 2, opAssoc.LEFT, self.__expOp),
(multop, 2, opAssoc.LEFT, self.__multOp),
(plusop, 2, opAssoc.LEFT, self.__addOp),
])
# comp_expr = Group(arith_expr + compop + arith_expr)
comp_expr << operatorPrecedence(arith_expr,
[
(compop, 2, opAssoc.LEFT, self.__evalComparisonOp),
])
logic_expr << operatorPrecedence(comp_expr,
[
(andop, 2, opAssoc.LEFT, self.__evalLogicOp),
(orop, 2, opAssoc.LEFT, self.__evalLogicOp)
])
pattern = logic_expr + StringEnd()
return pattern
def __init__(self, EvaluateVariableChild=None, EvaluateNumberChild=None):
EvaluateVariableChild = EvaluateVariableChild or EvaluateVariable
EvaluateNumberChild = EvaluateNumberChild or EvaluateNumber
# what is a float number
floatNumber = Regex(r'[-]?\d+(\.\d*)?([eE][-+]?\d+)?')
# a variable is a combination of letters, numbers, and underscor
variable = Word(alphanums + "_")
# a sign is plus or minus
signOp = oneOf('+ -')
# an operand is a variable or a floating point number
operand = floatNumber ^ variable
# when a floatNumber is found, parse it with evaluate number
floatNumber.setParseAction(EvaluateNumberChild)
# when a variable is found, parse it with the EvaluateVariableChild
# or EvaluateVariable
variable.setParseAction(EvaluateVariableChild)
# comparisons include lt,le,gt,ge,eq,ne
comparisonOp = oneOf("< <= > >= == !=")
# negation of the boolean is !
notOp = oneOf("!")
# an expression is a either a comparison or
# a NOT operation (where NOT a is essentially (a == False))
comparisonExpression = operatorPrecedence(operand,
[
(comparisonOp,
2,
opAssoc.LEFT,
EvaluateComparison
),
(notOp,
1,
opAssoc.RIGHT,
EvaluateNot
),
])
# boolean logic of AND or OR
boolOp = oneOf("& |")
# a bool expression contains a nested bool expression or a comparison,
# joined with a boolean operation
boolExpression = Forward()
boolPossible = boolExpression | comparisonExpression
self.boolExpression = operatorPrecedence(boolPossible,
[
(boolOp,
2,
opAssoc.RIGHT,
EvaluateOrAnd
),
])
return
def _getPattern(self):
arith_expr = Forward()
comp_expr = Forward()
logic_expr = Forward()
LPAR, RPAR, SEMI = map(Suppress, "();")
identifier = Word(alphas + "_", alphanums + "_")
multop = oneOf('* /')
plusop = oneOf('+ -')
expop = Literal("^")
compop = oneOf('> < >= <= != ==')
andop = Literal("AND")
orop = Literal("OR")
current_value = Literal(".")
assign = Literal("=")
# notop = Literal('NOT')
function = oneOf(' '.join(self.FUNCTIONS))
function_call = Group(
function.setResultsName('fn') + LPAR +
Optional(delimitedList(arith_expr)) + RPAR)
aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}')
single_column = QuotedString(quoteChar='[', endQuoteChar=']')
integer = Regex(r"-?\d+")
real = Regex(r"-?\d+\.\d*")
# quotedString enables strings without quotes to pass
operand = \
function_call.setParseAction(self.__evalFunction) | \
aggregate_column.setParseAction(self.__evalAggregateColumn) | \
single_column.setParseAction(self.__evalSingleColumn) | \
((real | integer).setParseAction(self.__evalConstant)) | \
quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \
current_value.setParseAction(self.__evalCurrentValue) | \
identifier.setParseAction(self.__evalString)
arith_expr << operatorPrecedence(operand, [
(expop, 2, opAssoc.LEFT, self.__expOp),
(multop, 2, opAssoc.LEFT, self.__multOp),
(plusop, 2, opAssoc.LEFT, self.__addOp),
])
# comp_expr = Group(arith_expr + compop + arith_expr)
comp_expr << operatorPrecedence(arith_expr, [
(compop, 2, opAssoc.LEFT, self.__evalComparisonOp),
])
logic_expr << operatorPrecedence(
comp_expr, [(andop, 2, opAssoc.LEFT, self.__evalLogicOp),
(orop, 2, opAssoc.LEFT, self.__evalLogicOp)])
pattern = logic_expr + StringEnd()
return pattern
def buildMapSyntax(self):
'''Subset of grammar for DMM files.
"aai" = (/obj/structure/sign/securearea{desc = "A warning sign which reads 'HIGH VOLTAGE'"; icon_state = "shock"; name = "HIGH VOLTAGE"; pixel_y = -32},/turf/space,/area)
'''
dreamList = pyp.Forward()
# Literals
singlelineString = pyp.QuotedString('"', '\\').setResultsName('string').setParseAction(self.makeListString)
fileRef = pyp.QuotedString("'", '\\').setResultsName('fileRef').setParseAction(self.makeFileRef)
multilineString = pyp.QuotedString(quoteChar='{"', endQuoteChar='"}', multiline=True).setResultsName('string').setParseAction(self.makeListString)
number = pyp.Regex(r'\-?\d+(\.\d*)?([eE]\d+)?').setResultsName('number').setParseAction(self.makeListNumber)
# Paths
relpath = pyp.ident | relpath + SLASH + pyp.ident
abspath = SLASH + relpath
path = (abspath | relpath).setParseAction(self.handlePath)
pathslash = path + SLASH
# Other symbols
listStart = pyp.Suppress('list(')
openParen = pyp.Suppress("(")
closeParen = pyp.Suppress(')')
# Grammar
listConstant = singlelineString | fileRef | multilineString | number | dreamList | abspath
listElement = listConstant | (listConstant + '=' + listConstant)
listElement = pyp.operatorPrecedence(listElement, [
("=", 2, pyp.opAssoc.LEFT,),
])
listContents = pyp.delimitedList(listElement)
dreamList << pyp.Group(listStart + listContents + closeParen)
dreamList.setParseAction(self.makeList)
# DMM Atom definition
atomDefProperty = pyp.ident + "=" + listConstant
atomDefProperty = pyp.operatorPrecedence(atomDefProperty, [
("=", 2, pyp.opAssoc.LEFT,),
])
atomDefPropertyListContents = pyp.delimitedList(listElement, delim=';')
atomDefProperties = pyp.Suppress("{") + atomDefPropertyListContents + pyp.Suppress("}")
atomDef = abspath | abspath + atomDefProperties
# DMM Tile Definition
tileDefListContents = pyp.delimitedList(atomDef)
tileDefAtomList = openParen + tileDefListContents + closeParen
tileDef = singlelineString + '=' + tileDefAtomList
tileDef.setParseAction(self.makeTileDef)
return tileDef
def _algebra_to_ast(algebra_notation, inputs=None):
"""
Converts a Boolean Algebra expression to an abstract syntax tree.
"""
#convert each of the VHDL input terms into a grammar literal
if inputs:
inputTerms = reduce(lambda x, y : x | pyparsing.Literal(y.strip()), inputs)
else:
#create a basic identifier grammar element
IdentifierChars = pyparsing.alphanums + '_[]'
inputTerms = pyparsing.Word(IdentifierChars)
#define the algebra operators, in order of precedence
algebraOperators = \
[
("'", 1, pyparsing.opAssoc.LEFT), #NOT
(pyparsing.Optional("*", default='*'), 2, pyparsing.opAssoc.LEFT), #AND, including implied AND
("+", 2, pyparsing.opAssoc.LEFT), #OR
("^", 2, pyparsing.opAssoc.LEFT) #XOR
]
#define a new grammar in terms of the input terms and algebraic operators
algebraExpresion = pyparsing.operatorPrecedence(inputTerms, algebraOperators)
#use the newly created grammar to convert the boolean expression into an abstract syntax list
try:
ast = algebraExpresion.parseString(algebra_notation)[0]
except pyparsing.ParseException, e:
raise InvalidExpressionException("I couldn't figure out what you meant by '" + algebra_notation + "'.")
def _query_expression():
operand = quotedString.setParseAction(removeQuotes)
return operatorPrecedence(operand, [
(CaselessLiteral('not'), 1, opAssoc.RIGHT, _not_expression),
(CaselessLiteral('and'), 2, opAssoc.LEFT, _and_expression),
(CaselessLiteral('or'), 2, opAssoc.LEFT, _or_expression)
])
def parse(text ):
global index
index+=1
left_parenthesis, right_parenthesis = "(",")"
implies = Literal("=>")
or_ = Literal("|")
and_ = Literal("&")
not_ = Literal("~")
symbol = (Optional(not_)+ Word(alphas) + left_parenthesis + delimitedList(Word(alphas)) + right_parenthesis).setParseAction(actCombineWord) #, alphanums)
term = Forward()
term << (Group(symbol + Group(left_parenthesis +
delimitedList(term) + right_parenthesis)) | symbol)
#term << (Group(symbol + Group(left_parenthesis + symbol + right_parenthesis)) |symbol )
formula = Forward()
operand = term
formula << operatorPrecedence(operand, [
(and_, 2, opAssoc.LEFT),
(or_, 2, opAssoc.LEFT),
(implies, 2, opAssoc.RIGHT)])
try:
result = formula.parseString(text, parseAll=True)
assert len(result) == 1
return result[0].asList()
except (ParseException, ParseSyntaxException) as err:
print("Syntax error:\n{0.line}\n{1}^".format(err,
" " * (err.column - 1)))
return []
def _getParser(self):
if not self.parser:
period = pp.Literal('.').suppress()
lparen = pp.Literal('(').suppress()
rparen = pp.Literal(')').suppress()
forall = pp.Literal(self.symbols['forall'])
exists = pp.Literal(self.symbols['exists'])
bottom = pp.Literal(self.symbols['bottom'])
top = pp.Literal(self.symbols['top'])
uppers = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
lowers = uppers.lower()
conceptName = bottom | top | pp.Combine(pp.Word(uppers, exact=1) + pp.Word(pp.alphas))
relationName = pp.Combine(pp.Word(lowers, exact=1) + pp.Word(pp.alphas))
valueOp = pp.Group(pp.Optional(forall | exists, default=forall) + relationName + period)
dlExpression = pp.Forward()
dlOperand = lparen + dlExpression + rparen | conceptName
dlExpression << pp.operatorPrecedence(dlOperand,
[
(self.symbols['not'], 1, pp.opAssoc.RIGHT, ),
(valueOp, 1, pp.opAssoc.RIGHT, ),
(self.symbols['and'], 2, pp.opAssoc.LEFT, _toNested(2)),
(self.symbols['or'], 2, pp.opAssoc.LEFT, _toNested(2)),
])
self.parser = dlExpression
return self.parser
def parse_prereq_text(text):
number = pp.Regex(r"\d+")
expr = pp.operatorPrecedence(number, [
("&&", 2, pp.opAssoc.LEFT, ),
("||", 2, pp.opAssoc.LEFT, ),
])
return expr.parseString(text)
def _grammar():
from pyparsing import alphas, alphanums, nums
from pyparsing import oneOf, Suppress, Optional, Group, ZeroOrMore, NotAny
from pyparsing import Forward, operatorPrecedence, opAssoc, Word, White
from pyparsing import delimitedList, Combine, Literal, OneOrMore
expression = Forward()
LPAR, RPAR, DOT, LBRAC, RBRAC = map(Suppress, "().{}")
nw = NotAny(White())
identifier = Word(alphas + "_", alphanums + "_")
integer = Word(nums)
integer.setParseAction(IntegerNode)
fractional = Combine(Word('+' + '-' + nums, nums) + '.' + Word(nums))
fractional.setParseAction(FloatNode)
literal = fractional | integer
arglist = delimitedList(expression)
seqrange = LBRAC + expression + Suppress('..') + expression + RBRAC
seqrange.setParseAction(lambda t: SequenceNode(start=t[0], stop=t[1]))
seqexplicit = LBRAC + Optional(arglist) + RBRAC
seqexplicit.setParseAction(lambda t: SequenceNode(lst=t))
sequence = seqrange | seqexplicit
rollmod = nw + Group(oneOf("d k r e x") + Optional(integer))
numdice = Optional(integer, default=1)
roll = numdice + nw + Suppress("d") + nw + (integer | sequence)
roll += Group(ZeroOrMore(rollmod))
roll.setParseAction(DieRollNode)
call = LPAR + Group(Optional(arglist)) + RPAR
function = identifier + call
function.setParseAction(FunctionNode)
seqexpr = ((roll | sequence | function) +
Group(OneOrMore(DOT + identifier + call)))
seqexpr.setParseAction(SeqMethodNode)
variable = Word(alphas + "_", alphanums + "_ ")
variable.setParseAction(VariableNode)
atom = seqexpr | roll | literal | sequence | function | variable
expoop = Literal('^')
signop = oneOf("+ -")
multop = oneOf("* /")
plusop = oneOf("+ -")
# noinspection PyUnresolvedReferences
expression << operatorPrecedence(atom, [
(expoop, 2, opAssoc.LEFT, BinaryOpNode),
(signop, 1, opAssoc.RIGHT, UnaryOpNode),
(multop, 2, opAssoc.LEFT, BinaryOpNode),
(plusop, 2, opAssoc.LEFT, BinaryOpNode),
])
return expression
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack, rbrack, lbrace, rbrace, lparen, rparen = map(Literal, "[]{}()")
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~ reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t"
reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack)
reLiteral = (escapedChar | oneOf(list(reLiteralChar)))
reDot = Literal(".")
repetition = (
(lbrace + Word(nums).setResultsName("count") + rbrace) |
(lbrace + Word(nums).setResultsName("minCount") + "," + Word(nums).setResultsName("maxCount") + rbrace) |
oneOf(list("*+?"))
)
reRange.setParseAction(handle_range)
reLiteral.setParseAction(handle_literal)
reMacro.setParseAction(handle_macro)
reDot.setParseAction(handle_dot)
reTerm = (reLiteral | reRange | reMacro | reDot)
reExpr = operatorPrecedence(reTerm, [
(repetition, 1, opAssoc.LEFT, handle_repetition),
(None, 2, opAssoc.LEFT, handle_sequence),
(Suppress('|'), 2, opAssoc.LEFT, handle_alternative),
])
_parser = reExpr
return _parser
def parse(self):
col = pp.Word(pp.alphanums)
val = pp.Word(pp.alphanums)
comp = pp.Regex("==|!=|<=|>=|<|>")
cond = pp.Group(col + comp + val)
expr = pp.operatorPrecedence(cond, [("&&", 2, pp.opAssoc.LEFT), ("||", 2, pp.opAssoc.LEFT)])
return expr.parseString(self.cond).asList()[0]
def parse_condition(input_string):
r = get_redis_connection()
metrics_workload = r.keys("metric:*")
services = map(lambda x: "".join(x.split(":")[1]), metrics_workload)
services_options = oneOf(services)
operand = oneOf("< > == != <= >=")
number = Regex(r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?")
condition = Group(services_options + operand("operand") +
number("limit_value"))
condition_list = operatorPrecedence(condition, [
(
"AND",
2,
opAssoc.LEFT,
),
(
"OR",
2,
opAssoc.LEFT,
),
])
rule = condition_list('condition_list')
return rule.parseString(input_string).condition_list.asList()
def __init__(self, expression):
# define the parser
integer = Word(nums)
real = Combine(Word(nums) + "." + Word(nums))
# nums have been added to allow for aggregation variables such as `sum_250_field`
variable = Word(alphas + ":" + "_" + nums)
boolean = oneOf('true false', caseless=True)
num = integer | real
keyword = oneOf('time day period')
boolean.setParseAction(EvalBool)
variable.setParseAction(EvalVar)
num.setParseAction(EvalNum)
keyword.setParseAction(EvalKeyword)
bool_op = oneOf('&& ||')
sign_op = oneOf('+ -')
comparison_op = oneOf("< <= > >= != ==")
atom = boolean | keyword | num | variable | bool_op | sign_op | comparison_op
self.expr = operatorPrecedence(
atom, [(sign_op, 1, opAssoc.RIGHT, EvalSignOp),
(comparison_op, 2, opAssoc.LEFT, EvalComparisonOp),
(bool_op, 2, opAssoc.LEFT, EvalLogical)])
self.parsed = self.expr.parseString(expression)[0]
def _create_parser() -> Forward:
"""Create an instance of a dice roll string parser."""
atom = (CaselessLiteral("d%") | pyparsing_common.number
| CaselessKeyword("pi") | CaselessKeyword("e"))
expression = operatorPrecedence(
atom,
[
(Literal('-'), 1, opAssoc.RIGHT),
(CaselessLiteral('sqrt'), 1, opAssoc.RIGHT),
(oneOf('^ **'), 2, opAssoc.RIGHT),
(Literal('-'), 1, opAssoc.RIGHT),
(Literal('!'), 1, opAssoc.LEFT),
(CaselessLiteral('d%'), 1, opAssoc.LEFT),
(CaselessLiteral('d'), 2, opAssoc.RIGHT),
# This line causes the recursion debug to go off.
# Will have to find a way to have an optional left
# operator in this case.
(CaselessLiteral('d'), 1, opAssoc.RIGHT),
(oneOf('* / % //'), 2, opAssoc.LEFT),
(oneOf('+ -'), 2, opAssoc.LEFT),
])
return expression
def load_rules(filepath):
operator = pp.Regex(">=|<=|!=|>|<|==").setName("operator")
number = pp.Regex(r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?")
identifier = pp.Word(pp.alphas + "[]", pp.alphanums + "_[]")
comparison_term = identifier | number
a_comparison = comparison_term + operator + comparison_term
stmt = a_comparison | identifier
condition = pp.Group(stmt)
expr = pp.operatorPrecedence(condition,[
("NOT", 1, pp.opAssoc.RIGHT, ),
("AND", 2, pp.opAssoc.LEFT, ),
("OR", 2, pp.opAssoc.LEFT, ),
])
rules = []
with open(filepath,'r') as f:
lines = f.readlines()
for line in lines:
rule_and_acceptance = line.split("=>")
ptree = expr.parseString(rule_and_acceptance[0].strip())
acceptance = True if rule_and_acceptance[1].strip() == 'True' else False
rules.append((json.loads(ptree[0].dump().replace("\'","\"")), acceptance)) # make flat list
return rules
def pyparse_flo(text):
left_parenthesis, right_parenthesis, colon = map(Suppress, '():')
boolean = Keyword('false') | Keyword('true')
forall = Keyword('forall')
exists = Keyword('exists')
and_ = Literal('&')
or_ = Literal('|')
not_ = Literal('~')
equals = Literal('=')
implies = Literal('->')
symbol = Word(alphas, alphanums)
term = Forward()
term << (Group(symbol + Group(left_parenthesis + delimitedList(term) +
right_parenthesis)) | symbol)
formula = Forward()
forall_expression = forall + symbol + colon + formula
exists_expression = exists + symbol + colon + formula
operand = forall_expression | exists_expression | boolean | term
formula << operatorPrecedence(operand, [(equals, 2, opAssoc.RIGHT),
(not_, 1, opAssoc.RIGHT),
(and_, 2, opAssoc.LEFT),
(or_, 2, opAssoc.LEFT),
(implies, 2, opAssoc.RIGHT)])
try:
result = formula.parseString(text, parseAll=True)
assert len(result) == 1
return result[0].asList()
except (ParseException, ParseSyntaxException) as parse_err:
print('Syntax Error:\n{}\n{}^'.format(parse_err.line,
' ' * (parse_err.column - 1)))
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack, rbrack, lbrace, rbrace, lparen, rparen = map(Literal, "[]{}()")
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(
c for c in printables if c not in r"\[]{}().*?+|") + " \t"
reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack)
reLiteral = (escapedChar | oneOf(list(reLiteralChar)))
reDot = Literal(".")
repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) |
(lbrace + Word(nums).setResultsName("minCount") + "," +
Word(nums).setResultsName("maxCount") + rbrace)
| oneOf(list("*+?")))
reRange.setParseAction(handle_range)
reLiteral.setParseAction(handle_literal)
reMacro.setParseAction(handle_macro)
reDot.setParseAction(handle_dot)
reTerm = (reLiteral | reRange | reMacro | reDot)
reExpr = operatorPrecedence(reTerm, [
(repetition, 1, opAssoc.LEFT, handle_repetition),
(None, 2, opAssoc.LEFT, handle_sequence),
(Suppress('|'), 2, opAssoc.LEFT, handle_alternative),
])
_parser = reExpr
return _parser
def get_grammar():
global _grammar
if _grammar is None:
from pyparsing import (Literal, Word, QuotedString, Optional, operatorPrecedence,
nums, alphas, opAssoc, ParseResults)
# literals
AND = Literal("AND")
OR = Literal("OR")
NOT = Literal("NOT")
SPAN = (Literal("W/") + Word(nums).setResultsName("slop"))
OP = Optional(AND | OR | NOT | SPAN, default="implicit_OR").setResultsName("operator")
COLON = Literal(":").suppress()
TILDE = Literal("~").suppress()
LETTERS = u''.join(unichr(c) for c in xrange(65536)
if not unichr(c).isspace() and unichr(c) not in '":()~')
# terms
term = Word(LETTERS)
slop = Word(nums).setResultsName("slop")
quote = QuotedString('"').setResultsName("quote") + Optional(TILDE + slop)
#quote.setParseAction(Quote)
field = Word(alphas).setResultsName("field")
fterm = Optional(field + COLON) + (quote | term).setResultsName("term")
fterm.setParseAction(get_term)
# boolean combination
boolean_expr = operatorPrecedence(fterm, [
(OP, 2, opAssoc.LEFT)
])
boolean_expr.setParseAction(get_boolean_or_term)
_grammar = boolean_expr
return _grammar
def _parseFeatureSignature(sig):
"""This function parses a given feature-signature."""
mal = set()
def _rewriteOne(p):
return ""
def _rewriteTwo(p):
return ""
def _addIdentifier2Mal(p):
mal.add(p[0])
operand = __string | __hexadec | __function | __integer | __identifier.setParseAction(_addIdentifier2Mal)
compoperator = pypa.oneOf("< > <= >= == !=")
calcoperator = pypa.oneOf("+ - * / % & | << >>")
expr = pypa.operatorPrecedence(
operand,
[
("defined", 1, pypa.opAssoc.RIGHT, _rewriteOne),
("!", 1, pypa.opAssoc.RIGHT, _rewriteOne),
(calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo),
(compoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo),
("&&", 2, pypa.opAssoc.LEFT, _rewriteTwo),
("||", 2, pypa.opAssoc.LEFT, _rewriteTwo),
],
)
try:
rsig = expr.parseString(sig)[0]
except pypa.ParseException, e:
print("ERROR (parse): cannot parse sig (%s) -- (%s)" % (sig, e.col))
return sig
def get_parse_expression(parse_func, parse_var_id):
"""return a parse expression with for the
input parseActions
"""
var_id = Group(FUNC_NAME + COLON +
IDENTIFIER) | REAL | INTEGER | IDENTIFIER
var_id.setParseAction(parse_var_id)
# Forward declaration for an Arithmetic Expression
arith_expr = Forward()
func_call = Group(FUNC_NAME + LPAREN +
Optional(Group(delimitedList(arith_expr))) + RPAREN)
# An Arithmetic expression can have a var_id or
# a function call as an operand
# pylint: disable=expression-not-assigned
arith_expr << operatorPrecedence(func_call | var_id, [
(EXPONENTIATION_OPS, 2, opAssoc.LEFT, eval_binary_op),
(UNARY_OPS, 1, opAssoc.RIGHT, eval_unary_op),
(MULT_OPS, 2, opAssoc.LEFT, eval_binary_op),
(SUM_OPS, 2, opAssoc.LEFT, eval_binary_op),
(REL_OPS, 2, opAssoc.LEFT, eval_binary_op),
(LOGICAL_OPS, 2, opAssoc.LEFT, eval_binary_op)
])
# pylint: enable=expression-not-assigned
# Argument expression for a function call
# An argument to a function can be an
# IDENTIFIER, Arithmetic expression, REAL number, INTEGER or a
# Function call itself
func_call.setParseAction(parse_func)
return arith_expr
def __init__(self):
"""
BNF HERE
"""
#integer = pp.Word(nums)
#floatNumber = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?')
point = pp.Literal( "." )
e = pp.CaselessLiteral( "E" )
# Regex string representing the set of possible operators
# Example : ">=|<=|!=|>|<|="
OPERATOR_RX = '|'.join([re.sub('\|', '\|', o) for o in Predicate.operators.keys()])
# predicate
field = pp.Word(pp.alphanums + '_')
operator = pp.Regex(OPERATOR_RX).setName("operator")
value = pp.QuotedString('"') #| pp.Combine( pp.Word( "+-"+ pp.nums, pp.nums) + pp.Optional( point + pp.Optional( pp.Word( pp.nums ) ) ) + pp.Optional( e + pp.Word( "+-"+pp.nums, pp.nums ) ) )
predicate = (field + operator + value).setParseAction(self.handlePredicate)
# clause of predicates
and_op = pp.CaselessLiteral("and") | pp.Keyword("&&")
or_op = pp.CaselessLiteral("or") | pp.Keyword("||")
not_op = pp.Keyword("!")
predicate_precedence_list = [
(not_op, 1, pp.opAssoc.RIGHT, lambda x: self.handleClause(*x)),
(and_op, 2, pp.opAssoc.LEFT, lambda x: self.handleClause(*x)),
(or_op, 2, pp.opAssoc.LEFT, lambda x: self.handleClause(*x))
]
clause = pp.operatorPrecedence(predicate, predicate_precedence_list)
self.bnf = clause
def get_parse_expression(parse_func, parse_var_id):
"""return a parse expression with for the
input parseActions
"""
var_id = Group(FUNC_NAME + COLON + IDENTIFIER) | REAL | INTEGER | IDENTIFIER
var_id.setParseAction(parse_var_id)
# Forward declaration for an Arithmetic Expression
arith_expr = Forward()
func_call = Group(FUNC_NAME + LPAREN + Optional(Group(delimitedList(arith_expr))) + RPAREN)
# An Arithmetic expression can have a var_id or
# a function call as an operand
# pylint: disable=expression-not-assigned
arith_expr << operatorPrecedence(
func_call | var_id,
[
(EXPONENTIATION_OPS, 2, opAssoc.LEFT, eval_binary_op),
(UNARY_OPS, 1, opAssoc.RIGHT, eval_unary_op),
(MULT_OPS, 2, opAssoc.LEFT, eval_binary_op),
(SUM_OPS, 2, opAssoc.LEFT, eval_binary_op),
(REL_OPS, 2, opAssoc.LEFT, eval_binary_op),
(LOGICAL_OPS, 2, opAssoc.LEFT, eval_binary_op),
],
)
# pylint: enable=expression-not-assigned
# Argument expression for a function call
# An argument to a function can be an
# IDENTIFIER, Arithmetic expression, REAL number, INTEGER or a
# Function call itself
func_call.setParseAction(parse_func)
return arith_expr
def _parseFeatureSignature(sig):
"""This function parses a given feature-signature."""
mal = set()
def _rewriteOne(p):
return ''
def _rewriteTwo(p):
return ''
def _addIdentifier2Mal(p):
mal.add(p[0])
operand = __string | __hexadec | __function | __integer | \
__identifier.setParseAction(_addIdentifier2Mal)
compoperator = pypa.oneOf('< > <= >= == !=')
calcoperator = pypa.oneOf('+ - * / % & | << >>')
expr = pypa.operatorPrecedence(operand, [
('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne),
('!', 1, pypa.opAssoc.RIGHT, _rewriteOne),
(calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo),
(compoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo),
('&&', 2, pypa.opAssoc.LEFT, _rewriteTwo),
('||', 2, pypa.opAssoc.LEFT, _rewriteTwo),
])
try:
rsig = expr.parseString(sig)[0]
except pypa.ParseException, e:
print 'ERROR (parse): cannot parse sig (%s) -- (%s)' % (
sig,
e.col,
)
return sig
def _expression_parser():
variable = Combine(Literal('?').suppress() + Word(alphas)) \
.setParseAction(lambda s, loc, toks: VariableExpression(toks[0]))
literal = _literal.copy().setParseAction(lambda s, loc, toks: LiteralExpression(toks[0]))
value = variable | literal
expr=Forward()
exprList = delimitedList(expr)
funcCall = (Word(alphas + "_") + \
Literal('(').suppress() + \
Optional(exprList) + \
Literal(')').suppress()) .setParseAction(lambda s, loc, toks: FunctionCallExpression(toks[0], toks[1:]))
baseExpr = funcCall | value
expr << operatorPrecedence(baseExpr,[
(oneOf('!'), 1, opAssoc.RIGHT, _unaryOpAction),
(oneOf('+ -'), 1, opAssoc.RIGHT, _unaryOpAction),
(oneOf('* /'), 2, opAssoc.LEFT, _binOpAction),
(oneOf('+ -'), 2, opAssoc.LEFT, _binOpAction),
(oneOf('<= >= < >'), 2, opAssoc.LEFT, _binOpAction),
(oneOf('= !='), 2, opAssoc.LEFT, _binOpAction),
('&&', 2, opAssoc.LEFT, _binOpAction),
('||', 2, opAssoc.LEFT, _binOpAction),
])
return (Literal('(').suppress() + expr + Literal(')').suppress()) | funcCall
def parse(formula):
"""Parse formula string and create abstract syntax tree (AST).
"""
# LTL expression
_ltl_expr = pp.operatorPrecedence(
_proposition,
[
("'", 1, pp.opAssoc.LEFT, ASTUnTempOp),
("!", 1, pp.opAssoc.RIGHT, ASTNot),
(_UnaryTempOps, 1, pp.opAssoc.RIGHT, ASTUnTempOp),
(pp.oneOf("& &&"), 2, pp.opAssoc.LEFT, ASTAnd),
(pp.oneOf("| ||"), 2, pp.opAssoc.LEFT, ASTOr),
(pp.oneOf("xor ^"), 2, pp.opAssoc.LEFT, ASTXor),
("->", 2, pp.opAssoc.RIGHT, ASTImp),
("<->", 2, pp.opAssoc.RIGHT, ASTBiImp),
(pp.oneOf("= == !="), 2, pp.opAssoc.RIGHT, ASTComparator),
(pp.oneOf("U V R"), 2, pp.opAssoc.RIGHT, ASTBiTempOp),
],
)
_ltl_expr.ignore(pp.LineStart() + "--" + pp.restOfLine)
# Increase recursion limit for complex formulae
sys.setrecursionlimit(2000)
try:
return _ltl_expr.parseString(formula, parseAll=True)[0]
except RuntimeError:
raise pp.ParseException("Maximum recursion depth exceeded," "could not parse")
def cross_link_db_filter_parser(inputstring):
'''
example '"{ID_Score}" > 28 AND "{Sample}" ==
"%10_1%" OR ":Sample}" == "%10_2%" OR ":Sample}"
== "%10_3%" OR ":Sample}" == "%8_1%" OR ":Sample}" == "%8_2%"'
'''
import pyparsing as pp
operator = pp.Regex(">=|<=|!=|>|<|==|in").setName("operator")
value = pp.QuotedString(
'"') | pp.Regex(
r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?")
identifier = pp.Word(pp.alphas, pp.alphanums + "_")
comparison_term = identifier | value
condition = pp.Group(comparison_term + operator + comparison_term)
expr = pp.operatorPrecedence(condition, [
("OR", 2, pp.opAssoc.LEFT, ),
("AND", 2, pp.opAssoc.LEFT, ),
])
parsedstring = str(expr.parseString(inputstring)) \
.replace("[", "(") \
.replace("]", ")") \
.replace(",", " ") \
.replace("'", " ") \
.replace("%", "'") \
.replace("{", "float(entry['") \
.replace("}", "'])") \
.replace(":", "str(entry['") \
.replace("}", "'])") \
.replace("AND", "and") \
.replace("OR", "or")
return parsedstring
def _make():
# Order is important - multi-char expressions need to come before narrow
# ones.
parts = []
for klass in filter_unary:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd()
f.setParseAction(klass.make)
parts.append(f)
simplerex = "".join(c for c in pp.printables if c not in "()~'\"")
alphdevanagari = pp.pyparsing_unicode.Devanagari.alphas
alphcyrillic = pp.pyparsing_unicode.Cyrillic.alphas
alphgreek = pp.pyparsing_unicode.Greek.alphas
alphchinese = pp.pyparsing_unicode.Chinese.alphas
alpharabic = pp.pyparsing_unicode.Arabic.alphas
alphhebrew = pp.pyparsing_unicode.Hebrew.alphas
alphjapanese = pp.pyparsing_unicode.Japanese.alphas
alphkorean = pp.pyparsing_unicode.Korean.alphas
alphlatin1 = pp.pyparsing_unicode.Latin1.alphas
alphlatinA = pp.pyparsing_unicode.LatinA.alphas
alphlatinB = pp.pyparsing_unicode.LatinB.alphas
rex = pp.Word(simplerex) |\
pp.Word(alphcyrillic) |\
pp.Word(alphgreek) |\
pp.Word(alphchinese) |\
pp.Word(alpharabic) |\
pp.Word(alphdevanagari) |\
pp.Word(alphhebrew) |\
pp.Word(alphjapanese) |\
pp.Word(alphkorean) |\
pp.Word(alphlatin1) |\
pp.Word(alphlatinA) |\
pp.Word(alphlatinB) |\
pp.QuotedString("\"", escChar='\\') |\
pp.QuotedString("'", escChar='\\')
for klass in filter_rex:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + rex.copy()
f.setParseAction(klass.make)
parts.append(f)
for klass in filter_int:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + pp.Word(pp.nums)
f.setParseAction(klass.make)
parts.append(f)
# A naked rex is a URL rex:
f = rex.copy()
f.setParseAction(FUrl.make)
parts.append(f)
atom = pp.MatchFirst(parts)
expr = pp.operatorPrecedence(atom, [
(pp.Literal("!").suppress(), 1, pp.opAssoc.RIGHT, lambda x: FNot(*x)),
(pp.Literal("&").suppress(), 2, pp.opAssoc.LEFT, lambda x: FAnd(*x)),
(pp.Literal("|").suppress(), 2, pp.opAssoc.LEFT, lambda x: FOr(*x)),
])
expr = pp.OneOrMore(expr)
return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
def parse(self):
col = pp.Word(pp.alphanums)
val = pp.Word(pp.alphanums)
comp = pp.Regex('==|!=|<=|>=|<|>')
cond = pp.Group(col + comp + val)
expr = pp.operatorPrecedence(cond, [('&&', 2, pp.opAssoc.LEFT),
('||', 2, pp.opAssoc.LEFT)])
return expr.parseString(self.cond).asList()[0]
def __infer_precedence(user_query: str) -> str:
# Put square brackets that indicate precedence of operations.
extra_chars = pp.srange(r"[\0x80-\0x7FF]") # Support Cyrillic letters.
tag = pp.Word(pp.alphas + '_' + extra_chars)
parser = pp.operatorPrecedence(tag, [("NOT", 1, pp.opAssoc.RIGHT),
("AND", 2, pp.opAssoc.LEFT),
("OR", 2, pp.opAssoc.LEFT)])
parsed_expression = parser.parseString(user_query)[0]
return str(parsed_expression)
def parse(self, request):
input = request._rest_context.get('filter')
if not input:
return None
condition_positions = []
operator = pp.Regex('|'.join(self.ALLOWED_OPERATORS))
number = pp.Regex(r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?")
AND = pp.Literal(LOGICAL_OPERATORS.AND)
OR = pp.Literal(LOGICAL_OPERATORS.OR)
NOT = pp.Literal(LOGICAL_OPERATORS.NOT)
identifier = pp.Regex(r"[a-zA-Z]+[a-zA-Z0-9]*(_[a-zA-Z0-9]+)*")
identifiers = pp.Group(pp.delimitedList(identifier, delim="__", combine=False))
comparison_term = pp.Forward()
list_term = (
pp.Group(
pp.Suppress('[') + pp.delimitedList(comparison_term, delim=",", combine=False) + pp.Suppress(']')
) |
pp.Group(
pp.Suppress('(') + pp.delimitedList(comparison_term, delim=",", combine=False) + pp.Suppress(')')
) |
pp.Group(
pp.Suppress('{') + pp.delimitedList(comparison_term, delim=",", combine=False) + pp.Suppress('}')
)
)
string = (
pp.QuotedString("'", escChar='\\', unquoteResults=True) | pp.QuotedString('"', escChar='\\',
unquoteResults=True)
)
null = pp.Literal('null').setParseAction(lambda s,l,t: None)
boolean = pp.Regex('|'.join(('true', 'false'))).setParseAction(lambda s, l, t: t[0] == 'true')
comparison_term << (string | number | list_term | null | boolean)
condition = pp.Group(identifiers + operator + comparison_term).setResultsName('condition')
condition.setParseAction(lambda s, loc, tocs: condition_positions.append(loc))
expr = pp.operatorPrecedence(
condition, [
(NOT, 1, pp.opAssoc.RIGHT,),
(AND, 2, pp.opAssoc.LEFT,),
(OR, 2, pp.opAssoc.LEFT,),
]
)
try:
return self._parse_to_conditions(
expr.parseString(input, parseAll=True).asList()[0], list(condition_positions), condition, input
)
except pp.ParseException as ex:
raise FilterParserError(
mark_safe(ugettext('Invalid filter value "{}"').format(input))
)
def __init__(self):
self.select_stmt = Forward().setName("select statement")
self.itemName = MatchFirst(Keyword("itemName()")).setParseAction(
self.ItemName)
self.count = MatchFirst(Keyword("count(*)")).setParseAction(self.Count)
self.identifier = ((~keyword + Word(alphas, alphanums + "_"))
| QuotedString("`"))
self.column_name = (self.itemName | self.identifier.copy())
self.table_name = self.identifier.copy()
self.function_name = self.identifier.copy()
# expression
self.expr = Forward().setName("expression")
self.integer = Regex(r"[+-]?\d+")
self.string_literal = QuotedString("'")
self.literal_value = self.string_literal
self.expr_term = (
self.itemName | self.function_name + LPAR +
Optional(delimitedList(self.expr)) + RPAR
| self.literal_value.setParseAction(self.Literal)
| NULL.setParseAction(self.Null)
| self.identifier.setParseAction(self.Identifier) |
(EVERY + LPAR + self.identifier.setParseAction(self.Identifier) +
RPAR).setParseAction(self.EveryIdentifier) | (LPAR + Optional(
delimitedList(self.literal_value.setParseAction(
self.Literal))) + RPAR).setParseAction(self.ValueList))
self.expr << (operatorPrecedence(self.expr_term, [
(NOT, UNARY, opAssoc.RIGHT, self.BoolNot),
(oneOf('< <= > >='), BINARY, opAssoc.LEFT,
self.BinaryComparisonOperator),
(oneOf('= == != <>') | Group(IS + NOT) | IS | IN | LIKE, BINARY,
opAssoc.LEFT, self.BinaryComparisonOperator),
((BETWEEN, AND), TERNARY, opAssoc.LEFT, self.BetweenXAndY),
(OR, BINARY, opAssoc.LEFT, self.BoolOr),
(AND, BINARY, opAssoc.LEFT, self.BoolAnd),
(INTERSECTION, BINARY, opAssoc.LEFT, self.Intersection),
])).setParseAction(self.dont_allow_non_comparing_terms)
self.ordering_term = (self.itemName
| self.identifier) + Optional(ASC | DESC)
self.single_source = self.table_name("table")
self.result_column = Group(
"*" | self.count | delimitedList(self.column_name))("columns")
self.select_core = (SELECT + self.result_column + FROM +
self.single_source +
Optional(WHERE + self.expr("where_expr")))
self.select_stmt << (self.select_core + Optional(ORDER + BY + Group(
delimitedList(self.ordering_term))).setParseAction(
self.OrderByTerms)("order_by_terms") +
Optional(LIMIT + self.integer)("limit_terms"))
def make_parser(clause):
expression = operatorPrecedence(clause, [
(factop, 1, opAssoc.LEFT, Expression),
(expop, 2, opAssoc.RIGHT, Expression),
(signop, 1, opAssoc.RIGHT, Expression),
(multop, 2, opAssoc.LEFT, Expression),
(plusop, 2, opAssoc.LEFT, Expression),
])
return expression.setParseAction(Expression).setResultsName('expression')
def conditions(self):
"""
conditions ::= condition | condition logical_binary_op conditions
Note: By default lpar and rpar arguments are suppressed.
"""
return operatorPrecedence(baseExpr=self.condition,
opList=[(self.not_op, 1, opAssoc.RIGHT),
(self.logical_binary_op, 2,
opAssoc.LEFT)],
lpar=self.syntax.paren_left,
rpar=self.syntax.paren_right)
def parse_prereq_text(text):
"""Konvertuje text prerekvizit do seznamu [[['7', '&&', '12'], '||', '4']]
Seznam na danem zanoreni obsahuje bud teminal, nebo seznam tri prvku
"""
number = pp.Regex(r"\d+")
expr = pp.operatorPrecedence(number, [
("&&", 2, pp.opAssoc.LEFT, ),
("||", 2, pp.opAssoc.LEFT, ),
])
return expr.parseString(text)
def conditions(self):
"""
conditions ::= condition | condition logical_binary_op conditions
Note: By default lpar and rpar arguments are suppressed.
"""
return operatorPrecedence(
baseExpr=self.condition,
opList=[(self.not_op, 1, opAssoc.RIGHT),
(self.logical_binary_op, 2, opAssoc.LEFT)],
lpar=self.syntax.paren_left,
rpar=self.syntax.paren_right)
def __init__(self, fields):
self._word = pyparsing.Word(QueryParser._wordchars)
self._word.setParseAction(partial(self._parse_word, fields))
self._phrase = pyparsing.QuotedString('"', '\\')
self._phrase.setParseAction(partial(self._parse_phrase, fields))
self._token = self._phrase | self._word
self._query = pyparsing.Forward()
self._query << pyparsing.operatorPrecedence(self._token, [
QueryParser._not_op, QueryParser._and_op, QueryParser._or_op])
self._multiword = pyparsing.OneOrMore(self._word)
self._multiword.setParseAction(self._parse_multi)
def gen_base_expr():
variable = pyparsing.Word(pyparsing.alphas + "_$.",
pyparsing.alphanums + "_")
variable.setParseAction(parse_id)
operand = str_int | variable
base_expr = pyparsing.operatorPrecedence(operand, [
("!", 1, pyparsing.opAssoc.RIGHT, parse_op),
(logicop, 2, pyparsing.opAssoc.RIGHT, parse_op),
(signop, 1, pyparsing.opAssoc.RIGHT, parse_op),
(multop, 2, pyparsing.opAssoc.LEFT, parse_op),
(plusop, 2, pyparsing.opAssoc.LEFT, parse_op),
])
return variable, operand, base_expr
def __init__(self, expr, **kws): self.expr, self.params = expr, kws self.g = g = self.grammar_cls() atom = g.value | g.symbol.copy().setParseAction(lambda s,l,t: self.params[t[0].lower()]) self.syntax = pp.operatorPrecedence(atom, [ (g.logic_func, 1, pp.opAssoc.RIGHT, lambda s,l,t: g.ops_func[t[0][0]](t[0][1])), (g.math_mult, 2, pp.opAssoc.LEFT, self.eval_math), (g.math_add, 2, pp.opAssoc.LEFT, self.eval_math), (g.logic_sets, 2, pp.opAssoc.LEFT, lambda s,l,t: g.ops_sets[t[0][1]](t[0][0], t[0][2])), (g.logic_comparison, 2, pp.opAssoc.LEFT, self.eval_comparison), (g.logic_not, 1, pp.opAssoc.RIGHT, lambda s,l,t: not t[0][1]), (g.logic_and, 2, pp.opAssoc.LEFT, lambda s,l,t: t[0][0] and t[0][2]), (g.logic_or, 2, pp.opAssoc.LEFT, lambda s,l,t: t[0][0] or t[0][2]) ])
def queryparser():
'''generate a linggle query syntax parser
linggle query syntax -> hbase query'''
word = Word(alphas + "'" + '.' + ',' + '.' + '<>')
wildcard, any_wildcard, maybe = Literal('_'), Literal('*'), Literal('?')
POS = (Literal('adj.') | Literal('n.') | Literal('v.') | Literal('adv.')
| Literal('det.') | Literal('prep.'))
atom = (word | wildcard | any_wildcard | maybe | POS)
query = operatorPrecedence(
atom, [('|', 2, opAssoc.LEFT, Alternatives),
(Optional('&', default='&'), 2, opAssoc.LEFT, Query)])
return query
def __init__(self):
# supported operators
operator = pp.Regex(r"<=|>=|<>|\!=|==|<|>|not|in|regex_partial|regex_exact|geo_box|geo_radius|geo_polygon|contains_any|substr|contains_near|any|contains_substr|near|contains").setName("operator").addParseAction(self.validateOperator)
# literals
number = pp.Regex(r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?").setName("number")
numberList = pp.Group(pp.Literal('[') + number + pp.ZeroOrMore("," + number) + pp.Literal(']')).setName("numberList")
string = pp.dblQuotedString
literals = number | numberList | string
# symbols
identifier = pp.Regex(r"[a-z][a-z_]+(?:\.[a-z][a-z_]+)+").addParseAction(self.validateIdentifier).setName("identifier")
# we'll get there...
subExpr = pp.Forward()
# predicates
stream = pp.Group(pp.Literal("stream") + string).setName("stream")
exists = pp.Group(identifier + pp.Literal("exists")).setName("exists")
# boolean predicates
comparison = pp.Group(
identifier + operator + literals
| literals + operator + identifier
).setName("comparison")
condition = comparison | stream | exists | subExpr
subExpr << pp.nestedExpr(content=condition)
# standard boolean operator precedence
expr = pp.operatorPrecedence(condition,[
(pp.CaselessLiteral("not"), 1, pp.opAssoc.RIGHT, ),
(pp.CaselessLiteral("AND"), 2, pp.opAssoc.LEFT, ),
(pp.CaselessLiteral("OR"), 2, pp.opAssoc.LEFT, ),
])
# tag "thing" { expr }
tag = pp.Group(pp.Literal("tag") + pp.quotedString + pp.nestedExpr("{", "}", expr)).setName("tag")
# return { expr }
a_return = pp.Group(pp.Literal("return") + pp.nestedExpr("{", "}", expr)).setName("return")
# a single expression or tag [, tag, ...] return { expression }
parser = expr | (pp.OneOrMore(tag) + a_return)
# handle multilines
parser.setDefaultWhitespaceChars(" \t\n\r")
# handle // comments
parser.ignore("//" + pp.restOfLine)
self.parser = parser
def check_boolean_exprs(exprs=None, operand=(), send_error=True):
'''
Check whether a boolean expression is properly formed.
:param exprs: The string to evaluate.
:param operand: The name of the operands.
:param send_error: Whether to throw an error if expression is malformed.
:return: A boolean.
:Example:
>>> from pygtftk.utils import check_boolean_exprs
>>> assert check_boolean_exprs('s > 1 and (s < 2 or y < 2.5)', operand=['s', 'y'])
'''
lparen = Literal("(")
rparen = Literal(")")
and_operator = CaselessLiteral("and")
or_operator = CaselessLiteral("or")
comparison_operator = oneOf(['==', '!=', '>', '>=', '<', '<='])
point = Literal('.')
exponent = CaselessLiteral('E')
plusorminus = Literal('+') | Literal('-')
number = Word(nums)
integer = Combine(Optional(plusorminus) + number)
float_nb = Combine(integer +
Optional(point + Optional(number)) +
Optional(exponent + integer))
value = float_nb
identifier = oneOf(operand, caseless=False) # .setParseAction(_embed)
group_1 = identifier + comparison_operator + value
group_2 = value + comparison_operator + identifier
comparison = group_1 | group_2
boolean_expr = operatorPrecedence(comparison,
[(and_operator, 2, opAssoc.LEFT),
(or_operator, 2, opAssoc.LEFT)])
boolean_expr_par = lparen + boolean_expr + rparen
expression = Forward()
expression << boolean_expr | boolean_expr_par
try:
expression.parseString(exprs, parseAll=True)
return True
except ParseException as err:
if send_error:
message(err.msg, force=True)
message('Operand should be one of: ' + ", ".join(operand))
message("Boolean expression not supported.", type="ERROR")
return False
class att_syntax:
divide = False
noprefix = False
pfx = pp.oneOf([
'data16', 'addr16', 'data32', 'addr32', 'lock', 'wait', 'rep', 'repe',
'repne'
])
comment = pp.Regex(r'\#.*')
symbol = pp.Regex(r'[A-Za-z_.$][A-Za-z0-9_.$]*')
integer = pp.Regex(r'[1-9][0-9]*')
hexa = pp.Regex(r'0[xX][0-9a-fA-F]+')
octa = pp.Regex(r'0[0-7]+')
bina = pp.Regex(r'0[bB][01]+')
char = pp.Regex(r"('.)|('\\\\)")
number = integer | hexa | octa | bina | char
term = symbol | number
exp = pp.Forward()
op_one = pp.oneOf("- ~")
op_sig = pp.oneOf("+ -")
op_mul = pp.oneOf("* /")
op_cmp = pp.oneOf("== != <= >= < > <>")
op_bit = pp.oneOf("^ && || & |")
operators = [
(op_one, 1, pp.opAssoc.RIGHT),
(op_sig, 2, pp.opAssoc.LEFT),
(op_mul, 2, pp.opAssoc.LEFT),
(op_cmp, 2, pp.opAssoc.LEFT),
(op_bit, 2, pp.opAssoc.LEFT),
]
exp << pp.operatorPrecedence(term, operators)
reg = '%' + symbol
bis = '(' + pp.Optional(reg) + pp.Optional(',' + reg +
pp.Optional(',' + exp)) + ')'
adr = pp.Optional(exp) + bis
mem = pp.Optional(reg + ':') + adr
imm = pp.Optional(pp.Literal('$')) + exp
opd = mem | reg | imm
opds = pp.Group(pp.delimitedList(opd))
instr = pp.Optional(pfx) + symbol + pp.Optional(opds) + pp.Optional(
comment)
def gen_base_expr():
variable = pyparsing.Word(pyparsing.alphas + "_$.", pyparsing.alphanums + "_")
variable.setParseAction(parse_id)
operand = str_int | variable
base_expr = pyparsing.operatorPrecedence(
operand,
[
("!", 1, pyparsing.opAssoc.RIGHT, parse_op),
(logicop, 2, pyparsing.opAssoc.RIGHT, parse_op),
(signop, 1, pyparsing.opAssoc.RIGHT, parse_op),
(multop, 2, pyparsing.opAssoc.LEFT, parse_op),
(plusop, 2, pyparsing.opAssoc.LEFT, parse_op),
],
)
return variable, operand, base_expr
def logicParse(inStr,pm = None):
variable = oneOf('a b c d e f g h i j k l m n o p q r s t u w x y z ' + trueConstants + falseConstants)
expr = operatorPrecedence(variable,
[
(oneOf(notOps), 1, opAssoc.RIGHT),
(oneOf(orOps), 2, opAssoc.LEFT),
(oneOf(andOps), 2, opAssoc.LEFT),
(oneOf(impOps), 2, opAssoc.LEFT)#,
# (oneOf(bimpOps),2,opAssoc.LEFT),
# (oneOf(xorOps), 2, opAssoc.LEFT)
])
parse = expr.parseString(inStr)[0]
if pm == None: return parseToStatement(parse, propMap())
else: return parseToStatement(parse, pm)
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack = Literal("[")
rbrack = Literal("]")
lbrace = Literal("{")
rbrace = Literal("}")
lparen = Literal("(")
rparen = Literal(")")
reMacro = Suppress("\\") + oneOf(list("dwsZ"))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in string.printable if c not in r"\[]{}().*?+|")
reRange = Combine(lbrack.suppress() + SkipTo(rbrack,ignore=escapedChar) + rbrack.suppress())
reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) )
reDot = Literal(".")
repetition = (
( lbrace + Word(nums).setResultsName("count") + rbrace ) |
( lbrace + Word(nums).setResultsName("minCount")+","+ Word(nums).setResultsName("maxCount") + rbrace ) |
oneOf(list("*+?"))
)
reExpr = Forward()
reGroup = (lparen.suppress() +
Optional(Literal("?").suppress() + oneOf(list(":P"))).setResultsName("option") +
reExpr.setResultsName("expr") +
rparen.suppress())
reTerm = ( reLiteral | reRange | reMacro | reDot | reGroup )
reExpr << operatorPrecedence( reTerm,
[
(repetition, 1, opAssoc.LEFT, create(Repetition)),
(None, 2, opAssoc.LEFT, create(Sequence)),
(Suppress('|'), 2, opAssoc.LEFT, create(Alternation)),
]
)
reGroup.setParseAction(create(Group))
reRange.setParseAction(create(Range))
reLiteral.setParseAction(create(Character))
reMacro.setParseAction(create(Macro))
reDot.setParseAction(create(Dot))
_parser = reExpr
return _parser
def __init__(self):
self.select_stmt = Forward().setName("select statement")
self.itemName = MatchFirst(Keyword("itemName()")).setParseAction(self.ItemName)
self.count = MatchFirst(Keyword("count(*)")).setParseAction(self.Count)
self.identifier = ((~keyword + Word(alphas, alphanums+"_")) | QuotedString("`"))
self.column_name = (self.itemName | self.identifier.copy())
self.table_name = self.identifier.copy()
self.function_name = self.identifier.copy()
# expression
self.expr = Forward().setName("expression")
self.integer = Regex(r"[+-]?\d+")
self.string_literal = QuotedString("'")
self.literal_value = self.string_literal
self.expr_term = (
self.itemName |
self.function_name + LPAR + Optional(delimitedList(self.expr)) + RPAR |
self.literal_value.setParseAction(self.Literal) |
NULL.setParseAction(self.Null) |
self.identifier.setParseAction(self.Identifier) |
(EVERY + LPAR + self.identifier.setParseAction(self.Identifier) + RPAR).setParseAction(self.EveryIdentifier) |
(LPAR + Optional(delimitedList(self.literal_value.setParseAction(self.Literal))) + RPAR).setParseAction(self.ValueList)
)
self.expr << (operatorPrecedence(self.expr_term,
[
(NOT, UNARY, opAssoc.RIGHT, self.BoolNot),
(oneOf('< <= > >='), BINARY, opAssoc.LEFT, self.BinaryComparisonOperator),
(oneOf('= == != <>') | Group(IS + NOT) | IS | IN | LIKE, BINARY, opAssoc.LEFT, self.BinaryComparisonOperator),
((BETWEEN,AND), TERNARY, opAssoc.LEFT, self.BetweenXAndY),
(OR, BINARY, opAssoc.LEFT, self.BoolOr),
(AND, BINARY, opAssoc.LEFT, self.BoolAnd),
(INTERSECTION, BINARY, opAssoc.LEFT, self.Intersection),
])).setParseAction(self.dont_allow_non_comparing_terms)
self.ordering_term = (self.itemName | self.identifier) + Optional(ASC | DESC)
self.single_source = self.table_name("table")
self.result_column = Group("*" | self.count | delimitedList(self.column_name))("columns")
self.select_core = (SELECT + self.result_column + FROM + self.single_source + Optional(WHERE + self.expr("where_expr")))
self.select_stmt << (self.select_core +
Optional(ORDER + BY + Group(delimitedList(self.ordering_term))).setParseAction(self.OrderByTerms)("order_by_terms") +
Optional(LIMIT + self.integer)("limit_terms"))
class intel_syntax:
divide = False
noprefix = False
pfx = pp.oneOf([
'data16', 'addr16', 'data32', 'addr32', 'lock', 'wait', 'rep', 'repe',
'repne'
])
spfx = pp.oneOf(['dword', 'word', 'byte'], caseless=True)
mpfx = spfx + pp.oneOf(['ptr'], caseless=True)
sfx = pp.oneOf(['far'], caseless=True)
comment = pp.Regex(r'\#.*')
symbol = pp.Regex(r'[A-Za-z_.$][A-Za-z0-9_.$]*')
integer = pp.Regex(r'[1-9][0-9]*')
hexa = pp.Regex(r'0[xX][0-9a-fA-F]+')
octa = pp.Regex(r'0[0-7]+')
bina = pp.Regex(r'0[bB][01]+')
char = pp.Regex(r"('.)|('\\\\)")
number = integer | hexa | octa | bina | char
term = symbol | number
exp = pp.Forward()
op_one = pp.oneOf("- ~")
op_sig = pp.oneOf("+ -")
op_mul = pp.oneOf("* /")
op_cmp = pp.oneOf("== != <= >= < > <>")
op_bit = pp.oneOf("^ && || & |")
operators = [
(op_one, 1, pp.opAssoc.RIGHT),
(op_sig, 2, pp.opAssoc.LEFT),
(op_mul, 2, pp.opAssoc.LEFT),
(op_cmp, 2, pp.opAssoc.LEFT),
(op_bit, 2, pp.opAssoc.LEFT),
]
exp << pp.operatorPrecedence(term, operators)
adr = '[' + exp + ']'
mem = pp.Optional(mpfx) + pp.Optional(symbol + ':') + adr
opd = mem | exp
opds = pp.Group(pp.delimitedList(opd))
instr = pp.Optional(pfx) + symbol + pp.Optional(sfx) + pp.Optional(
opds) + pp.Optional(comment)
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack = Literal("[")
rbrack = Literal("]")
lbrace = Literal("{")
rbrace = Literal("}")
lparen = Literal("(")
rparen = Literal(")")
reMacro = Suppress("\\") + oneOf(list("dwsZ"))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in string.printable
if c not in r"\[]{}().*?+|")
reRange = Combine(lbrack.suppress() +
SkipTo(rbrack, ignore=escapedChar) +
rbrack.suppress())
reLiteral = (escapedChar | oneOf(list(reLiteralChar)))
reDot = Literal(".")
repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) |
(lbrace + Word(nums).setResultsName("minCount") + "," +
Word(nums).setResultsName("maxCount") + rbrace)
| oneOf(list("*+?")))
reExpr = Forward()
reGroup = (lparen.suppress() +
Optional(Literal("?").suppress() +
oneOf(list(":P"))).setResultsName("option") +
reExpr.setResultsName("expr") + rparen.suppress())
reTerm = (reLiteral | reRange | reMacro | reDot | reGroup)
reExpr << operatorPrecedence(reTerm, [
(repetition, 1, opAssoc.LEFT, create(Repetition)),
(None, 2, opAssoc.LEFT, create(Sequence)),
(Suppress('|'), 2, opAssoc.LEFT, create(Alternation)),
])
reGroup.setParseAction(create(Group))
reRange.setParseAction(create(Range))
reLiteral.setParseAction(create(Character))
reMacro.setParseAction(create(Macro))
reDot.setParseAction(create(Dot))
_parser = reExpr
return _parser
def __init__(self, func, *args, **kwargs):
# treats kwarg boolOperand specially!
self.args = args
self.kwargs = kwargs
self.func = func
if "boolOperand" in kwargs:
boolOperand = kwargs["boolOperand"]
del kwargs["boolOperand"]
else:
boolOperand = Word(alphanums + "-.,_=<>!@$%^&*[]{}:;|/\\")
boolOperand = boolOperand.setParseAction(self._check)
self.boolExpr = operatorPrecedence(boolOperand, [
("not ", 1, opAssoc.RIGHT, BoolNot),
("or", 2, opAssoc.LEFT, BoolOr),
("and", 2, opAssoc.LEFT, BoolAnd),
])
def queryparser():
'''generate a linggle query syntax parser
linggle query syntax -> hbase query'''
word = Word(alphas + "'" + '.' + ',' + '.' + '<>')
wildcard, any_wildcard, maybe = Literal('_'), Literal('*'), Literal('?')
POS = (Literal('adj.')| Literal('n.')|Literal('v.')|Literal('adv.')| Literal('det.')| Literal('prep.'))
atom = (word | wildcard | any_wildcard | maybe | POS)
query = operatorPrecedence(
atom,
[
('|', 2, opAssoc.LEFT, Alternatives),
(Optional('&', default='&'), 2, opAssoc.LEFT, Query)
])
return query
def make_parser(payload_specification):
"""
The parser has to be payload_specification specific because it needs to know
the types of the records to build a FieldSelection.
"""
def make_selection(string, location, tokens):
variable, comparator, value = tokens
if payload_specification and variable not in payload_specification:
raise RuntimeError("%s is not a value in the folder payload "
"specification")
elif not payload_specification:
return (variable, comparator, value)
typeid = payload_specification[variable].storageType().id()
return make_fieldselection(variable, typeid, comparator, value)
from PyCool import cool
EQ = Literal("==").setParseAction(lambda t: cool.FieldSelection.EQ)
NE = Literal("!=").setParseAction(lambda t: cool.FieldSelection.NE)
GE = Literal(">=").setParseAction(lambda t: cool.FieldSelection.GE)
LE = Literal("<=").setParseAction(lambda t: cool.FieldSelection.LE)
GT = Literal(">").setParseAction(lambda t: cool.FieldSelection.GT)
LT = Literal("<").setParseAction(lambda t: cool.FieldSelection.LT)
comparator = EQ | NE | GE | LE | GT | LT
operand = (variable + comparator + value).setParseAction(make_selection)
orop = Literal("or").suppress()
andop = Literal("and").suppress()
def logic_builder(connective):
def thunk(string, location, tokens):
vec = make_selection_vector()
for token in tokens[0]:
vec.push_back(token)
return cool.CompositeSelection(connective, vec)
return thunk
expr = StringStart() + operatorPrecedence(operand, [
(andop, 2, opAssoc.LEFT, logic_builder(cool.CompositeSelection.AND)),
(orop, 2, opAssoc.LEFT, logic_builder(cool.CompositeSelection.OR)),
]) + StringEnd()
def go(selection):
return expr.parseString(selection)[0]
return go
class intel_syntax:
divide = False
noprefix = False
pfx = pp.oneOf([
"data16", "addr16", "data32", "addr32", "lock", "rep", "repe", "repne"
])
spfx = pp.oneOf(["dword", "word", "byte"], caseless=True)
mpfx = spfx + pp.oneOf(["ptr"], caseless=True)
sfx = pp.oneOf(["far"], caseless=True)
comment = pp.Regex(r"\#.*")
symbol = pp.Regex(r"[A-Za-z_.$][A-Za-z0-9_.$]*")
integer = pp.Regex(r"[1-9][0-9]*")
hexa = pp.Regex(r"0[xX][0-9a-fA-F]+")
octa = pp.Regex(r"0[0-7]+")
bina = pp.Regex(r"0[bB][01]+")
char = pp.Regex(r"('.)|('\\\\)")
number = integer | hexa | octa | bina | char
term = symbol | number
exp = pp.Forward()
op_one = pp.oneOf("- ~")
op_sig = pp.oneOf("+ -")
op_mul = pp.oneOf("* /")
op_cmp = pp.oneOf("== != <= >= < > <>")
op_bit = pp.oneOf("^ && || & |")
operators = [
(op_one, 1, pp.opAssoc.RIGHT),
(op_sig, 2, pp.opAssoc.LEFT),
(op_mul, 2, pp.opAssoc.LEFT),
(op_cmp, 2, pp.opAssoc.LEFT),
(op_bit, 2, pp.opAssoc.LEFT),
]
exp << pp.operatorPrecedence(term, operators)
adr = "[" + exp + "]"
mem = pp.Optional(mpfx) + pp.Optional(symbol + ":") + adr
opd = mem | exp
opds = pp.Group(pp.delimitedList(opd))
instr = (pp.Optional(pfx) + symbol + pp.Optional(sfx) + pp.Optional(opds) +
pp.Optional(comment))
class Syntax:
input_received = S(K("in") + L(":")) + pyparsing_common.identifier
output_produced = S(K("out") + L(":")) + pyparsing_common.identifier
basic = input_received | output_produced
language = operatorPrecedence(basic,
[
(S(L('*')), 1, opAssoc.LEFT, on_zero_or_more),
(S(L('+')), 1, opAssoc.LEFT, on_one_or_more),
(S(L('?')), 1, opAssoc.LEFT, on_zero_or_one),
(S(L(';')), 2, opAssoc.LEFT, on_in_sequence),
(S(L('|')), 2, opAssoc.LEFT, on_either),
])
input_received.setParseAction(on_input_received)
output_produced.setParseAction(on_output_produced)
def __init__(self):
self.left_parenthesis, self.right_parenthesis = map(Suppress, "()")
self.implies = Literal("->")
self.or_ = Literal("|")
self.and_ = Literal("&")
self.not_ = Literal("!") | Literal ("~")
self.boolean = Keyword("false") | Keyword("true")
self.symbol = Word(alphas, alphanums)
self.formula = Forward()
self.operand = self.boolean | self.symbol
self.formula << operatorPrecedence(self.operand, [
(self.not_, 1, opAssoc.RIGHT, NotConnective),
(self.and_, 2, opAssoc.LEFT, AndConnective),
(self.or_, 2, opAssoc.LEFT, OrConnective),
(self.implies, 2, opAssoc.RIGHT, ImpliesConnective)])
def _make():
# Order is important - multi-char expressions need to come before narrow
# ones.
parts = []
for klass in filt_unary:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd()
f.setParseAction(klass.make)
parts.append(f)
simplerex = "".join(c for c in pp.printables if c not in "()~'\"")
rex = pp.Word(simplerex) |\
pp.QuotedString("\"", escChar='\\') |\
pp.QuotedString("'", escChar='\\')
for klass in filt_rex:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + rex.copy()
f.setParseAction(klass.make)
parts.append(f)
for klass in filt_int:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + pp.Word(pp.nums)
f.setParseAction(klass.make)
parts.append(f)
# A naked rex is a URL rex:
f = rex.copy()
f.setParseAction(FUrl.make)
parts.append(f)
atom = pp.MatchFirst(parts)
expr = pp.operatorPrecedence(atom,
[(pp.Literal("!").suppress(),
1,
pp.opAssoc.RIGHT,
lambda x: FNot(*x)),
(pp.Literal("&").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FAnd(*x)),
(pp.Literal("|").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FOr(*x)),
])
expr = pp.OneOrMore(expr)
return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
def _make():
# Order is important - multi-char expressions need to come before narrow
# ones.
parts = []
for klass in filt_unary:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd()
f.setParseAction(klass.make)
parts.append(f)
simplerex = "".join(c for c in pp.printables if c not in "()~'\"")
rex = pp.Word(simplerex) |\
pp.QuotedString("\"", escChar='\\') |\
pp.QuotedString("'", escChar='\\')
for klass in filt_rex:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + rex.copy()
f.setParseAction(klass.make)
parts.append(f)
for klass in filt_int:
f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + pp.Word(pp.nums)
f.setParseAction(klass.make)
parts.append(f)
# A naked rex is a URL rex:
f = rex.copy()
f.setParseAction(FUrl.make)
parts.append(f)
atom = pp.MatchFirst(parts)
expr = pp.operatorPrecedence(atom,
[(pp.Literal("!").suppress(),
1,
pp.opAssoc.RIGHT,
lambda x: FNot(*x)),
(pp.Literal("&").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FAnd(*x)),
(pp.Literal("|").suppress(),
2,
pp.opAssoc.LEFT,
lambda x: FOr(*x)),
])
expr = pp.OneOrMore(expr)
return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
def grammar(self):
number = Combine(Word(nums) + Optional("." + OneOrMore(Word(nums))))
table = Combine(Word(alphas) + OneOrMore(Word("_"+alphanums)))
number.setParseAction(self._number_parse_action)
table.setParseAction(self._table_parse_action)
signop = oneOf('+ -')
multop = oneOf('* /')
plusop = oneOf('+ -')
operand = number | table
return operatorPrecedence(operand, [
(signop, 1, opAssoc.RIGHT),
(multop, 2, opAssoc.LEFT),
(plusop, 2, opAssoc.LEFT)
])