def parse(lines: str, allowed_colors: str = '') -> Union[Rule, Color]:
lines = strip_comments(lines)
if allowed_colors == '':
allowed_colors = string.ascii_lowercase + string.ascii_uppercase
color = Word(allowed_colors, exact=1)
integer = Word(string.digits).setParseAction(lambda t: int(t[0]))
selector = Word(allowed_colors + '-', exact=1) * 9
selector.setParseAction(lambda s: Selector(''.join(s)))
num_operation = infixNotation((selector | integer), [
('*', 2, opAssoc.LEFT, parse_selector_operator),
('/', 2, opAssoc.LEFT, parse_selector_operator),
('+', 2, opAssoc.LEFT, parse_selector_operator),
('-', 2, opAssoc.LEFT, parse_selector_operator),
('%', 2, opAssoc.LEFT, parse_selector_operator),
])
operator = oneOf('>= <= != > < ==')
comparison_token = num_operation | selector | integer
comparison = (comparison_token + operator + comparison_token).\
setParseAction(lambda t: Comparison(*t))
bool_expr = infixNotation(comparison, [
('and', 2, opAssoc.LEFT, parse_bool_expr),
('or', 2, opAssoc.LEFT, parse_bool_expr),
])
rule = Forward()
condition = (Word('if') + bool_expr + Word(':') + rule + Word('else:') +
rule)
rule << (condition | color).setParseAction(parse_rule)
return rule.parseString(lines, parseAll=True)[0]
def parse_expr(s):
LPAR = pyp.Literal('(').suppress()
RPAR = pyp.Literal(')').suppress()
quote = pyp.Literal('"').suppress()
sp = pyp.OneOrMore(pyp.White()).suppress()
sps = pyp.ZeroOrMore(pyp.White()).suppress()
nums = pyp.Word(pyp.srange("[0-9]"))
num = (nums + pyp.Optional(pyp.Literal('.') + nums))\
.setParseAction(lambda toks: Num(''.join(toks)))
var = (pyp.Word(pyp.alphas.lower(), pyp.srange("[a-zA-Z0-9]")) + pyp.Optional(pyp.Literal("'"))).\
setParseAction(lambda toks: Var(toks[0], toks[1] if len(toks) > 1 else None))
chars = (pyp.QuotedString('"')).setParseAction(lambda toks: Charstr(toks[0]))
boolean = (pyp.oneOf("True False true false")).setParseAction(lambda toks: Bool(toks[0]))
term = pyp.Forward()
pterm = (LPAR + sps + term + sps + RPAR).setParseAction(lambda toks: toks[0])
term << pyp.infixNotation(num | var | pterm | chars | boolean, [
(pyp.oneOf("+ -"), 2, pyp.opAssoc.LEFT, lambda ts: BinOp(ts[0][0], ts[0][1], ts[0][2])),
])
formula = pyp.Forward()
cmpop = pyp.oneOf("== < > <= >= !=")
atom = (sps + term + sps + cmpop + sps + term + sps).\
setParseAction(lambda toks: Cmp(toks[1], toks[0], toks[2]))
patom = (LPAR + sps + atom + sps + RPAR).setParseAction(lambda toks: toks[0])
formula << pyp.infixNotation(patom, [
(pyp.oneOf("&& ||"), 2, pyp.opAssoc.LEFT, lambda ts: BinCon(ts[0][0], ts[0][1], ts[0][2])),
])
res = formula.parseString(s)
r = res[0] if len(res) > 0 else None
return r
def mathematical_expression() -> Token:
binary_adding_operator = Literal("+") | Literal("-")
multiplying_operator = Literal("*") | Literal("/")
highest_precedence_operator = Literal("**")
array_aggregate = (Literal("(").setParseAction(lambda s, l, t: l) +
numeric_literal() + (comma() - numeric_literal()) *
(0, ) + Literal(")").setParseAction(lambda s, l, t: l))
array_aggregate.setParseAction(parse_array_aggregate)
string = QuotedString('"')
string.setParseAction(parse_string)
concatenation = (infixNotation(
array_aggregate | string,
[(Suppress(Keyword("&")), 2, opAssoc.LEFT, parse_concatenation)],
)).setName("Concatenation")
term = numeric_literal() | attribute_reference() | qualified_identifier(
) | concatenation
term.setParseAction(parse_term)
return (infixNotation(
term,
[
(highest_precedence_operator, 2, opAssoc.LEFT,
parse_mathematical_expression),
(multiplying_operator, 2, opAssoc.LEFT,
parse_mathematical_expression),
(binary_adding_operator, 2, opAssoc.LEFT,
parse_mathematical_expression),
],
)).setName("MathematicalExpression")
def parse_equation(eq: str) -> dict:
ParserElement.enablePackrat()
if getrecursionlimit() <= LOWER_RECURSION_LIMIT:
setrecursionlimit(LOWER_RECURSION_LIMIT)
# Define atoms
NUM = pyparsing_common.number
VARIABLE = Word(['x', 'y'], exact=True)
operand = NUM | VARIABLE
# Define production rules
expr = infixNotation(operand, [(Literal(op), 1, opAssoc.RIGHT, op_rep)
for op in uniops] +
[(Literal(op), 2, opAssoc.LEFT, op_rep)
for op in binops])
comp = infixNotation(expr, [(Literal(op), 2, opAssoc.LEFT, op_rep)
for op in compops])
cond = infixNotation(comp, [(Literal(op), 1, opAssoc.RIGHT, op_rep)
for op in logicuniops] +
[(Literal(op), 2, opAssoc.LEFT, op_rep)
for op in logicbinops])
try:
return cond.parseString(eq, parseAll=True)[0]
except ParseException as pex:
print('Error while parsing "%s": %s' % (eq, str(pex)), file=stderr)
return None
def __init__(self):
# speed up infixNotation considerably at the price of some cache memory
ParserElement.enablePackrat()
boolean = Keyword('True') | Keyword('False')
none = Keyword('None')
integer = Word(nums)
real = Combine(Word(nums) + "." + Word(nums))
string = (QuotedString('"', escChar='\\')
| QuotedString("'", escChar='\\'))
regex = QuotedString('/', escChar='\\')
identifier = Word(alphas, alphanums + '_')
dereference = infixNotation(identifier, [
(Literal('.'), 2, opAssoc.LEFT, EvalArith),
])
result = (Keyword('bad') | Keyword('fail') | Keyword('good')
| Keyword('ignore') | Keyword('unknown'))
rval = boolean | none | real | integer | string | regex | result | dereference
rvallist = Group(
Suppress('[') + Optional(delimitedList(rval)) + Suppress(']'))
rvalset = Group(
Suppress('{') + Optional(delimitedList(rval)) + Suppress('}'))
operand = rval | rvallist | rvalset
# parse actions replace the parsed tokens with an instantiated object
# which we can later call into for evaluation of its content
boolean.setParseAction(EvalBoolean)
none.setParseAction(EvalNone)
integer.setParseAction(EvalInteger)
real.setParseAction(EvalReal)
string.setParseAction(EvalString)
regex.setParseAction(EvalRegex)
identifier.setParseAction(EvalIdentifier)
result.setParseAction(EvalResult)
rvallist.setParseAction(EvalList)
rvalset.setParseAction(EvalSet)
identity_test = Keyword('is') + ~Keyword('not') | Combine(
Keyword('is') + Keyword('not'), adjacent=False, joinString=' ')
membership_test = Keyword('in') | Combine(
Keyword('not') + Keyword('in'), adjacent=False, joinString=' ')
comparison_op = oneOf('< <= > >= != == isdisjoint')
comparison = identity_test | membership_test | comparison_op
self.parser = infixNotation(operand, [
(Literal('**'), 2, opAssoc.LEFT, EvalPower),
(oneOf('+ - ~'), 1, opAssoc.RIGHT, EvalModifier),
(oneOf('* / // %'), 2, opAssoc.LEFT, EvalArith),
(oneOf('+ -'), 2, opAssoc.LEFT, EvalArith),
(oneOf('<< >>'), 2, opAssoc.LEFT, EvalArith),
(Literal('&'), 2, opAssoc.LEFT, EvalArith),
(Literal('^'), 2, opAssoc.LEFT, EvalArith),
(Literal('|'), 2, opAssoc.LEFT, EvalArith),
(comparison, 2, opAssoc.LEFT, EvalLogic),
(Keyword('not'), 1, opAssoc.RIGHT, EvalModifier),
(Keyword('and'), 2, opAssoc.LEFT, EvalLogic),
(Keyword('or'), 2, opAssoc.LEFT, EvalLogic),
(Keyword('->'), 2, opAssoc.LEFT, EvalArith),
])
def enableDot(self, action=DotOpAction):
EXP = self.expr
dotop = pp.Suppress('.') + IDEN('attr')
dotop.setParseAction(action)
self.opList.insert(0, dotop)
self.expr <<= pp.infixNotation(self.baseExpr, self.opList, self.lpar,
self.rpar)
def expression_parser():
"""A function returning a (pyparsing) parser for parsing C expressions.
Returns:
a (pyparsing) parser for parsing C expressions.
"""
precedence = []
for operators, arity, associativity in _PRECEDENCE:
if arity <= 2:
operators = pyparsing.Or(map(pyparsing.Literal, operators))
else:
operators = tuple(map(pyparsing.Literal, operators))
precedence.append((
operators,
arity,
associativity,
_construct_operator(arity),
))
expression = pyparsing.Forward()
# pylint: disable=expression-not-assigned
expression << pyparsing.infixNotation(
baseExpr=_base_or_array_expression(expression),
opList=precedence,
lpar=pyparsing.NoMatch(),
rpar=pyparsing.NoMatch(),
)
expression.ignore(pyparsing.cppStyleComment)
return expression
def __init__(self, s):
num_int = pp.Regex(r"[+-]?[0-9]+").setParseAction(
lambda toks: int(toks[0]))
num_hex = pp.Regex(r"0x[0-9a-fA-F]+").setParseAction(
lambda toks: int(toks[0], 0))
num_float = pp.Regex(r"[+-]?[0-9]*\.?[0-9]+(:?[eE][+-]?[0-9]+)?"
).setParseAction(lambda toks: float(toks[0]))
num = num_int ^ num_hex ^ num_float
identifier = pp.Regex(r"[A-Za-z_][A-Za-z0-9_.]*")
functor = identifier
varname = identifier.setParseAction(self.add_var)
lparen = pp.Literal("(").suppress()
rparen = pp.Literal(")").suppress()
expr = pp.Forward()
func_args = pp.delimitedList(expr, ',')
func_call = pp.Group(functor + lparen + pp.Optional(func_args) +
rparen)
atom = func_call | num | varname
arith_expr = pp.infixNotation(atom, self.arith_def)
expr <<= arith_expr
self.expr = expr
self.vars = set()
self.parsed = expr.parseString(s, parseAll=True)
def build_parser():
operators = (Literal("=")
| Literal("==")
| Literal("eq")
| Literal("<")
| Literal("lt")
| Literal(">")
| Literal("gt")
| Literal("<=")
| Literal("le")
| Literal(">=")
| Literal("ge")
| Literal("!=")
| Literal("ne")
| Literal("in")
| Literal("and")
| Literal("or"))
field = Word(alphanums)
value = Word(alphanums)
comparison = field + operators + value
query = infixNotation(
comparison,
[
("and", 2, opAssoc.LEFT, NestedExpr),
("or", 2, opAssoc.LEFT, NestedExpr),
("in", 2, opAssoc.LEFT, NestedExpr),
],
)
comparison.addParseAction(ComparisonExpr)
return query
def makeparser(values):
SimpleExpression = PF_KEYWORD('pfvalue') + AttOperator('operator') + AttOperand('testvalue')
booleanrule = infixNotation( SimpleExpression,
[
("!", 1, opAssoc.RIGHT, BoolNot),
("&&", 2, opAssoc.LEFT, BoolAnd),
("||", 2, opAssoc.LEFT, BoolOr),
])
def evalResult(loc,pos,tokens):
modifiers=None
l=len(tokens)
if l==3:
pfixname,op,checkval=tokens
elif l==4:
pfixname,op,checkval,modifiers=tokens
else:
pfixname = op = checkval = None
logging.error("Parser error, got unexpected token amount, tokens=%s"%tokens)
#print "checking %s %s %s"%(pfixname,op,checkval)
return ValueChecker(values,pfixname,op,checkval,modifiers)
SimpleExpression.setParseAction(evalResult)
#SimpleExpression.setDebug()
configline=booleanrule + ACTION + restOfLine
return configline
def BoolstrResult(expr, true_variables):
"""Determine if a boolean expression is satisfied.
BoolstrResult('A and B and not C', {'A', 'C'}) -> False
Args:
expr: The orginal boolean expression, like 'A and B'.
true_variables: Collection to be checked whether satisfy the boolean expr.
Returns:
True if the given |true_variables| cause the boolean expression |expr| to
be satisfied, False otherwise.
"""
boolstr = _ExprOverwrite(expr, true_variables)
# Define the boolean logic
TRUE = pyparsing.Keyword('True')
FALSE = pyparsing.Keyword('False')
boolOperand = TRUE | FALSE
boolOperand.setParseAction(_BoolOperand)
# Define expression, based on expression operand and list of operations in
# precedence order.
boolExpr = pyparsing.infixNotation(boolOperand, [
('not', 1, pyparsing.opAssoc.RIGHT, _BoolNot),
('and', 2, pyparsing.opAssoc.LEFT, _BoolAnd),
('or', 2, pyparsing.opAssoc.LEFT, _BoolOr),
])
try:
res = boolExpr.parseString(boolstr)[0]
return bool(res)
except (AttributeError, pyparsing.ParseException):
raise BoolParseError(
'Cannot parse the boolean expression string "%s".' % expr)
def makeparser(values):
SimpleExpression = PF_KEYWORD('pfvalue') + AttOperator('operator') + AttOperand('testvalue')
booleanrule = infixNotation( SimpleExpression,
[
("!", 1, opAssoc.RIGHT, BoolNot),
("&&", 2, opAssoc.LEFT, BoolAnd),
("||", 2, opAssoc.LEFT, BoolOr),
])
def evalResult(loc,pos,tokens):
modifiers=None
l=len(tokens)
if l==3:
pfixname,op,checkval=tokens
elif l==4:
pfixname,op,checkval,modifiers=tokens
else:
pfixname = op = checkval = None
logging.error("Parser error, got unexpected token amount, tokens=%s"%tokens)
#print "checking %s %s %s"%(pfixname,op,checkval)
return ValueChecker(values,pfixname,op,checkval,modifiers)
SimpleExpression.setParseAction(evalResult)
#SimpleExpression.setDebug()
configline=booleanrule + ACTION + restOfLine
return configline
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack, rbrack, lbrace, rbrace, lparen, rparen, colon, qmark = map(
Literal, "[]{}():?")
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(
c for c in printables if c not in r"\[]{}().*?+|") + " \t"
reRange = Combine(lbrack + SkipTo(rbrack, ignore=escapedChar) + rbrack)
reLiteral = (escapedChar | oneOf(list(reLiteralChar)))
reNonCaptureGroup = Suppress("?:")
reDot = Literal(".")
repetition = ((lbrace + Word(nums).setResultsName("count") + rbrace) |
(lbrace + Word(nums).setResultsName("minCount") + "," +
Word(nums).setResultsName("maxCount") + rbrace)
| oneOf(list("*+?")))
reRange.setParseAction(handleRange)
reLiteral.setParseAction(handleLiteral)
reMacro.setParseAction(handleMacro)
reDot.setParseAction(handleDot)
reTerm = (reLiteral | reRange | reMacro | reDot | reNonCaptureGroup)
reExpr = infixNotation(reTerm, [
(repetition, 1, opAssoc.LEFT, handleRepetition),
(None, 2, opAssoc.LEFT, handleSequence),
(Suppress('|'), 2, opAssoc.LEFT, handleAlternative),
])
_parser = reExpr
return _parser
def __init__(self, baseExpr, opList=[], lpar=LPAREN, rpar=RPAREN, *args, **kwargs):
super(MixedExpression, self).__init__(baseExpr, *args, **kwargs)
self.baseExpr = baseExpr
self.opList = opList
self.lpar = lpar
self.rpar = rpar
self.expr = pp.infixNotation(baseExpr, opList, lpar, rpar)
def query_from_string(cls, filter_string):
'''
TODO:
* handle values with " via: a.b.c.d="hello\"world"
* handle keys with " via: a.\"b.c="yeah"
* handle key with __ in it
'''
filter_string_raw = filter_string
filter_string = str(filter_string)
unicode_spaces = list(set(str(c) for c in filter_string if c.isspace()))
unicode_spaces_other = unicode_spaces + [u'(', u')', u'=', u'"']
atom = CharsNotIn(unicode_spaces_other)
atom_inside_quotes = CharsNotIn(u'"')
atom_quoted = Literal('"') + Optional(atom_inside_quotes) + Literal('"')
EQUAL = Literal('=')
grammar = ((atom_quoted | atom) + EQUAL + Optional((atom_quoted | atom)))
grammar.setParseAction(cls.BoolOperand)
boolExpr = infixNotation(grammar, [
("and", 2, opAssoc.LEFT, cls.BoolAnd),
("or", 2, opAssoc.LEFT, cls.BoolOr),
])
try:
res = boolExpr.parseString('(' + filter_string + ')')
except ParseException:
raise RuntimeError(u"Invalid query %s" % filter_string_raw)
if len(res) > 0:
return res[0].result
raise RuntimeError("Parsing the filter_string %s went terribly wrong" % filter_string)
def parse_payload(self, payload):
expr = Forward()
LPAR, RPAR, SEMI = map(Suppress, "();")
identifier = Word(alphas + "_", alphanums + "_")
function_call = identifier.setResultsName("name") + LPAR + Group(
Optional(delimitedList(expr))) + RPAR
integer = Regex(r"-?\d+")
real = Regex(r"-?\d+\.\d*")
qstr = QuotedString(quoteChar='"', escChar='\\', unquoteResults=False)
qstrsingle = QuotedString(quoteChar="'",
escChar='\\',
unquoteResults=False)
operand = (identifier | real | integer | qstr | qstrsingle)
plusop = oneOf('+ -')
expr << infixNotation(operand, [(plusop, 2, opAssoc.LEFT)])
out = []
for t, s, e in function_call.scanString(payload):
out.append({
"action":
t[0],
"arguments":
t[1].asList() if type(t[1]) != str else t[1]
})
return out
def _parse_expr(text, ldelim='(', rdelim=')'):
""" Parse mathematical expression using PyParsing """
var = pyparsing.Word(pyparsing.alphas+'_', pyparsing.alphanums+'_')
point = pyparsing.Literal('.')
exp = pyparsing.CaselessLiteral('E')
number = pyparsing.Combine(
pyparsing.Word('+-'+pyparsing.nums, pyparsing.nums)+
pyparsing.Optional(
point+pyparsing.Optional(pyparsing.Word(pyparsing.nums))
)+
pyparsing.Optional(
exp+pyparsing.Word('+-'+pyparsing.nums, pyparsing.nums)
)
)
atom = var | number
oplist = [
(pyparsing.Literal('**'), 2, pyparsing.opAssoc.RIGHT),
(pyparsing.oneOf('+ - ~'), 1, pyparsing.opAssoc.RIGHT),
(pyparsing.oneOf('* / // %'), 2, pyparsing.opAssoc.LEFT),
(pyparsing.oneOf('+ -'), 2, pyparsing.opAssoc.LEFT),
(pyparsing.oneOf('<< >>'), 2, pyparsing.opAssoc.LEFT),
(pyparsing.Literal('&'), 2, pyparsing.opAssoc.LEFT),
(pyparsing.Literal('^'), 2, pyparsing.opAssoc.LEFT),
(pyparsing.Literal('|'), 2, pyparsing.opAssoc.LEFT),
]
# Get functions
expr = pyparsing.infixNotation(
atom,
oplist,
lpar=pyparsing.Suppress(ldelim),
rpar=pyparsing.Suppress(rdelim)
)
return expr.parseString(text)[0]
def _parse_expr(text, ldelim="(", rdelim=")"):
"""Parse mathematical expression using PyParsing."""
var = pyparsing.Word(pyparsing.alphas + "_", pyparsing.alphanums + "_")
point = pyparsing.Literal(".")
exp = pyparsing.CaselessLiteral("E")
number = pyparsing.Combine(
pyparsing.Word("+-" + pyparsing.nums, pyparsing.nums)
+ pyparsing.Optional(point + pyparsing.Optional(pyparsing.Word(pyparsing.nums)))
+ pyparsing.Optional(
exp + pyparsing.Word("+-" + pyparsing.nums, pyparsing.nums)
)
)
atom = var | number
oplist = [
(pyparsing.Literal("**"), 2, pyparsing.opAssoc.RIGHT),
(pyparsing.oneOf("+ - ~"), 1, pyparsing.opAssoc.RIGHT),
(pyparsing.oneOf("* / // %"), 2, pyparsing.opAssoc.LEFT),
(pyparsing.oneOf("+ -"), 2, pyparsing.opAssoc.LEFT),
(pyparsing.oneOf("<< >>"), 2, pyparsing.opAssoc.LEFT),
(pyparsing.Literal("&"), 2, pyparsing.opAssoc.LEFT),
(pyparsing.Literal("^"), 2, pyparsing.opAssoc.LEFT),
(pyparsing.Literal("|"), 2, pyparsing.opAssoc.LEFT),
]
# Get functions
expr = pyparsing.infixNotation(
atom, oplist, lpar=pyparsing.Suppress(ldelim), rpar=pyparsing.Suppress(rdelim)
)
return expr.parseString(text)[0]
def __init__(self, nominator: str, denominator: str):
func = Word(alphas)
unit_type = Word(alphas + "_").setParseAction(UnitType)
agg_type = Word(alphas).setParseAction(AggType)
goal = Word(alphas + "_" + nums).setParseAction(Goal)
number = Word(nums).setParseAction(Number)
dimension = Word(alphas + "_").setParseAction(Dimension)
dimension_value = Word(alphanums + "_" + "-" + "." + "%").setParseAction(DimensionValue)
ep_goal = (func + "(" + unit_type + "." + agg_type + "." + goal + ")").setParseAction(EpGoal)
ep_goal_with_dimension = (
func + "(" + unit_type + "." + agg_type + "." + goal + "(" + dimension + "=" + dimension_value + ")" + ")"
).setParseAction(EpGoal)
operand = number | ep_goal | ep_goal_with_dimension
multop = oneOf("*")
divop = oneOf("/")
plusop = oneOf("+")
subop = oneOf("-")
expr = infixNotation(
operand,
[
(multop, 2, opAssoc.LEFT, MultBinOp),
(divop, 2, opAssoc.LEFT, DivBinOp),
(plusop, 2, opAssoc.LEFT, PlusBinOp),
(subop, 2, opAssoc.LEFT, SubBinOp),
],
)
self._nominator_expr = expr.parseString(nominator)[0]
self._denominator_expr = expr.parseString(denominator)[0]
def BoolstrResult(expr, true_variables):
"""Determine if a boolean expression is satisfied.
BoolstrResult('A and B and not C', {'A', 'C'}) -> False
Args:
expr: The orginal boolean expression, like 'A and B'.
true_variables: Collection to be checked whether satisfy the boolean expr.
Returns:
True if the given |true_variables| cause the boolean expression |expr| to
be satisfied, False otherwise.
"""
boolstr = _ExprOverwrite(expr, true_variables)
# Define the boolean logic
TRUE = pyparsing.Keyword('True')
FALSE = pyparsing.Keyword('False')
boolOperand = TRUE | FALSE
boolOperand.setParseAction(_BoolOperand)
# Define expression, based on expression operand and list of operations in
# precedence order.
boolExpr = pyparsing.infixNotation(
boolOperand, [('not', 1, pyparsing.opAssoc.RIGHT, _BoolNot),
('and', 2, pyparsing.opAssoc.LEFT, _BoolAnd),
('or', 2, pyparsing.opAssoc.LEFT, _BoolOr),])
try:
res = boolExpr.parseString(boolstr)[0]
return bool(res)
except (AttributeError, pyparsing.ParseException):
raise BoolParseError('Cannot parse the boolean expression string "%s".'
% expr)
def __init__(self):
ParserElement.enablePackrat()
decimal_integer = Word(nums).setName('decimal integer') \
.setParseAction(lambda t: int(''.join(t)))
hexadecimal_integer = Combine(Word(nums, hexnums) + Word('hH')) \
.setName('hexadecimal integer') \
.setParseAction(lambda t: int((''.join(t))[:-1], 16))
identifier = Word(alphas, alphanums + '_@?') \
.setName('identifier')
# XXX and maybe dollar sign?
baseExpr = (hexadecimal_integer | decimal_integer | identifier)
operators = [
(oneOf('+ - ~'), 1, opAssoc.RIGHT, self.nest_operand_pairs),
(oneOf('* /'), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('+ -'), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('<< >>'), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('&'), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('^'), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('|'), 2, opAssoc.LEFT, self.nest_operand_pairs),
]
self.expr = infixNotation(baseExpr, operators) + StringEnd()
def _evaluate_expression(expr_list, keyword_list, referred_checks_result):
# Convert the expression now in the format to be parsed by pyparsing module
parsed_list = []
for expr in expr_list:
if expr.upper() not in keyword_list:
# Check reference is passed. Pass the fetched value instead of original check id
parsed_list.append(referred_checks_result.get(expr))
else:
parsed_list.append(expr.upper())
parsed_expr = " ".join(parsed_list)
# Logic to use boolean expression parser using pyparsing library
# We are passing the boolean expression in the following form:
# check1 and not (check2 or (check3 and not check4) )
# --> check1 and not ( check2 or ( check3 and not check4 ) )
# --> True and not ( False or ( True and not False ) )
ParserElement.enablePackrat()
TRUE = Keyword("True")
FALSE = Keyword("False")
boolOperand = TRUE | FALSE | Word(alphas, max=1)
boolOperand.setParseAction(BoolOperand)
boolExpr = infixNotation(
boolOperand,
[
("NOT", 1, opAssoc.RIGHT, BoolNot),
("AND", 2, opAssoc.LEFT, BoolAnd),
("OR", 2, opAssoc.LEFT, BoolOr),
],
)
return boolExpr.parseString(parsed_expr)[0]
def _create_parser(self):
OPERATORS = ComparisonExpr.OPERATORS.keys()
AND = oneOf(LogicExpr.AND)
OR = oneOf(LogicExpr.OR)
FIELD = Word(alphanums + '_')
OPERATOR = oneOf(OPERATORS)
VALUE = (Word(nums + '-.')
| QuotedString(quoteChar="'", unquoteResults=False)(alphanums)
| QuotedString('[', endQuoteChar=']',
unquoteResults=False)(alphanums + "'-.")
| CaselessKeyword('true') | CaselessKeyword('false')
| CaselessKeyword('notblank'))
COMPARISON = FIELD + OPERATOR + VALUE
QUERY = infixNotation(COMPARISON, [
(
AND,
2,
opAssoc.LEFT,
),
(
OR,
2,
opAssoc.LEFT,
),
])
COMPARISON.addParseAction(ComparisonExpr)
AND.addParseAction(LogicExpr)
OR.addParseAction(LogicExpr)
return QUERY
def parse_string(logicstr):
''' Parse the logic string using pyparsing '''
and_ = pyparsing.Keyword('and')
or_ = pyparsing.Keyword('or')
nor_ = pyparsing.Keyword('nor')
nand_ = pyparsing.Keyword('nand')
xor_ = pyparsing.Keyword('xor')
xnor_ = pyparsing.Keyword('xnor')
not_ = pyparsing.Keyword('not')
true_ = pyparsing.Keyword('true')
false_ = pyparsing.Keyword('false')
not_op = not_ | '~' | '¬'
and_op = and_ | nand_ | '&' | '∧'
xor_op = xor_ | xnor_ | '⊕' | '⊻'
or_op = or_ | nor_ | '|' | '∨' | '+'
expr = pyparsing.Forward()
identifier = ~(and_ | or_ | nand_ | nor_ | not_ | true_ | false_) + \
pyparsing.Word('$' + pyparsing.alphas + '_', pyparsing.alphanums +'_'+'$')
atom = identifier | pyparsing.Group('(' + expr + ')')
factor = pyparsing.Group(pyparsing.ZeroOrMore(not_op) + atom)
term = pyparsing.Group(factor + pyparsing.ZeroOrMore(and_op + factor))
expr = pyparsing.infixNotation(true_ | false_ | identifier,
[(not_op, 1, pyparsing.opAssoc.RIGHT),
(and_op, 2, pyparsing.opAssoc.LEFT),
(or_op, 2, pyparsing.opAssoc.LEFT),
(xor_op, 2, pyparsing.opAssoc.LEFT)])
return expr.parseString(logicstr)[0]
def parse1(expression):
arith = pp.infixNotation(
pp.Word(pp.nums),
[(pp.oneOf("+ *"), 2, pp.opAssoc.LEFT)],
)
return arith.parseString(expression).asList()
def __init__(self):
ParserElement.enablePackrat()
hexadecimal_integer = Combine(CaselessLiteral('0x') + Word(hexnums)) \
.setName('hexadecimal integer') \
.setParseAction(lambda *t: int(t[2][0][2:], 16))
decimal_integer = Word(nums) \
.setName('decimal integer') \
.setParseAction(lambda t: int(''.join(t)))
identifier = Word(alphanums + '_$') \
.setName('identifier')
baseExpr = (hexadecimal_integer | decimal_integer | identifier)
operators = [
(oneOf('+ - ~ !'), 1, opAssoc.RIGHT, self.nest_operand_pairs),
(oneOf('* /'), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('+ -'), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('<< >>'), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('<= < > >='), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('== !='), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('&'), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('^'), 2, opAssoc.LEFT, self.nest_operand_pairs),
(oneOf('|'), 2, opAssoc.LEFT, self.nest_operand_pairs),
]
self.expr = infixNotation(baseExpr, operators) + StringEnd()
def _get_condition_expression():
global _condition_expression
if not _condition_expression:
operator = pp.Regex('==|!=|>=|>|<=|<').setName('operator')
operator.setParseAction(_Operator)
identifier = pp.Word('$', pp.alphanums + '_', min=2).setName('identifier')
identifier.setParseAction(_Identifier)
value = pp.Word(pp.alphanums + '_-').setName('value')
value.setParseAction(_Value)
double_quoted_value = pp.QuotedString('"').setName(
'double_quoted_value')
double_quoted_value.setParseAction(_Value)
single_quoted_value = pp.QuotedString("'").setName(
'single_quoted_value')
single_quoted_value.setParseAction(_Value)
comparison_term = identifier | value | double_quoted_value | \
single_quoted_value
condition = pp.Group(comparison_term + operator + comparison_term).setName('condition')
condition.setParseAction(_Condition)
_condition_expression = infixNotation(
condition, [
('and', 2, pp.opAssoc.LEFT, _And),
('or', 2, pp.opAssoc.LEFT, _Or),
])
return _condition_expression
def __init__(self, nominator: str, denominator: str):
func = Word(alphas)
unit_type = Word(alphas + '_').setParseAction(UnitType)
agg_type = Word(alphas).setParseAction(AggType)
goal = Word(alphas + '_').setParseAction(Goal)
number = Word(nums).setParseAction(Number)
dimension = Word(alphas + '_').setParseAction(Dimension)
dimension_value = Word(alphanums + '_' + '-' +
'.').setParseAction(DimensionValue)
ep_goal = (func + '(' + unit_type + '.' + agg_type + '.' + goal +
')').setParseAction(EpGoal)
ep_goal_with_dimension = (func + '(' + unit_type + '.' + agg_type +
'.' + goal + '(' + dimension + '=' +
dimension_value + ')' +
')').setParseAction(EpGoal)
operand = number | ep_goal | ep_goal_with_dimension
multop = oneOf('*')
divop = oneOf('/')
plusop = oneOf('+')
subop = oneOf('-')
expr = infixNotation(
operand,
[
(multop, 2, opAssoc.LEFT, MultBinOp),
(divop, 2, opAssoc.LEFT, DivBinOp),
(plusop, 2, opAssoc.LEFT, PlusBinOp),
(subop, 2, opAssoc.LEFT, SubBinOp),
],
)
self._nominator_expr = expr.parseString(nominator)[0]
self._denominator_expr = expr.parseString(denominator)[0]
def _create_parser() -> ParserElement:
# operators in the format later used by infixNotation
operator_list = [
(None, 2, opAssoc.LEFT, BooleanAndOperation._create_from_implicit_tokens),
(CaselessKeyword('not') | "~" | "!", 1, opAssoc.RIGHT, BooleanNotOperation._create_from_tokens),
(CaselessKeyword('and') | "&", 2, opAssoc.LEFT, BooleanAndOperation._create_from_tokens),
(CaselessKeyword('xor') | "^", 2, opAssoc.LEFT, BooleanXorOperation._create_from_tokens),
(CaselessKeyword('or') | "|", 2, opAssoc.LEFT, BooleanOrOperation._create_from_tokens),
]
# terms (atoms) that will be combined with the boolean operators
term_list = [
(CaselessKeyword('tag'), TagFilterTerm._create_from_tokens),
(CaselessKeyword('ingr'), IngredientFilterTerm._create_from_tokens),
(CaselessKeyword('unit'), UnitFilterTerm._create_from_tokens),
(None, AnyFilterTerm._create_from_tokens),
]
# extract keywords that can
operator_expressions = [om[0] for om in operator_list if om[0] is not None]
term_expressions = [tm[0] for tm in term_list if tm[0] is not None]
reserved_expressions = operator_expressions + term_expressions
# quoted string indicates exact macthc
quoted_filter_string = (QuotedString('"', escChar='\\') | QuotedString("'", escChar='\\')).setResultsName('string')
# quoted_filter_string.setDebug(True)
quoted_filter_string.setName("quoted_filter_string")
quoted_filter_string.setParseAction(ExactFilterString._create_from_tokens)
# not quoted string is inexact match, can't contain whitespace or be an operator
unquoted_filter_string = ~MatchFirst(reserved_expressions) + Regex(r'[^\s\(\)]+', flags=re.U).setResultsName('string')
# unquoted_filter_string.setDebug(True)
unquoted_filter_string.setName("unquoted_filter_string")
unquoted_filter_string.setParseAction(FuzzyFilterString._create_from_tokens)
# regular expressions aren't parsed in the grammar but delegated to python re.compile in the parser action
regex_filter_string = QuotedString('/', escChar='\\')
regex_filter_string.setName("regex_filter_string")
regex_filter_string.setParseAction(RegexFilterString._create_from_tokens)
# unquoted_filter_string must be last, so that initial quotes are handled correctly
filter_string = regex_filter_string | quoted_filter_string | unquoted_filter_string
filter_string.setParseAction(lambda toks: toks[0])
filter_terms = []
for prefix_expression, term_action in term_list:
if prefix_expression is not None:
filter_term = Combine(prefix_expression + ':' + filter_string.setResultsName("filter_string"))
filter_term.setName("filter_term_"+str(prefix_expression.match))
else:
filter_term = filter_string.setResultsName("filter_string")
filter_term.setName("filter_term_None")
# filter_term.setDebug(True)
filter_term.addParseAction(term_action)
filter_terms.append(filter_term)
filter_term = MatchFirst(filter_terms)
filter_expr = infixNotation(filter_term, operator_list)
return filter_expr
def enableCall(self, action=CallOpAction):
EXP = self.expr
KWARG = IDEN + pp.Suppress('=') + EXP
# STAR = pp.Suppress('*') + EXP, DBLSTAR = pp.Suppress('**') + EXP
callop = LPAREN + pp.Optional(pp.delimitedList(EXP))('args') + pp.Optional(pp.delimitedList(KWARG))('kwargs') + RPAREN
callop.setParseAction(action)
self.opList.insert(0, callop)
self.expr <<= pp.infixNotation(self.baseExpr, self.opList, self.lpar, self.rpar)
def parse2(expression):
arith = pp.infixNotation(
pp.Word(pp.nums),
[(pp.Literal("+"), 2, pp.opAssoc.LEFT),
(pp.Literal("*"), 2, pp.opAssoc.LEFT)],
)
return arith.parseString(expression).asList()
def enableIndex(self, action=IndexOpAction):
# start:stop:step
EXP = pp.Forward()
SLICE = pp.Optional(EXP)('start') + COLON + pp.Optional(EXP)('stop') + pp.Optional(COLON + pp.Optional(EXP)('step'))
indexop = LBRACK + (SLICE('slice') | EXP('index')) + RBRACK
indexop.setParseAction(action)
self.opList.insert(0, indexop)
self.expr <<= pp.infixNotation(EXP, self.opList, self.lpar, self.rpar)
def generate_parser(self, xyz=None):
"""Creates the pyparsing expression based on geometry.
The syntax is as follows:
- ``i+`` are indices of xyz and return vectors.
- ``i+.j`` are floating point numbers (j optional).
- ``i[j]`` is the j-th (scalar) element of xyz[i].
- ``X, Y, Z`` are unit vectors along x, y and z axes (uppercase only).
- ``+`` and ``-`` are addition/subtraction of vectors or scalars.
- ``*`` and ``/`` are multiplication/division of vectors and scalars
(elementwise).
- ``o`` and ``x`` are scalar/vector products of vectors only.
- ``^`` is the power of a vector/scalar by a scalar (elementwise).
- ``(`` and ``)`` specify order of operation.
- ``[i, j, k]`` gives a vector with scalar elements i, j and k.
Parameters
----------
xyz : (N, 3), array_like, optional
The cartesian geometry used in index expressions. If not
provided, strings containing indices will raise an error.
Returns
-------
pyparsing.Forward
A pyparsing grammar definition.
"""
expr = pp.Forward()
# operand types: int, int with index, float, axis or delimited list
intnum = pp.Word(pp.nums)
fltind = pp.Word(pp.nums) + '[' + pp.Word(pp.nums) + ']'
fltnum = pp.Combine(
pp.Word(pp.nums) + '.' + pp.Optional(pp.Word(pp.nums)))
alphax = pp.oneOf(' '.join(self.axes))
dllist = pp.Suppress('[') + pp.delimitedList(expr) + pp.Suppress(']')
intnum.setParseAction(lambda t: xyz[int(t[0])])
fltind.setParseAction(lambda t: xyz[int(t[0])][int(t[2])])
fltnum.setParseAction(lambda t: float(t[0]))
alphax.setParseAction(lambda t: self.axes[t[0]])
dllist.setParseAction(lambda t: np.array(t[:]))
operand = dllist | alphax | fltnum | fltind | intnum
# operators: unary, power, binary multiplication/division,
# binary addition/subtraction
sgnop = pp.oneOf(' '.join(self.unop))
expop = pp.Literal('^')
mulop = pp.oneOf(' '.join(self.bnmul))
addop = pp.oneOf(' '.join(self.bnadd))
# set operator precedence
prec = [(sgnop, 1, pp.opAssoc.RIGHT, self._eval_unary),
(expop, 2, pp.opAssoc.LEFT, self._eval_power),
(mulop, 2, pp.opAssoc.LEFT, self._eval_binary),
(addop, 2, pp.opAssoc.LEFT, self._eval_binary)]
return expr << pp.infixNotation(operand, prec)
def get_term_bnf():
global term_bnf
if not term_bnf:
val = Word(alphanums).setParseAction(DescValNode.get_parse_action())
term_bnf = infixNotation(
val,
[
(None, 2, opAssoc.LEFT, DescXorNode.get_parse_action())
]
)
return term_bnf
def eval_query(query_str):
global WORD_CHARS
boolOperand = Word(WORD_CHARS)
boolOperand.setParseAction(BoolOperand)
boolExpr = infixNotation(
boolOperand,
[("not", 1, opAssoc.RIGHT, BoolNot), ("and", 2, opAssoc.LEFT, BoolAnd), ("or", 2, opAssoc.LEFT, BoolOr)],
)
return boolExpr.parseString(query_str.lower())[0].calcop()
def __init__(self):
"""
Create a parser that parse arithmetic expressions. They can
contains variable identifiers or raw numbers. The meaning
for the identifiers is left to the
"""
number = p.Regex(r'\d+(\.\d*)?([eE]\d+)?')
identifier = p.Word(p.alphas)
terminal = identifier | number
self._expr = p.infixNotation(terminal, [
(p.oneOf('* /'), 2, p.opAssoc.LEFT),
(p.oneOf('+ -'), 2, p.opAssoc.LEFT)
]) + p.stringEnd()
def get_parser():
op_literal = ((Word(alphanums + ",.-_")
| dblQuotedString.setParseAction(removeQuotes)
| sglQuotedString.addParseAction(removeQuotes)).
addParseAction(toks0(Literal)))
op_tag = (Keyword('tag')
+ Suppress('[')
+ op_literal
+ Suppress(']')).setParseAction(Tag)
op_value = (op_tag
| (oneOf(" ".join(Variable.VARIABLES))
.setParseAction(toks0(Variable))))
op_lhs = op_value
op_rhs = op_value | op_literal
op_compare = (Keyword("=") | Keyword("~") | Keyword("!=") | Keyword("!~"))
op_and = Keyword("and")
op_or = Keyword("or")
op_not = Keyword("not")
op_true = Suppress(Keyword("true")).setParseAction(toksz(AlwaysTrue))
op_false = Suppress(Keyword("false")).setParseAction(toksz(AlwaysFalse))
op_compare_expression = ((op_lhs
+ op_compare
+ op_rhs)
.addParseAction(toks(Comparison)))
op_test_expression = (Group(op_lhs)
.addParseAction(lambda s, l, t: t[0])
.addParseAction(NotNull))
op_value_expression = (op_false
| op_true
| op_compare_expression
| op_test_expression)
op_expression = (
StringStart()
+ infixNotation(
op_value_expression,
[(Suppress(op_not), 1, opAssoc.RIGHT, toks00(Not)),
(Suppress(op_and), 2, opAssoc.LEFT, toks0(And)),
(Suppress(op_or), 2, opAssoc.LEFT, toks0(Or))])
+ StringEnd())
return op_expression
def get_expression_grammar():
identifier = SDKConfig.IDENTIFIER.setResultsName("identifier")
operator = SDKConfig.OPERATOR.setResultsName("operator")
value = SDKConfig.VALUE.setResultsName("value")
test_binary = identifier + operator + value
test_single = identifier
test = test_binary | test_single
condition = Group(Optional("(").suppress() + test + Optional(")").suppress())
grammar = infixNotation(condition, [
("!", 1, opAssoc.RIGHT),
("&&", 2, opAssoc.LEFT),
("||", 2, opAssoc.LEFT)])
return grammar
def _makeExpressionGrammar(atom):
"""
Define the complex string selector grammar using PyParsing (which supports
logical operations and nesting)
"""
#define operators
and_op = Literal('and')
or_op = Literal('or')
delta_op = oneOf(['exc','except'])
not_op = Literal('not')
def atom_callback(res):
return _SimpleStringSyntaxSelector(res)
atom.setParseAction(atom_callback) #construct a simple selector from every matched
#define callback functions for all operations
def and_callback(res):
items = res.asList()[0][::2] #take every secend items, i.e. all operands
return reduce(AndSelector,items)
def or_callback(res):
items = res.asList()[0][::2] #take every secend items, i.e. all operands
return reduce(SumSelector,items)
def exc_callback(res):
items = res.asList()[0][::2] #take every secend items, i.e. all operands
return reduce(SubtractSelector,items)
def not_callback(res):
right = res.asList()[0][1] #take second item, i.e. the operand
return InverseSelector(right)
#construct the final grammar and set all the callbacks
expr = infixNotation(atom,
[(and_op,2,opAssoc.LEFT,and_callback),
(or_op,2,opAssoc.LEFT,or_callback),
(delta_op,2,opAssoc.LEFT,exc_callback),
(not_op,1,opAssoc.RIGHT,not_callback)])
return expr
def expression_parser(self):
"""A function returning a (pyparsing) parser for parsing C expressions.
Returns:
a (pyparsing) parser for parsing C expressions.
"""
precedence = (self._build_precedence(_UNARY_MACROS) +
self._build_precedence(_PRECEDENCE))
self.expression = pyparsing.Forward()
# pylint: disable=expression-not-assigned
self.expression << (
pyparsing.infixNotation(
baseExpr=self._base_or_array_expression(),
opList=precedence,
)
)
return self.expression
def parseLTL(s):
TRUE = Keyword("True")
FALSE = Keyword("False")
symbol = TRUE | FALSE | Word("abcdefghijklmnopqrstuvwxyz")
equation = symbol + "==" + symbol | symbol + "<" + symbol | symbol
symbol.setParseAction(Const)
expr= infixNotation(symbol,
[
(negop, 1, opAssoc.RIGHT, Negltl),
(fop, 1, opAssoc.RIGHT, Fltl),
(gop, 1, opAssoc.RIGHT, Gltl),
(xop, 1, opAssoc.RIGHT, Xltl),
(andop, 2, opAssoc.RIGHT, Andltl),
(orop, 2, opAssoc.RIGHT, Orltl),
(impop, 2, opAssoc.RIGHT, Impltl),
(uop, 2, opAssoc.RIGHT, Ultl),
(vop, 2, opAssoc.RIGHT, Vltl)
])
return expr.parseString(s)[0]
def parser():
global _parser
if _parser is None:
ParserElement.setDefaultWhitespaceChars("")
lbrack,rbrack,lbrace,rbrace,lparen,rparen,colon,qmark = map(Literal,"[]{}():?")
reMacro = Combine("\\" + oneOf(list("dws")))
escapedChar = ~reMacro + Combine("\\" + oneOf(list(printables)))
reLiteralChar = "".join(c for c in printables if c not in r"\[]{}().*?+|") + " \t"
reRange = Combine(lbrack + SkipTo(rbrack,ignore=escapedChar) + rbrack)
reLiteral = ( escapedChar | oneOf(list(reLiteralChar)) )
reNonCaptureGroup = Suppress("?:")
reDot = Literal(".")
repetition = (
( lbrace + Word(nums).setResultsName("count") + rbrace ) |
( lbrace + Word(nums).setResultsName("minCount")+","+ Word(nums).setResultsName("maxCount") + rbrace ) |
oneOf(list("*+?"))
)
reRange.setParseAction(handleRange)
reLiteral.setParseAction(handleLiteral)
reMacro.setParseAction(handleMacro)
reDot.setParseAction(handleDot)
reTerm = ( reLiteral | reRange | reMacro | reDot | reNonCaptureGroup)
reExpr = infixNotation( reTerm,
[
(repetition, 1, opAssoc.LEFT, handleRepetition),
(None, 2, opAssoc.LEFT, handleSequence),
(Suppress('|'), 2, opAssoc.LEFT, handleAlternative),
]
)
_parser = reExpr
return _parser
def _handle_logical_argument(arg):
if not isinstance(arg, (BinaryExpression, UnaryExpression, bool, BooleanClauseList)):
assert isinstance(arg, str), 'Invalid argument: {!r}'.format(arg)
arg = get_current_logic().get_fallback_filter(arg)
return arg
alphanums_plus = alphanums + '_-/@.:'
identifier = Word(alphanums_plus)
LPAR, RPAR = map(Suppress, '()')
DQUOTE = Suppress('"')
SQUOTE = Suppress("'")
func_call = Group(oneOf(list(_FUNCTIONS)) + LPAR +
identifier + RPAR).setParseAction(_handle_func_call)
atom = func_call | identifier | (DQUOTE + Word(alphanums_plus + ' <>') + DQUOTE) | (SQUOTE + Word(alphanums_plus + ' <>') + SQUOTE)
binop = oneOf(list(_OPERATORS))
and_ = Keyword("and", caseless=True)
or_ = Keyword("or", caseless=True)
grammar = infixNotation(atom, [
(binop, 2, opAssoc.LEFT, _handle_binop),
(and_, 2, opAssoc.LEFT, _handle_and),
(or_, 2, opAssoc.LEFT, _handle_or),
])
integer = Word(nums)
real = Combine(Word(nums) + "." + Word(nums))
variable = Word(alphas,exact=1)
operand = real | integer | variable
signop = oneOf('+ -')
multop = oneOf('* /')
plusop = oneOf('+ -')
expop = Literal('**')
# use parse actions to attach EvalXXX constructors to sub-expressions
operand.setParseAction(EvalConstant)
arith_expr = infixNotation(operand,
[
(signop, 1, opAssoc.RIGHT, EvalSignOp),
(expop, 2, opAssoc.LEFT, EvalPowerOp),
(multop, 2, opAssoc.LEFT, EvalMultOp),
(plusop, 2, opAssoc.LEFT, EvalAddOp),
])
comparisonop = oneOf("< <= > >= != = <> LT GT LE GE EQ NE")
comp_expr = infixNotation(arith_expr,
[
(comparisonop, 2, opAssoc.LEFT, EvalComparisonOp),
])
def main():
# sample expressions posted on comp.lang.python, asking for advice
# in safely evaluating them
rules=[
'( A - B ) = 0',
upto_N_words = Group(lpar + 'upto' + pyparsing_common.integer('numberofwords') + 'words' + rpar)
upto_N_chars = Group(lpar + 'upto' + pyparsing_common.integer('numberofchars') + 'characters' + rpar)
phrase_term = Group(OneOrMore(phrase_word) + ZeroOrMore(((upto_N_words) | ZeroOrMore(upto_N_chars)) + OneOrMore(phrase_word))) | quotedString # changed phrase_word to OneOrMore(phrase_word) to allow multi-word phrases
# phrase-term
# want to know if there is a way to just say that the phrase_term can contain both types of elements instead of
# having to give the exact grammar for it as p_w+upto+p_W
phrase_expr = infixNotation(phrase_term,
[
((BEFORE | AFTER | JOIN), 2, opAssoc.LEFT,), # (opExpr, numTerms, rightLeftAssoc, parseAction)
(NOT, 1, opAssoc.RIGHT,),
(AND, 2, opAssoc.LEFT,),
(OR, 2, opAssoc.LEFT),
],
lpar=Suppress('{'), rpar=Suppress('}')
) # structure of a single phrase with its operators
line_term = Group((LINE_CONTAINS | LINE_STARTSWITH | LINE_ENDSWITH)("line_directive") +
Group(phrase_expr)("phrase")) # basically giving structure to a single sub-rule having line-term and phrase
line_contents_expr = infixNotation(line_term,
[(NOT, 1, opAssoc.RIGHT,),
(AND, 2, opAssoc.LEFT,),
(OR, 2, opAssoc.LEFT),
]
) # grammar for the entire rule/sentence
sample1 = """
def __init__(self):
self._AST = Syntax_tree()
# keywords
self.int_ = Keyword('Int')
self.false_ = Keyword('False')
self.true_ = Keyword('True')
self.bit_ = Combine(Optional(Literal("@")) + Keyword('Bit'))
self.sbox_ = Keyword('Sbox')
self.l_shift_ = Keyword('<<')
self.r_shift_ = Keyword('>>')
self.circ_l_shift_ = Keyword('<<<')
self.circ_r_shift_ = Keyword('>>>')
self.bit_val = self.false_ ^ self.true_
self.if_ = Keyword('if')
self.for_ = Keyword('for')
self.return_ = Keyword('return')
self.void_ = Keyword('void')
self.ID = NotAny(self.sbox_ ^ self.int_ ^ self.bit_ ^ self.false_ ^ self.true_ ^ self.if_ ^ self.for_ ^ self.sbox_) + Word(alphas + '_', alphanums + '_') # NOQA
self.ID_ = NotAny(self.sbox_ ^ self.int_ ^ self.bit_ ^ self.false_ ^ self.true_ ^ self.if_ ^ self.for_ ^ self.sbox_) + Word(alphas + '_', alphanums + '_')
# Other Tokens
self.l_bracket = Literal('(')
self.r_bracket = Literal(')')
self.eq_set = Literal('=')("set")
self.term_st = Literal(';')
self.b_2_num = Combine(Literal("0b") + Word("01"))
self.b_2_num.setParseAction(self.semantic_analyser.convert_base_to_str)
self.b_16_num = Combine(Literal("0x") + Word(srange("[0-9a-fA-F]")))
self.b_16_num.setParseAction(self.semantic_analyser.convert_base_to_str)
self.b_10_num = Word(nums)
self.bit_and = Literal('&')
self.bit_or = Keyword('|')
self.bit_xor = Keyword('^')
self.bit_not = Literal('~')
self.eq_compare = Literal('==')
self.neq_compare = Literal('!=')
self.l_brace = Literal('{')
self.r_brace = Literal('}')
self.bin_add = Literal('+')
self.bin_mult = Literal('*')
self.bin_sub = Literal('-')
self.bin_mod = Literal('%')
self.bin_div = Literal('/')
self.g_than = Literal('>')
self.ge_than = Literal('>=')
self.l_than = Literal('<')
self.le_than = Literal('<=')
self.log_and = Keyword('&&')
self.log_or = Keyword('||')
self.l_sq_b = Literal('[')
self.r_sq_b = Literal(']')
# Operator Productions
self.log_op = self.log_and ^ self.log_or
self.comparison_op = self.g_than ^ self.ge_than ^ self.l_than ^ self.le_than ^ self.eq_compare ^ self.neq_compare
self.arith_op = self.bin_add ^ self.bin_mult ^ self.bin_sub ^ self.bin_mod ^ self.bin_div
self.bitwise_op = self.bit_and ^ self.bit_or ^ self.bit_xor ^ self.bit_not ^ self.l_shift_ ^ self.r_shift_ ^ self.circ_l_shift_ ^ self.circ_r_shift_
# Grammar
self.stmt = Forward()
self.for_loop = Forward()
self.cast = Forward()
self.seq_val = Forward()
self.int_value = self.b_2_num ^ self.b_16_num ^ self.b_10_num
self.expr = Forward()
self.function_call = Forward()
self.index_select = Forward()
self.seq_ = Forward()
self.operand = Forward()
self.seq_range = Forward()
# #######Operands
self.sbox_call = Group((self.ID ^ self.seq_val) + ~White() + Literal(".") + ~White() + self.sbox_ + ~White() +
self.l_bracket + (self.ID ^ self.int_value) + self.r_bracket)
self.operand = self.index_select | self.seq_val | self.function_call | self.ID | self.int_value | self.cast | self.bit_val
self.seq_val.setParseAction(lambda t: ['Seq_val'] + [t.asList()])
self.index_select.setParseAction(lambda t: ['index_select'] + [t.asList()])
self.function_call.setParseAction(lambda t: ['function_call'] + [t.asList()])
self.ID.setParseAction(lambda t: ['ID'] + [t.asList()])
self.int_value.setParseAction(lambda t: ['Int_val'] + [t.asList()])
self.cast.setParseAction(lambda t: ['cast'] + [t.asList()])
self.bit_val.setParseAction(lambda t: ['Bit_val'] + [t.asList()])
self.seq_range.setParseAction(lambda t: ['seq_range'] + [t.asList()])
# #######Expressions
self.expr = Group(infixNotation(Group(self.operand), [(self.bitwise_op, 2, opAssoc.LEFT, self.nest_operand_pairs),
(self.comparison_op, 2, opAssoc.LEFT, self.nest_operand_pairs),
(self.log_op, 2, opAssoc.LEFT, self.nest_operand_pairs),
(self.arith_op, 2, opAssoc.LEFT, self.nest_operand_pairs)]))
# self.expr.setParseAction(self.expr_p)
self.int_size = Combine(Optional(Literal("@")) + self.int_)("decl") + ~White() + Suppress(self.l_bracket) + self.expr + Suppress(self.r_bracket)
self.sbox_size = self.sbox_ + ~White() + Suppress(self.l_bracket) + self.expr + Suppress(self.r_bracket)
self.seq_range << self.expr + Suppress(Literal(":")) + self.expr
self.seq_val << Suppress(self.l_sq_b) + Optional(Group(delimitedList(self.expr))) + Suppress(self.r_sq_b)
self.seq_ << (self.int_size | self.bit_ | self.sbox_size)("type") +\
Group(OneOrMore(~White() + Suppress(self.l_sq_b) + self.expr + Suppress(self.r_sq_b)))("seq_size")
self.function_call << self.ID("function_name") + ~White() + Suppress(self.l_bracket) +\
Optional(Group(delimitedList(self.expr)))("param_list") + Suppress(self.r_bracket)
self.cast << Suppress(self.l_bracket) + Group((self.seq_ | self.int_size | self.bit_)) +\
Suppress(self.r_bracket) + (self.expr)("target")
self.index_select << (self.ID("ID") ^ (Suppress(self.l_bracket) + self.cast + Suppress(self.r_bracket))("cast")) + ~White() +\
Group(OneOrMore(Suppress(self.l_sq_b) + Group(delimitedList(self.expr ^ Group(Group(self.seq_range))))("index") + Suppress(self.r_sq_b)))
# ####### Declarations
self.id_set = Group((Group(self.index_select) | self.ID_) + self.eq_set + self.expr)
self.id_set.setParseAction(self.AST.id_set)
self.int_decl = Group(self.int_size + delimitedList(Group((self.ID_("ID") + self.eq_set + self.expr("set_value")) |
self.ID_("ID")))("value")) # NOQA
self.int_decl.setParseAction(self.AST.int_decl)
self.bit_decl = Group(self.bit_("decl") + delimitedList(Group(self.ID_("ID")) ^
Group(self.ID_("ID") + self.eq_set + self.expr("set_value")))("value"))
self.bit_decl.setParseAction(self.AST.bit_decl)
self.seq_decl = Group(self.seq_("decl") + Group(self.ID)("ID") + Optional(self.eq_set + Group(self.expr))("value"))
self.seq_decl.setParseAction(self.AST.seq_decl)
self.decl = self.bit_decl ^ self.int_decl ^ self.seq_decl
# ###### Statements
self.return_stmt = Group(self.return_ + self.expr)
self.return_stmt.setParseAction(self.AST.return_stmt)
self.function_start = Literal("{")
self.function_start.setParseAction(self.AST.function_start)
self.function_end = Literal("}")
self.function_decl = Group((Group(self.seq_) | Group(self.int_size) | Group(self.bit_) | Group(self.void_))("return_type") + Group(self.ID)("func_ID") +
Suppress(self.l_bracket) + Group(Optional(delimitedList(Group((self.seq_ | self.int_size | self.bit_) + Group(self.ID)))))("func_param") + # NOQA
Suppress(self.r_bracket) + Suppress(self.function_start) + Group(self.stmt)("body") + Suppress(self.r_brace))
self.function_decl.setParseAction(self.AST.function_decl)
self.for_init = Literal('(')
self.for_init.setParseAction(self.AST.begin_for)
self.for_terminator = Literal(';')
self.for_terminator.setParseAction(self.AST.for_terminator)
self.for_increment = Literal(';')
self.for_increment.setParseAction(self.AST.for_increment)
self.terminator_expr = Group(infixNotation(Group(self.operand), [(self.log_op, 2, opAssoc.LEFT, self.nest_operand_pairs),
(self.bitwise_op, 2, opAssoc.LEFT, self.nest_operand_pairs),
(self.comparison_op, 2, opAssoc.LEFT, self.nest_operand_pairs),
(self.arith_op, 2, opAssoc.LEFT, self.nest_operand_pairs)]))
self.terminator_expr.setParseAction(self.AST.terminator_expr)
self.for_body = Literal('{')
self.for_body.setParseAction(self.AST.for_body)
self.end_for = Literal('}')
self.end_for.setParseAction(self.AST.end_for)
self.for_loop << Group(self.for_ + ~White() + Suppress(self.for_init) +
Optional(delimitedList(self.decl ^ self.id_set))("init") + Suppress(self.for_terminator) +
Optional(self.terminator_expr) + Suppress(self.for_increment) +
Optional(delimitedList(self.id_set))("increm") + Suppress(self.r_bracket) +
Suppress(self.for_body) + self.stmt("loop_body") + Suppress(self.end_for))
self.if_condition = Suppress(self.l_bracket) + self.expr + Suppress(self.r_bracket)
self.if_condition.setParseAction(self.AST.if_cond)
self.if_.setParseAction(self.AST.begin_if)
self.if_body_st = Literal('{')
self.if_body_st.setParseAction(self.AST.if_body_st)
self.if_body_end = Literal('}')
self.if_body_end.setParseAction(self.AST.if_body_end)
self.if_stmt = Group(self.if_ + self.if_condition("if_cond") + Suppress(self.if_body_st) + Group(self.stmt).setResultsName("body") + Suppress(self.if_body_end))
self.single_expr = self.expr + Suppress(self.term_st)
self.single_expr.setParseAction(self.AST.stand_alone_expr)
self.stmt << ZeroOrMore(self.decl + Suppress(self.term_st)
^ self.function_decl
^ self.id_set + Suppress(self.term_st)
^ self.single_expr
^ self.for_loop
^ self.if_stmt
^ self.return_stmt + Suppress(self.term_st)
^ self.sbox_call + Suppress(self.term_st))
self.grammar_test = self.stmt + StringEnd() # Allows single statements to be parsed
self.grammar = ZeroOrMore(self.function_decl
^ self.seq_decl + Suppress(self.term_st)) + StringEnd()
proto.setParseAction(proto_fn)
call = Forward()
literal = (integer | string | boolean | proto)
operand = (literal | call | array_ref | field)
multop = Word('*')
divop = Word('/')
plusop = Word('+')
minusop = Word('-')
signop = oneOf('- +')
operation = infixNotation( operand, [
(signop, 2, opAssoc.LEFT, binop_fn),
(multop, 2, opAssoc.LEFT, binop_fn),
(divop, 2, opAssoc.LEFT, binop_fn),
(plusop, 2, opAssoc.LEFT, binop_fn),
(minusop, 2, opAssoc.LEFT, binop_fn),
])
comparisonop = oneOf('< > ==')
comparison_operation = infixNotation( operation, [
(comparisonop, 2, opAssoc.LEFT, comparison_operator_fn)
])("comparison_operation")
array = Forward()
expression = (comparison_operation | operation | operand | array)
expression.setParseAction(expression_fn)
array << Suppress('{') + Group(delimitedList(expression)) + Suppress('}')
Literal("(").setName("func_param").setDebugActions(*debug) +
Optional(selectStmt | Group(delimitedList(expr)))("params") +
")"
).addParseAction(to_json_call).setDebugActions(*debug) |
ident.copy().setName("variable").setDebugActions(*debug)
)
expr << Group(infixNotation(
compound,
[
(
o,
3 if isinstance(o, tuple) else 2,
opAssoc.LEFT,
to_json_operator
)
for o in KNOWN_OPS
]+[
(
COLLATENOCASE,
1,
opAssoc.LEFT,
to_json_operator
)
]
).setName("expression").setDebugActions(*debug))
# SQL STATEMENT
selectColumn = Group(
Group(expr).setName("expression1")("value").setDebugActions(*debug) + Optional(Optional(AS) + ident.copy().setName("column_name1")("name").setDebugActions(*debug)) |
Literal('*')("value").setDebugActions(*debug)
).setName("column").addParseAction(to_select_call)
boost = CARAT + number("boost")
string_expr = Group(string + proximity_modifier) | string
word_expr = Group(valid_word + fuzzy_modifier) | valid_word
term << (
Optional(field_name("field") + COLON)
+ (word_expr | string_expr | range_search | Group(LPAR + expression + RPAR))
+ Optional(boost)
)
term.setParseAction(lambda t: [t] if "field" in t or "boost" in t else None)
expression << infixNotation(
term,
[
(required_modifier | prohibit_modifier, 1, opAssoc.RIGHT),
((not_ | "!").setParseAction(lambda: "NOT"), 1, opAssoc.RIGHT),
((and_ | "&&").setParseAction(lambda: "AND"), 2, opAssoc.LEFT),
(Optional(or_ | "||").setParseAction(lambda: "OR"), 2, opAssoc.LEFT),
],
)
# test strings taken from grammar description doc, and TestQueryParser.java
tests = r"""
a and b
a and not b
a and !b
a && !b
a&&!b
name:a
name:a and not title:b
(a^100 c d f) and !z
COMMA + Group(expr)("if_false") + RPAR)
statFunc = lambda name : Group(CaselessKeyword(name) + Group(LPAR + delimitedList(expr) + RPAR))
sumFunc = statFunc("sum")
minFunc = statFunc("min")
maxFunc = statFunc("max")
aveFunc = statFunc("ave")
funcCall = ifFunc | sumFunc | minFunc | maxFunc | aveFunc
multOp = oneOf("* /")
addOp = oneOf("+ -")
numericLiteral = pyparsing_common.number
operand = numericLiteral | funcCall | cellRange | cellRef
arithExpr = infixNotation(operand,
[
(multOp, 2, opAssoc.LEFT),
(addOp, 2, opAssoc.LEFT),
])
textOperand = dblQuotedString | cellRef
textExpr = infixNotation(textOperand,
[
('&', 2, opAssoc.LEFT),
])
expr << (arithExpr | textExpr)
(EQ + expr).runTests("""\
=3*A7+5
=3*Sheet1!$A$7+5
def __init__(self, cacheKey, aliases, stringFunctions, packageGenerator):
self.__cacheKey = cacheKey
self.__aliases = aliases
self.__stringFunctions = stringFunctions
self.__generator = packageGenerator
self.__root = None
self.__graph = None
# create parsing grammer
locationPath = pyparsing.Forward()
relativeLocationPath = pyparsing.Forward()
axisName = \
pyparsing.Keyword("descendant-or-self") \
| pyparsing.Keyword("child") \
| pyparsing.Keyword("descendant") \
| pyparsing.Keyword("direct-descendant-or-self") \
| pyparsing.Keyword("direct-child") \
| pyparsing.Keyword("direct-descendant") \
| pyparsing.Keyword("self")
nodeTest = pyparsing.Word(pyparsing.alphanums + "_.:+-*")
axisSpecifier = axisName + '@'
abbreviatedStep = pyparsing.Keyword('.')
sQStringLiteral = pyparsing.QuotedString("'")
sQStringLiteral.setParseAction(
lambda s, loc, toks: StringLiteral(s, loc, toks, False,
self.__stringFunctions,
self.__getGraphIter))
dQStringLiteral = pyparsing.QuotedString('"', '\\')
dQStringLiteral.setParseAction(
lambda s, loc, toks: StringLiteral(s, loc, toks, True,
self.__stringFunctions,
self.__getGraphIter))
stringLiteral = sQStringLiteral | dQStringLiteral
functionCall = pyparsing.Forward()
functionArg = stringLiteral | functionCall
functionCall << pyparsing.Word(pyparsing.alphas, pyparsing.alphanums+'-') + \
pyparsing.Suppress('(') + \
pyparsing.Optional(functionArg +
pyparsing.ZeroOrMore(pyparsing.Suppress(',') + functionArg)) + \
pyparsing.Suppress(')')
functionCall.setParseAction(
lambda s, loc, toks: FunctionCall(s, loc, toks, self.__stringFunctions,
self.__getGraphIter))
predExpr = pyparsing.infixNotation(
locationPath ^ stringLiteral ^ functionCall,
[
('!', 1, pyparsing.opAssoc.RIGHT, lambda s, loc, toks: NotOperator(s, loc, toks, self.__getGraphRoot)),
('<', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: BinaryStrOperator(s, loc, toks, self.__getGraphIter)),
('<=', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: BinaryStrOperator(s, loc, toks, self.__getGraphIter)),
('>', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: BinaryStrOperator(s, loc, toks, self.__getGraphIter)),
('>=', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: BinaryStrOperator(s, loc, toks, self.__getGraphIter)),
('==', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: BinaryStrOperator(s, loc, toks, self.__getGraphIter)),
('!=', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: BinaryStrOperator(s, loc, toks, self.__getGraphIter)),
('&&', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: BinaryBoolOperator(s, loc, toks)),
('||', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: BinaryBoolOperator(s, loc, toks))
])
predicate = '[' + predExpr + ']'
step = abbreviatedStep | (pyparsing.Optional(axisSpecifier) +
nodeTest + pyparsing.Optional(predicate))
step.setParseAction(lambda s, loc, toks: LocationStep(s, loc, toks))
abbreviatedRelativeLocationPath = step + '//' + relativeLocationPath
relativeLocationPath << (
abbreviatedRelativeLocationPath |
(step + '/' + relativeLocationPath) |
step)
abbreviatedAbsoluteLocationPath = '//' + relativeLocationPath
absoluteLocationPath = abbreviatedAbsoluteLocationPath | \
('/' + relativeLocationPath)
locationPath << (absoluteLocationPath | relativeLocationPath)
locationPath.setParseAction(
lambda s, loc, toks: LocationPath(s, loc, toks, self.__getGraphRoot))
self.__pathGrammer = locationPath
value_comparison_operator_p = pp.oneOf("== >= <= != < >")
value_comparison_p = attribute_identifier_p + value_comparison_operator_p + number_literal_p
value_comparison_p.setParseAction(lambda tokens: BinaryOperator(tokens[1], tokens[0::2]))
membership_comparison_operator_p = pp.Optional("not") + pp.Literal("in")
membership_comparison_operator_p.setParseAction(lambda tokens: " ".join(tokens))
membership_comparison_p = attribute_identifier_p + membership_comparison_operator_p + (number_list_literal_p | string_list_literal_p)
membership_comparison_p.setParseAction(lambda tokens: BinaryOperator(tokens[1], tokens[0::2]))
comparison_p = value_comparison_p | membership_comparison_p
logical_expression_p = pp.infixNotation(comparison_p,
[
(pp.Literal("and"), 2, pp.opAssoc.LEFT, lambda tokens: BinaryOperator(tokens[0][1], tokens[0][0::2])),
(pp.Literal("or"), 2, pp.opAssoc.LEFT, lambda tokens: BinaryOperator(tokens[0][1], tokens[0][0::2])),
])
function_call_p = pp.Forward()
function_argument_p = function_call_p | attribute_identifier_p | string_literal_p | float_literal_p | integer_literal_p
function_call_p << pp.Word(pp.alphas, pp.alphanums) + pp.Suppress("(") + pp.Optional(pp.delimitedList(function_argument_p, delim=",")) + pp.Suppress(")")
function_call_p.setParseAction(FunctionCall)
attribute_expression_p = logical_expression_p | function_call_p | attribute_identifier_p | slice_literal_p
# Define the arrays section of a hyperchunk.
arrays_expression_p = slices_literal_p
return not v
def __str__(self):
return "~" + str(self.arg)
__repr__ = __str__
__nonzero__ = __bool__
TRUE = Keyword("True")
FALSE = Keyword("False")
boolOperand = TRUE | FALSE | Word(alphas,max=1)
boolOperand.setParseAction(BoolOperand)
# define expression, based on expression operand and
# list of operations in precedence order
boolExpr = infixNotation( boolOperand,
[
("not", 1, opAssoc.RIGHT, BoolNot),
("and", 2, opAssoc.LEFT, BoolAnd),
("or", 2, opAssoc.LEFT, BoolOr),
])
if __name__ == "__main__":
p = True
q = False
r = True
tests = [("p", True),
("q", False),
("p and q", False),
("p and not q", True),
("not not p", True),
("not(p and q)", True),
("q or not p and r", False),
"""
the RHS of the rule is a list of actions, or no action "()"
"""
list_of_actions = Group("()" | delimitedList(predicate, delim=","))
"""
rules for parsing expressions
i.e. something that is evaluated to a value, like "2 + 1" or "1 / 6"
"""
simple_expression = Group(varName("variable")) | Group(float_num("float")) | Group(integer("integer"))
expression = infixNotation(simple_expression,
[
(oneOf("* /"), 2, opAssoc.LEFT),
(oneOf("+ -"), 2, opAssoc.LEFT)
])
"""
binary comparison symbols
"""
binary_comparison = Literal(">=") | Literal(">") | Literal("==") | Literal("<=") | Literal("<")
"""
rule for parsing binary conditions, like
"2 > 1" or "X <= 5 + Y"
"""
binary_condition = Group(expression("arg1") + binary_comparison("operator") + expression("arg2"))
return "~" + str(self.arg)
__repr__ = __str__
__nonzero__ = __bool__
T = Keyword("True") | Keyword("true") | Keyword("T") | Keyword("1")
F = Keyword("False") | Keyword("false")| Keyword("F")| Keyword("0")
operand = T | F | Word(alphas, max=1)
operand.setParseAction(Operand)
# define expression, based on expression operand and
# list of operations in precedence order
expression = infixNotation( operand,
[
("not", 1, opAssoc.RIGHT, Not),
("and", 2, opAssoc.LEFT, And),
("or", 2, opAssoc.LEFT, Or),
])
def test():
symbols['p'] = True
symbols['q'] = False
symbols['r'] = True
tests = [("p", True),
("q", False),
("p and q", False),
("p and not q", True),
("not not p", True),
("not(p and q)", True),
("q or not p and r", False),
def doArchiveClean(argv):
varReference = pyparsing.Word(pyparsing.alphanums+'.')
varReference.setParseAction(lambda s, loc, toks: VarReference(s, loc, toks))
stringLiteral = pyparsing.QuotedString('"', '\\')
stringLiteral.setParseAction(lambda s, loc, toks: StringLiteral(s, loc, toks))
expr = pyparsing.infixNotation(
stringLiteral | varReference,
[
('!', 1, pyparsing.opAssoc.RIGHT, lambda s, loc, toks: NotPredicate(s, loc, toks)),
('<', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: ComparePredicate(s, loc, toks)),
('<=', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: ComparePredicate(s, loc, toks)),
('>', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: ComparePredicate(s, loc, toks)),
('>=', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: ComparePredicate(s, loc, toks)),
('==', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: ComparePredicate(s, loc, toks)),
('!=', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: ComparePredicate(s, loc, toks)),
('&&', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: AndPredicate(s, loc, toks)),
('||', 2, pyparsing.opAssoc.LEFT, lambda s, loc, toks: OrPredicate(s, loc, toks))
])
parser = argparse.ArgumentParser(prog="bob archive clean")
parser.add_argument('expression', help="Expression of artifacts that shall be kept")
parser.add_argument('--dry-run', default=False, action='store_true',
help="Don't delete, just print what would be deleted")
parser.add_argument('-n', dest='noscan', action='store_true',
help="Skip scanning for new artifacts")
parser.add_argument("-v", "--verbose", action='store_true',
help="Verbose operation")
args = parser.parse_args(argv)
try:
retainExpr = expr.parseString(args.expression, True)[0]
except pyparsing.ParseBaseException as e:
raise BobError("Invalid retention expression: " + str(e))
scanner = ArchiveScanner()
retained = set()
with scanner:
if not args.noscan:
scanner.scan(args.verbose)
for bid,taint in scanner.getBuildIds():
if bid in retained: continue
if retainExpr.evalBool(scanner.getVars(bid, taint)):
retained.add(bid)
todo = set(scanner.getReferencedBuildIds(bid))
while todo:
n = todo.pop()
if n in retained: continue
retained.add(n)
todo.update(scanner.getReferencedBuildIds(n))
for bid,taint in scanner.getBuildIds():
if bid in retained: continue
victim = asHexStr(bid)
victim = os.path.join(victim[0:2], victim[2:4],
victim[4:] + "-1" + ("-"+asHexStr(taint) if taint else "") + ".tgz")
if args.dry_run:
print(victim)
else:
try:
if args.verbose:
print("rm", victim)
os.unlink(victim)
except FileNotFoundError:
pass
except OSError as e:
raise BobError("Cannot remove {}: {}".format(victim, str(e)))
scanner.remove(bid, taint)
mongo_op.setParseAction(
lambda t: t[0].replace('>=', '$gte').replace('>', '$gt').replace('<=', '$lte').replace('<', '$lt')
)
one_sided_range = Group(mongo_op('op') + valid_word('bound'))('onesidedrange')
term = (_range | one_sided_range | regex | wildcard | phrase | single_term)
clause << (Optional(field_name + COLON, default='__default_field__')('field') +
(term('term') | Group(LPAR + query_expr + RPAR)('subquery')))
clause.addParseAction(SearchTerm)
query_expr << infixNotation(clause,
[
(required_modifier | prohibit_modifier, 1, opAssoc.RIGHT, SearchModifier),
((not_ | '!').setParseAction(lambda: 'NOT'), 1, opAssoc.RIGHT, SearchNot),
((and_ | '&&').setParseAction(lambda: 'AND'), 2, opAssoc.LEFT, SearchAnd),
(Optional(or_ | '||').setParseAction(lambda: 'OR'), 2, opAssoc.LEFT, SearchOr),
])
class QueryParser:
DEFAULT_FIELD = 'text'
DEFAULT_OPERATOR = '$regex'
def parse(self, query, default_field=None, default_operator=None):
self.default_field = default_field or QueryParser.DEFAULT_FIELD
self.default_operator = default_operator or QueryParser.DEFAULT_OPERATOR
return repr(query_expr.parseString(query)[0])\
binop = oneOf('= == != <> < <= > >= not like contains has ilike '
'icontains ihas is').setName('binop')
domain = Word(alphas, alphanums).setName('domain')
domain_values = Group(value_list.copy())
domain_expression = (domain + Literal('=') + Literal('*') + stringEnd) \
| (domain + binop + domain_values + stringEnd)
AND_ = CaselessLiteral("and")
OR_ = CaselessLiteral("or")
NOT_ = CaselessLiteral("not") | Literal('!')
identifier = Group(delimitedList(Word(alphas, alphanums+'_'), '.')).setParseAction(IdentifierAction)
ident_expression = Group(identifier + binop + value).setParseAction(IdentExpressionAction)
query_expression = infixNotation(
ident_expression,
[ (NOT_, 1, opAssoc.RIGHT, SearchNotAction),
(AND_, 2, opAssoc.LEFT, SearchAndAction),
(OR_, 2, opAssoc.LEFT, SearchOrAction) ] )
query = (domain + Keyword('where', caseless=True).suppress() +
Group(query_expression) + stringEnd)
statement = (query.setParseAction(QueryAction)('query')
| domain_expression.setParseAction(domain_expression_action)('domain')
| value_list.setParseAction(value_list_action)('values')
).setParseAction(StatementAction)('statement')
###############################################################################
## the following 'search' object exposes the interface that will be
## implemented by the code that eventually will be part of bauble
class Exclude(object):
def __init__(self, t):
super(Exclude, self).__init__()
self.matcher = t[0][1]
def matches(self, metadata):
return not self.matcher.matches(metadata)
matcher = Word(alphanums + '._,-=:')
matcher.setParseAction(Include)
boolExpr = infixNotation(matcher, [
("not", 1, opAssoc.RIGHT, Exclude),
("and", 2, opAssoc.LEFT, AndMatching),
("or", 2, opAssoc.LEFT, OrMatching),
])
class Matcher(object):
def __init__(self, pattern):
super(Matcher, self).__init__()
self._matcher = boolExpr.parseString(pattern)[0]
def __repr__(self):
return repr(self._matcher)
def matches(self, metadata):
return self._matcher.matches(metadata)
