def parse(lines: str, allowed_colors: str = '') -> Union[Rule, Color]:
lines = strip_comments(lines)
if allowed_colors == '':
allowed_colors = string.ascii_lowercase + string.ascii_uppercase
color = Word(allowed_colors, exact=1)
integer = Word(string.digits).setParseAction(lambda t: int(t[0]))
selector = Word(allowed_colors + '-', exact=1) * 9
selector.setParseAction(lambda s: Selector(''.join(s)))
num_operation = infixNotation((selector | integer), [
('*', 2, opAssoc.LEFT, parse_selector_operator),
('/', 2, opAssoc.LEFT, parse_selector_operator),
('+', 2, opAssoc.LEFT, parse_selector_operator),
('-', 2, opAssoc.LEFT, parse_selector_operator),
('%', 2, opAssoc.LEFT, parse_selector_operator),
])
operator = oneOf('>= <= != > < ==')
comparison_token = num_operation | selector | integer
comparison = (comparison_token + operator + comparison_token).\
setParseAction(lambda t: Comparison(*t))
bool_expr = infixNotation(comparison, [
('and', 2, opAssoc.LEFT, parse_bool_expr),
('or', 2, opAssoc.LEFT, parse_bool_expr),
])
rule = Forward()
condition = (Word('if') + bool_expr + Word(':') + rule + Word('else:') +
rule)
rule << (condition | color).setParseAction(parse_rule)
return rule.parseString(lines, parseAll=True)[0]
def ImportFromString(self, text):
#alphabet + number + underbar
nameG = Word( srange("[a-zA-Z_]"), srange("[a-zA-Z0-9_]") )
stringDoubleG = Literal('"') + SkipTo('"') + Literal('"')
stringSingleG = Literal("'") + SkipTo("'") + Literal("'")
stringG = stringDoubleG | stringSingleG
#comment
comment = (Literal('/*').suppress() + SkipTo('*/').suppress() + Literal('*/').suppress())
digraphNameG = Word( srange("[a-zA-Z_]"), srange("[a-zA-Z0-9_]") )
digraphNameG.setParseAction( self.DigraphNameAction )
nodeNameG = Word( srange("[a-zA-Z_]"), srange("[a-zA-Z0-9_]") )
nodeNameG.setParseAction( self.NodeNameAction )
startG = Literal('digraph').suppress() + digraphNameG.suppress() + Literal('{').suppress()
endG = Literal('}').suppress()
attrBaseG = nameG+Literal('=').suppress()+(nameG ^ stringG)
attrBaseG.setParseAction( self.AttributeAction )
attrRepeatG = attrBaseG + ZeroOrMore( Literal(',').suppress() + attrBaseG )
attrG = ZeroOrMore( Literal('[').suppress() + ZeroOrMore( attrRepeatG ) + Literal(']').suppress() )
nodeG = nodeNameG + attrG
edgeNameG = nameG + Literal('->').suppress() + nameG
edgeNameG.setParseAction( self.EdgeNameAction )
edgeG = edgeNameG + attrG
sizeG = Literal('size') + Literal('=') + (quotedString | dblQuotedString)
baseG = (sizeG | (nodeG ^ edgeG)) + Literal(';').suppress()
baseG = baseG | comment
baseG = ZeroOrMore(baseG)
grammerG = startG + baseG + endG
grammerG.parseString(text)
def parse(self, query):
"""Parses a query string."""
# Parse instructions
quoted_string = QuotedString(quoteChar='"', escChar='\\', unquoteResults=True)
field_name = Word(alphas, alphanums + '_')
subexpression = Forward()
boolean_expression = Forward()
binary_operator = Literal('=') | Literal('<=') | Literal('<') | Literal('>=') | Literal('>')
boolean_operator = CaselessKeyword('AND') | CaselessKeyword('OR')
boolean_not = CaselessKeyword('NOT')
boolean_value = CaselessKeyword("true") ^ CaselessKeyword("false")
integer = Word(nums)
rvalue = quoted_string ^ boolean_value ^ integer
field_to_value = field_name + binary_operator + rvalue
expression = Optional(boolean_not) + ((subexpression + ZeroOrMore(boolean_expression)) | (field_to_value + ZeroOrMore(boolean_expression)))
boolean_expression << boolean_operator + expression
left_bracket = Literal('(')
right_bracket = Literal(')')
subexpression << (left_bracket + expression + right_bracket)
search_query = expression
# Parse actions for emitting special cases
field_to_value.setParseAction(do_field_to_value)
boolean_operator.setParseAction(do_boolean_operator)
boolean_not.setParseAction(do_boolean_operator)
boolean_value.setParseAction(do_boolean_value)
integer.setParseAction(do_integer)
left_bracket.setParseAction(do_bracket)
right_bracket.setParseAction(do_bracket)
self.tokens = search_query.parseString(query)
def _define_valued_characters_section(self, heading, characters,
parse_action, character_type):
"""Returns a parser object for a section specifying characters
and their valued features.
:param heading: section heading
:type heading: `str`
:param characters: valid characters
:type characters: `list` of `str`
:param parse_action: parse action for a character
:type parse_action: `function`
:param character_type: type of characters being described
:type character_type: `str`
"""
heading = Literal('[{}]'.format(heading))
character = Word(''.join(characters), exact=1).setResultsName(
'character')
character.setParseAction(self._handle_character)
feature = Word(alphas).setResultsName('feature')
feature.setParseAction(parse_action)
value = Literal('+') ^ Literal('-') ^ Literal('\N{MINUS SIGN}')
value.setParseAction(self._handle_feature_value)
feature_value = Group(value + feature)
feature_values = Group(delimitedList(feature_value))
character_definition = Dict(Group(character + Suppress(':') +
feature_values))
character_definitions = Group(OneOrMore(character_definition)).setResultsName(character_type)
section = Suppress(heading) + character_definitions
return section
def _construct_grammar():
logical_operator = get_logical_operator()
logical_expression = get_logical_expression()
facets_expression = get_facet_expression()
highlight_expression = get_highlight_expression()
sort_expression = get_sort_expression()
aggs_expression = get_aggregations_expression()
nested_expression = get_nested_expression()
# The below line describes how the type expression should be.
type_expression = Word('type')\
+ Word(':').suppress()\
+ Word(srange("[a-zA-Z0-9_]"))\
+ Optional(CaselessLiteral('AND')).suppress()
type_expression.setParseAction(parse_type_expression)
base_expression = Optional(highlight_expression)\
+ Optional(sort_expression)\
+ Optional(type_expression)\
+ ZeroOrMore(
(facets_expression
| aggs_expression
| nested_expression
| logical_expression)
+ Optional(logical_operator)
).setParseAction(parse_one_or_more_logical_expressions)
base_expression.setParseAction(parse_type_logical_facets_expression)
return base_expression
def _grammar():
from pyparsing import alphas, alphanums, nums
from pyparsing import oneOf, Suppress, Optional, Group, ZeroOrMore, NotAny
from pyparsing import Forward, operatorPrecedence, opAssoc, Word, White
from pyparsing import delimitedList, Combine, Literal, OneOrMore
expression = Forward()
LPAR, RPAR, DOT, LBRAC, RBRAC = map(Suppress, "().{}")
nw = NotAny(White())
identifier = Word(alphas + "_", alphanums + "_")
integer = Word(nums)
integer.setParseAction(IntegerNode)
fractional = Combine(Word('+' + '-' + nums, nums) + '.' + Word(nums))
fractional.setParseAction(FloatNode)
literal = fractional | integer
arglist = delimitedList(expression)
seqrange = LBRAC + expression + Suppress('..') + expression + RBRAC
seqrange.setParseAction(lambda t: SequenceNode(start=t[0], stop=t[1]))
seqexplicit = LBRAC + Optional(arglist) + RBRAC
seqexplicit.setParseAction(lambda t: SequenceNode(lst=t))
sequence = seqrange | seqexplicit
rollmod = nw + Group(oneOf("d k r e x") + Optional(integer))
numdice = Optional(integer, default=1)
roll = numdice + nw + Suppress("d") + nw + (integer | sequence)
roll += Group(ZeroOrMore(rollmod))
roll.setParseAction(DieRollNode)
call = LPAR + Group(Optional(arglist)) + RPAR
function = identifier + call
function.setParseAction(FunctionNode)
seqexpr = ((roll | sequence | function) +
Group(OneOrMore(DOT + identifier + call)))
seqexpr.setParseAction(SeqMethodNode)
variable = Word(alphas + "_", alphanums + "_ ")
variable.setParseAction(VariableNode)
atom = seqexpr | roll | literal | sequence | function | variable
expoop = Literal('^')
signop = oneOf("+ -")
multop = oneOf("* /")
plusop = oneOf("+ -")
# noinspection PyUnresolvedReferences
expression << operatorPrecedence(atom, [
(expoop, 2, opAssoc.LEFT, BinaryOpNode),
(signop, 1, opAssoc.RIGHT, UnaryOpNode),
(multop, 2, opAssoc.LEFT, BinaryOpNode),
(plusop, 2, opAssoc.LEFT, BinaryOpNode),
])
return expression
def __createGram(self):
if self.notNeedSpace:
lNot = Keyword(self.operators['not'])
else:
lNot = Literal(self.operators['not'])
lAnd = Literal(self.operators['and'])
lOr = Literal(self.operators['or'])
lImp = Literal(self.operators['impL'])
lEqu = Literal(self.operators['equ'])
lTrue = Keyword(self.constants['true'])
lFalse = Keyword(self.constants['false'])
lVar = Word(alphas, alphanums+'_')
lVar.setParseAction(self.ffactory.createLogicVariable)
lTrue.setParseAction(self.ffactory.createLogicTruth)
lFalse.setParseAction(self.ffactory.createLogicFalse)
factor = lTrue | lFalse | lVar
expression = myparsing.operatorPrecedence(factor,
[
(lNot, 1, opAssoc.RIGHT, self.ffactory.createNotOperation),
(lAnd, 2, opAssoc.LEFT, self.ffactory.createAndOperation),
(lOr, 2, opAssoc.LEFT, self.ffactory.createOrOperation),
(lImp, 2, opAssoc.LEFT, self.ffactory.createImpicationOperation),
(lEqu, 2, opAssoc.LEFT, self.ffactory.createEquvalenceOperation)
],
[('(', ')'), ('[', ']'), ('{', '}')])
self.final = expression + StringEnd()
def init_parser(self):
INTEGER = Word(nums)
INTEGER.setParseAction(lambda x: int(x[0]))
header = INTEGER("species_count") + INTEGER("sequence_length") +\
Suppress(restOfLine)
header.setParseAction(self.set_header)
sequence_name = Word(
alphas + nums + "!#$%&\'*+-./;<=>?@[\\]^_`{|}~",
max=100)
# Take a copy and disallow line breaks in the bases
bases = self.BASES.copy()
bases.setWhitespaceChars(" \t")
seq_start = sequence_name("species") + bases(
"sequence") + Suppress(LineEnd())
seq_start.setParseAction(self.set_seq_start)
seq_start_block = OneOrMore(seq_start)
seq_start_block.setParseAction(self.set_start_block)
seq_continue = bases("sequence") + Suppress(LineEnd())
seq_continue.setParseAction(self.set_seq_continue)
seq_continue_block = Suppress(LineEnd()) + OneOrMore(seq_continue)
seq_continue_block.setParseAction(self.set_continue_block)
return header + seq_start_block + ZeroOrMore(seq_continue_block)
def __init__(self, name=None, status_dic=None):
self._name = name
self._status_dic = dict(status_dic) if isinstance(status_dic,
dict) else {}
lpar = Literal('(').suppress()
rpar = Literal(')').suppress()
op = oneOf(' '.join([k for k in self._operator_map.keys()]))
op.setParseAction(self._get_operator_func)
# digits
d = Word(nums + '.')
d.setParseAction(lambda l: [float(i) for i in l])
# something like 3m, 50s, see
# https://humanfriendly.readthedocs.io/en/latest/#humanfriendly.parse_timespan
td = Regex(r'[\d]+[a-zA-Z]+')
td.setParseAction(self._parse_timedeltas)
# something like cpu_usage_rate, vnbe0.inbytes
metric = Word(alphas, alphanums + '._')
# order matters
any_v = td | metric | d
self.expr = Forward()
atom = any_v | Group(lpar + self.expr + rpar)
self.expr << atom + ZeroOrMore(op + self.expr)
def __init__(self, expression):
# define the parser
integer = Word(nums)
real = Combine(Word(nums) + "." + Word(nums))
# nums have been added to allow for aggregation variables such as `sum_250_field`
variable = Word(alphas + ":" + "_" + nums)
boolean = oneOf('true false', caseless=True)
num = integer | real
keyword = oneOf('time day period')
boolean.setParseAction(EvalBool)
variable.setParseAction(EvalVar)
num.setParseAction(EvalNum)
keyword.setParseAction(EvalKeyword)
bool_op = oneOf('&& ||')
sign_op = oneOf('+ -')
comparison_op = oneOf("< <= > >= != ==")
atom = boolean | keyword | num | variable | bool_op | sign_op | comparison_op
self.expr = operatorPrecedence(
atom, [(sign_op, 1, opAssoc.RIGHT, EvalSignOp),
(comparison_op, 2, opAssoc.LEFT, EvalComparisonOp),
(bool_op, 2, opAssoc.LEFT, EvalLogical)])
self.parsed = self.expr.parseString(expression)[0]
def __init__(self):
# speed up infixNotation considerably at the price of some cache memory
ParserElement.enablePackrat()
boolean = Keyword('True') | Keyword('False')
none = Keyword('None')
integer = Word(nums)
real = Combine(Word(nums) + "." + Word(nums))
string = (QuotedString('"', escChar='\\')
| QuotedString("'", escChar='\\'))
regex = QuotedString('/', escChar='\\')
identifier = Word(alphas, alphanums + '_')
dereference = infixNotation(identifier, [
(Literal('.'), 2, opAssoc.LEFT, EvalArith),
])
result = (Keyword('bad') | Keyword('fail') | Keyword('good')
| Keyword('ignore') | Keyword('unknown'))
rval = boolean | none | real | integer | string | regex | result | dereference
rvallist = Group(
Suppress('[') + Optional(delimitedList(rval)) + Suppress(']'))
rvalset = Group(
Suppress('{') + Optional(delimitedList(rval)) + Suppress('}'))
operand = rval | rvallist | rvalset
# parse actions replace the parsed tokens with an instantiated object
# which we can later call into for evaluation of its content
boolean.setParseAction(EvalBoolean)
none.setParseAction(EvalNone)
integer.setParseAction(EvalInteger)
real.setParseAction(EvalReal)
string.setParseAction(EvalString)
regex.setParseAction(EvalRegex)
identifier.setParseAction(EvalIdentifier)
result.setParseAction(EvalResult)
rvallist.setParseAction(EvalList)
rvalset.setParseAction(EvalSet)
identity_test = Keyword('is') + ~Keyword('not') | Combine(
Keyword('is') + Keyword('not'), adjacent=False, joinString=' ')
membership_test = Keyword('in') | Combine(
Keyword('not') + Keyword('in'), adjacent=False, joinString=' ')
comparison_op = oneOf('< <= > >= != == isdisjoint')
comparison = identity_test | membership_test | comparison_op
self.parser = infixNotation(operand, [
(Literal('**'), 2, opAssoc.LEFT, EvalPower),
(oneOf('+ - ~'), 1, opAssoc.RIGHT, EvalModifier),
(oneOf('* / // %'), 2, opAssoc.LEFT, EvalArith),
(oneOf('+ -'), 2, opAssoc.LEFT, EvalArith),
(oneOf('<< >>'), 2, opAssoc.LEFT, EvalArith),
(Literal('&'), 2, opAssoc.LEFT, EvalArith),
(Literal('^'), 2, opAssoc.LEFT, EvalArith),
(Literal('|'), 2, opAssoc.LEFT, EvalArith),
(comparison, 2, opAssoc.LEFT, EvalLogic),
(Keyword('not'), 1, opAssoc.RIGHT, EvalModifier),
(Keyword('and'), 2, opAssoc.LEFT, EvalLogic),
(Keyword('or'), 2, opAssoc.LEFT, EvalLogic),
(Keyword('->'), 2, opAssoc.LEFT, EvalArith),
])
def _define_grammar():
"""
Creates and returns a copy of the selector grammar.
Wrapped in a function to avoid polluting the module namespace.
"""
expr = Forward()
label_name = Word(LABEL_CHARS)
label_name.setParseAction(LabelNode)
string_literal = QuotedString('"') | QuotedString("'")
string_literal.setParseAction(LiteralNode)
set_literal = (Suppress("{") +
delimitedList(QuotedString('"') | QuotedString("'"), ",") +
Suppress("}"))
set_literal.setParseAction(SetLiteralNode)
eq_comparison = label_name + Suppress("==") + string_literal
eq_comparison.setParseAction(LabelToLiteralEqualityNode)
not_eq_comparison = label_name + Suppress("!=") + string_literal
not_eq_comparison.setParseAction(InequalityNode)
in_comparison = label_name + Suppress(Keyword("in")) + set_literal
in_comparison.setParseAction(LabelInSetLiteralNode)
not_in = Suppress(Keyword("not") + Keyword("in"))
not_in_comparison = label_name + not_in + set_literal
not_in_comparison.setParseAction(NotInNode)
has_check = (Suppress("has(") + Word(LABEL_CHARS) + Suppress(")"))
has_check.setParseAction(HasNode)
# For completeness, we allow an all() to occur in an expression like
# "! all()". Note: we special-case the trivial selectors "" and
# "all()" below for efficiency.
all_op = (Suppress("all()"))
all_op.setParseAction(AllNode)
comparison = (eq_comparison | not_eq_comparison | in_comparison
| not_in_comparison | has_check | all_op)
paren_expr = (Suppress("(") + expr + Suppress(")"))
value = ZeroOrMore("!") + (comparison | paren_expr)
value.setParseAction(simplify_negation_node)
and_expr = value + ZeroOrMore(Suppress("&&") + value)
and_expr.setParseAction(simplify_and_node)
or_expr = and_expr + ZeroOrMore(Suppress("||") + and_expr)
or_expr.setParseAction(simplify_or_node)
expr << or_expr
grammar = expr + StringEnd()
return grammar
def taking_action():
prefix = 'A Fistful of' + White()
fist_contents = Word(alphas)
fist_contents.setParseAction(uppercase_it)
title_parser = Combine(prefix + fist_contents)
for title in ('A Fistful of Dollars', 'A Fistful of Spaghetti',
'A Fistful of Doughnuts'):
print(title_parser.parseString(title))
def _define_grammar():
"""
Creates and returns a copy of the selector grammar.
Wrapped in a function to avoid polluting the module namespace.
"""
expr = Forward()
label_name = Word(LABEL_CHARS)
label_name.setParseAction(LabelNode)
string_literal = QuotedString('"') | QuotedString("'")
string_literal.setParseAction(LiteralNode)
set_literal = (Suppress("{") +
delimitedList(QuotedString('"') | QuotedString("'"), ",") +
Suppress("}"))
set_literal.setParseAction(SetLiteralNode)
eq_comparison = label_name + Suppress("==") + string_literal
eq_comparison.setParseAction(LabelToLiteralEqualityNode)
not_eq_comparison = label_name + Suppress("!=") + string_literal
not_eq_comparison.setParseAction(InequalityNode)
in_comparison = label_name + Suppress(Keyword("in")) + set_literal
in_comparison.setParseAction(LabelInSetLiteralNode)
not_in = Suppress(Keyword("not") + Keyword("in"))
not_in_comparison = label_name + not_in + set_literal
not_in_comparison.setParseAction(NotInNode)
has_check = (Suppress("has(") +
Word(LABEL_CHARS) +
Suppress(")"))
has_check.setParseAction(HasNode)
comparison = (eq_comparison |
not_eq_comparison |
in_comparison |
not_in_comparison |
has_check)
paren_expr = (Suppress("(") + expr + Suppress(")"))
value = comparison | paren_expr
and_expr = value + ZeroOrMore(Suppress("&&") + value)
and_expr.setParseAction(simplify_and_node)
or_expr = and_expr + ZeroOrMore(Suppress("||") + and_expr)
or_expr.setParseAction(simplify_or_node)
expr << or_expr
grammar = expr + StringEnd()
return grammar
def get_number():
from pyparsing import Word, nums, ParseException
def validate_and_convert_number(tokens):
try:
return float(tokens[0])
except ValueError:
raise ParseException("Invalid number (%s)" % tokens[0])
number = Word(nums + '-' + '+' + '.').setResultsName("value") # do not allow scientific notation
number.setParseAction(validate_and_convert_number)
return number
def get_highlight_expression():
field_expression = Word(srange("[a-zA-Z0-9_.*]"))
field_expression.setParseAction(parse_highlight_field_expression)
fields_expression = OneOrMore(
field_expression + Optional(',').suppress())
fields_expression.setParseAction(parse_highlight_expression)
highlight_expression = Word('highlight:').suppress() \
+ Word('[').suppress() \
+ fields_expression + Word(']').suppress()
return highlight_expression
def setup(self):
# some expressions that will be reused
units = []
for unit in time_units:
units.append(Keyword(unit))
units = get_match_first(units)
units = units.setResultsName("unit")
units.setParseAction(lambda s, l, tok: time_units[tok[0]])
multiplier = Word(nums)
multiplier = multiplier.setResultsName("multiply")
multiplier.setParseAction(self.parseMulti)
adder = []
for add in add_modifiers:
adder.append(CL(add))
adder = get_match_first(adder)
adder = adder.setResultsName("add")
adder.setParseAction(self.parseAdd)
modifier = (multiplier | adder) # + FollowedBy(units)
# ago
#
# e.g 5 days ago
ago = Optional(modifier) + units + Suppress(Word("ago"))
ago.setParseAction(self.parseAgo)
# time range
#
# e.g in the lat 10 days
time_range = Suppress(Optional(
CL("in the"))) + \
Suppress(Word("last") |
Word("past")) + \
Optional(modifier) + \
units
time_range.setParseAction(self.parseRange)
# special keyword handling
#
# e.g yesterday
# only handles yesterday right now, maybe need to be modified to do
# more
special_expr = []
for expr in special:
special_expr.append(
Keyword(expr).setParseAction(
lambda s, l, tok: special[tok[0]]))
special_expr = get_match_first(special_expr)
special_expr = special_expr.setResultsName("unit")
special_expr.setParseAction(self.parseAgo)
parser = (special_expr | ago | time_range)
return parser
def setup(self):
# some expressions that will be reused
units = []
for unit in time_units:
units.append(Keyword(unit))
units = get_match_first(units)
units = units.setResultsName("unit")
units.setParseAction(lambda s, l, tok: time_units[tok[0]])
multiplier = Word(nums)
multiplier = multiplier.setResultsName("multiply")
multiplier.setParseAction(self.parseMulti)
adder = []
for add in add_modifiers:
adder.append(CL(add))
adder = get_match_first(adder)
adder = adder.setResultsName("add")
adder.setParseAction(self.parseAdd)
modifier = (multiplier | adder) # + FollowedBy(units)
# ago
#
# e.g 5 days ago
ago = Optional(modifier) + units + Suppress(Word("ago"))
ago.setParseAction(self.parseAgo)
# time range
#
# e.g in the lat 10 days
time_range = Suppress(Optional(
CL("in the"))) + \
Suppress(Word("last") |
Word("past")) + \
Optional(modifier) + \
units
time_range.setParseAction(self.parseRange)
# special keyword handling
#
# e.g yesterday
# only handles yesterday right now, maybe need to be modified to do
# more
special_expr = []
for expr in special:
special_expr.append(
Keyword(expr).setParseAction(
lambda s, l, tok: special[tok[0]]))
special_expr = get_match_first(special_expr)
special_expr = special_expr.setResultsName("unit")
special_expr.setParseAction(self.parseAgo)
parser = (special_expr | ago | time_range)
return parser
def eval_query(query_str):
global WORD_CHARS
boolOperand = Word(WORD_CHARS)
boolOperand.setParseAction(BoolOperand)
boolExpr = infixNotation(
boolOperand,
[("not", 1, opAssoc.RIGHT, BoolNot), ("and", 2, opAssoc.LEFT, BoolAnd), ("or", 2, opAssoc.LEFT, BoolOr)],
)
return boolExpr.parseString(query_str.lower())[0].calcop()
def identifier_token():
'''
Grammar for identifiers
{[A-Z]|[a-z]|_)}{([A-Z]|[a-z]|[0-9]|_}*
'''
from grammar.symbols import UNDERLINE
# Identifier begin with letter and have letters, numbers or underline
identifier_tok = Word(alphas + UNDERLINE, alphanums + UNDERLINE)
# Convert identifier to upper case
identifier_tok.setParseAction(lambda t: t[0].lower())
return identifier_tok
class NodeParser:
def __init__(self):
self.num = Word(nums)
self.header = Regex(r"^UCLA.*")
self.comment = Regex(r"#.*")
self.bkid = Word(alphanums)
self.num_nodes = Literal("NumNodes") + Literal(":") + self.num(
"NumNodes")
self.num_terminals = Literal("NumTerminals") + Literal(":") + self.num(
"NumTerminals")
self.size = Group(self.num("width") + self.num("height"))
self.terminal = Optional(Literal("terminal"))
self.node = self.bkid("id") + self.size("size") + self.terminal
self.node_grammar = self.header + ZeroOrMore(self.comment) + self.num_nodes + self.num_terminals + \
OneOrMore(self.node)
self.coordinate = Group(self.num("x") + self.num("y"))
self.pl = self.bkid("id") + self.coordinate("coordinate") + \
Suppress(Literal(": N") + Optional(Literal(r"/FIXED")))
self.pl_grammar = self.header + ZeroOrMore(self.comment) + OneOrMore(
self.pl)
def compute_chip_size(self, benchmark):
benchmark_path = pathlib.Path(os.environ["BENCHMARK"])
node_file = benchmark_path / "ispd2005/{0}/{0}.nodes".format(benchmark)
pl_file = benchmark_path / "ispd2005/{0}/{0}.pl".format(benchmark)
print(node_file.as_posix())
print(pl_file.as_posix())
x_max = 0
y_max = 0
sizes = []
coordinates = []
self.size.setParseAction(
lambda tokens: sizes.append([tokens.width, tokens.height]))
self.coordinate.setParseAction(lambda tokens: coordinates.append(
(tokens.x, tokens.y)))
self.bkid.setParseAction(lambda tokens: print(tokens[0]))
self.node_grammar.parseFile(node_file.as_posix())
self.pl_grammar.parseFile(pl_file.as_posix())
for i in range(len(sizes)):
print(i)
if coordinates[i][0] + sizes[i][0] > x_max:
x_max = coordinates[i][0] + sizes[i][0]
if coordinates[i][1] + sizes[i][1] > y_max:
y_max = coordinates[i][1] + sizes[i][1]
return x_max, y_max
def __init__(self, EvaluateVariableChild=None, EvaluateNumberChild=None):
EvaluateVariableChild = EvaluateVariableChild or EvaluateVariable
EvaluateNumberChild = EvaluateNumberChild or EvaluateNumber
# what is a float number
floatNumber = Regex(r'[-]?\d+(\.\d*)?([eE][-+]?\d+)?')
# a variable is a combination of letters, numbers, and underscor
variable = Word(alphanums + "_")
# a sign is plus or minus
signOp = oneOf('+ -')
# an operand is a variable or a floating point number
operand = floatNumber ^ variable
# when a floatNumber is found, parse it with evaluate number
floatNumber.setParseAction(EvaluateNumberChild)
# when a variable is found, parse it with the EvaluateVariableChild
# or EvaluateVariable
variable.setParseAction(EvaluateVariableChild)
# comparisons include lt,le,gt,ge,eq,ne
comparisonOp = oneOf("< <= > >= == !=")
# negation of the boolean is !
notOp = oneOf("!")
# an expression is a either a comparison or
# a NOT operation (where NOT a is essentially (a == False))
comparisonExpression = operatorPrecedence(operand,
[
(comparisonOp,
2,
opAssoc.LEFT,
EvaluateComparison
),
(notOp,
1,
opAssoc.RIGHT,
EvaluateNot
),
])
# boolean logic of AND or OR
boolOp = oneOf("& |")
# a bool expression contains a nested bool expression or a comparison,
# joined with a boolean operation
boolExpression = Forward()
boolPossible = boolExpression | comparisonExpression
self.boolExpression = operatorPrecedence(boolPossible,
[
(boolOp,
2,
opAssoc.RIGHT,
EvaluateOrAnd
),
])
return
def _define_grammar():
"""
Creates and returns a copy of the selector grammar.
Wrapped in a function to avoid polluting the module namespace.
"""
expr = Forward()
label_name = Word(LABEL_CHARS)
label_name.setParseAction(LabelNode)
string_literal = QuotedString('"') | QuotedString("'")
string_literal.setParseAction(LiteralNode)
set_literal = (Suppress("{") +
delimitedList(QuotedString('"') | QuotedString("'"), ",") +
Suppress("}"))
set_literal.setParseAction(SetLiteralNode)
eq_comparison = label_name + Suppress("==") + string_literal
eq_comparison.setParseAction(LabelToLiteralEqualityNode)
not_eq_comparison = label_name + Suppress("!=") + string_literal
not_eq_comparison.setParseAction(InequalityNode)
in_comparison = label_name + Suppress(Keyword("in")) + set_literal
in_comparison.setParseAction(LabelInSetLiteralNode)
not_in = Suppress(Keyword("not") + Keyword("in"))
not_in_comparison = label_name + not_in + set_literal
not_in_comparison.setParseAction(NotInNode)
has_check = (Suppress("has(") + Word(LABEL_CHARS) + Suppress(")"))
has_check.setParseAction(HasNode)
comparison = (eq_comparison | not_eq_comparison | in_comparison
| not_in_comparison | has_check)
paren_expr = (Suppress("(") + expr + Suppress(")"))
value = comparison | paren_expr
and_expr = value + ZeroOrMore(Suppress("&&") + value)
and_expr.setParseAction(simplify_and_node)
or_expr = and_expr + ZeroOrMore(Suppress("||") + and_expr)
or_expr.setParseAction(simplify_or_node)
expr << or_expr
grammar = expr + StringEnd()
return grammar
def _define_base_characters_section (self):
heading = Literal('[Base Characters]')
character = Word(''.join(BASE_CHARACTERS), exact=1).setResultsName(
'character')
character.setParseAction(self._handle_character)
feature = Word(alphas).setResultsName('feature')
feature.setParseAction(self._handle_character_base_feature)
features = delimitedList(feature)
character_definition = Group(character + Suppress(':') + \
Optional(features))
character_definitions = Group(OneOrMore(character_definition)).setResultsName('base_characters')
section = Suppress(heading) + character_definitions
return section
def main(s):
lpar = Literal('(').suppress()
rpar = Literal(')').suppress()
integer = Word(nums)
element = Word(alphas, exact=1)
formula = Forward()
term = Group((element | Group(lpar + formula + rpar)('subgroup')) +
Optional(integer, default=1)('mult'))
formula << OneOrMore(term)
integer.setParseAction(process_integer)
term.setParseAction(process_term)
formula.setParseAction(process_formula)
return formula.parseString(s)[0]
def create_parser():
"""Creates the parser using PyParsing functions."""
# Day details (day number, superscript and day name)
daynum = Word(nums, max=2)
superscript = oneOf("th rd st nd", caseless=True)
day = oneOf(
"Mon Monday Tue Tues Tuesday Wed Weds Wednesday "
"Thu Thur Thurs Thursday Fri Friday Sat Saturday Sun Sunday",
caseless=True)
full_day_string = daynum + Optional(superscript).suppress()
full_day_string.setParseAction(check_day)
full_day_string.leaveWhitespace()
# Month names, with abbreviations, with action to convert to equivalent month number
month = oneOf(list(MONTHS.keys()), caseless=True) + \
Optional(Literal(".").suppress())
month.setParseAction(month_to_number)
# Year
year = Word(nums, exact=4)
year.setParseAction(lambda tokens: int(tokens[0]))
time_sep = oneOf(": .")
am_pm = oneOf("am pm", caseless=True)
hours = Word(nums, max=2)
mins = Word(nums, max=2)
time = hours("hour") + time_sep.suppress() + \
mins("mins") + Optional(am_pm)("meridian")
# date pattern
date = (Optional(time).suppress() & Optional(full_day_string("day"))
& Optional(day).suppress() & Optional(month("month"))
& Optional(year("year")))
# Possible separators
separator = oneOf("- -- to until through till untill \u2013 \u2014 ->",
caseless=True)
# Strings to completely ignore (whitespace ignored by default)
ignoreable_chars = oneOf(", from starting beginning of", caseless=True)
# Final putting together of everything
daterange = (Optional(
date("start") + Optional(time).suppress() + separator.suppress()) +
date("end") + Optional(time).suppress() + stringEnd())
daterange.ignore(ignoreable_chars)
return daterange
def __init__(self):
plus, minus, mult, div, mod = map(Literal, '+-*/%')
lpar = Literal('(')
rpar = Literal(')')
comma = Literal(',')
powop = Literal( '^' )
productop = mult | div
modop = Literal( '%' )
sumop = plus | minus
tupl = Forward()
number = Regex(r'[+-]?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?')
number.setParseAction(number_action)
ident = Word(alphas, alphanums+'_')
ident.setParseAction(ident_action)
funccall = ident + tupl
funccall.setParseAction(funccall_action)
atom = funccall | ident | number
atom.setParseAction(atom_action)
patom = lpar + atom + rpar
patom.setParseAction(patom_action)
powexpr = Forward()
powexpr << Group( atom + ZeroOrMore( ( powop + powexpr ) ) )
powexpr.setParseAction(powexpr_action)
modexpr = Forward()
modexpr << Group( powexpr + ZeroOrMore( ( modop + modexpr ) ) )
modexpr.setParseAction(modexpr_action)
product = Group( modexpr + ZeroOrMore( ( productop + modexpr ) ) )
product.setParseAction(product_action)
sumexpr = Group( product + Group( ZeroOrMore( sumop + product ) ) )
sumexpr.setParseAction(sum_action)
tupl << lpar + Optional(sumexpr + ZeroOrMore( comma + sumexpr ) ) + rpar
tupl.setParseAction(tupl_action)
expr = sumexpr | tupl
expr.setParseAction(expr_action)
self.bnf = expr
def parse (input):
# parse a string into an element of the abstract representation
# Grammar:
#
# <expr> ::= <integer>
# true
# false
# <identifier>
# ( if <expr> <expr> <expr> )
# ( let ( ( <name> <expr> ) ) <expr )
# ( + <expr> <expr> )
# ( * <expr> <expr> )
#
idChars = alphas+"_+*-?!=<>"
pIDENTIFIER = Word(idChars, idChars+"0123456789")
pIDENTIFIER.setParseAction(lambda result: EId(result[0]))
# A name is like an identifier but it does not return an EId...
pNAME = Word(idChars,idChars+"0123456789")
pINTEGER = Word("-0123456789","0123456789")
pINTEGER.setParseAction(lambda result: EInteger(int(result[0])))
pBOOLEAN = Keyword("true") | Keyword("false")
pBOOLEAN.setParseAction(lambda result: EBoolean(result[0]=="true"))
pEXPR = Forward()
pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")"
pIF.setParseAction(lambda result: EIf(result[2],result[3],result[4]))
pBINDING = "(" + pNAME + pEXPR + ")"
pBINDING.setParseAction(lambda result: (result[1],result[2]))
pLET = "(" + Keyword("let") + "(" + pBINDING + ")" + pEXPR + ")"
pLET.setParseAction(lambda result: ELet([result[3]],result[5]))
pPLUS = "(" + Keyword("+") + pEXPR + pEXPR + ")"
pPLUS.setParseAction(lambda result: ECall("+",[result[2],result[3]]))
pTIMES = "(" + Keyword("*") + pEXPR + pEXPR + ")"
pTIMES.setParseAction(lambda result: ECall("*",[result[2],result[3]]))
pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pIF | pLET | pPLUS | pTIMES)
result = pEXPR.parseString(input)[0]
return result # the first element of the result is the expression
def parse(input):
# parse a string into an element of the abstract representation
# Grammar:
#
# <expr> ::= <integer>
# true
# false
# <identifier>
# ( if <expr> <expr> <expr> )
# ( let ( ( <name> <expr> ) ) <expr )
# ( + <expr> <expr> )
# ( * <expr> <expr> )
#
idChars = alphas + "_+*-?!=<>"
pIDENTIFIER = Word(idChars, idChars + "0123456789")
pIDENTIFIER.setParseAction(lambda result: EId(result[0]))
# A name is like an identifier but it does not return an EId...
pNAME = Word(idChars, idChars + "0123456789")
pINTEGER = Word("-0123456789", "0123456789")
pINTEGER.setParseAction(lambda result: EInteger(int(result[0])))
pBOOLEAN = Keyword("true") | Keyword("false")
pBOOLEAN.setParseAction(lambda result: EBoolean(result[0] == "true"))
pEXPR = Forward()
pIF = "(" + Keyword("if") + pEXPR + pEXPR + pEXPR + ")"
pIF.setParseAction(lambda result: EIf(result[2], result[3], result[4]))
pBINDING = "(" + pNAME + pEXPR + ")"
pBINDING.setParseAction(lambda result: (result[1], result[2]))
pLET = "(" + Keyword("let") + "(" + pBINDING + ")" + pEXPR + ")"
pLET.setParseAction(lambda result: ELet([result[3]], result[5]))
pPLUS = "(" + Keyword("+") + pEXPR + pEXPR + ")"
pPLUS.setParseAction(lambda result: ECall("+", [result[2], result[3]]))
pTIMES = "(" + Keyword("*") + pEXPR + pEXPR + ")"
pTIMES.setParseAction(lambda result: ECall("*", [result[2], result[3]]))
pEXPR << (pINTEGER | pBOOLEAN | pIDENTIFIER | pIF | pLET | pPLUS | pTIMES)
result = pEXPR.parseString(input)[0]
return result # the first element of the result is the expression
def create_parser():
"""Creates the parser using PyParsing functions."""
# Day details (day number, superscript and day name)
daynum = Word(nums, max=2)
superscript = oneOf("th rd st nd", caseless=True)
day = oneOf("Mon Monday Tue Tues Tuesday Wed Weds Wednesday "
"Thu Thur Thurs Thursday Fri Friday Sat Saturday Sun Sunday", caseless=True)
full_day_string = daynum + Optional(superscript).suppress()
full_day_string.setParseAction(check_day)
full_day_string.leaveWhitespace()
# Month names, with abbreviations, with action to convert to equivalent month number
month = oneOf(list(MONTHS.keys()), caseless=True) + \
Optional(Literal(".").suppress())
month.setParseAction(month_to_number)
# Year
year = Word(nums, exact=4)
year.setParseAction(lambda tokens: int(tokens[0]))
time_sep = oneOf(": .")
am_pm = oneOf("am pm", caseless=True)
hours = Word(nums, max=2)
mins = Word(nums, max=2)
time = hours("hour") + time_sep.suppress() + \
mins("mins") + Optional(am_pm)("meridian")
# date pattern
date = (
Optional(time).suppress() & Optional(full_day_string("day")) & Optional(day).suppress() &
Optional(month("month")) & Optional(year("year"))
)
# Possible separators
separator = oneOf("- -- to until through till untill \u2013 \u2014 ->", caseless=True)
# Strings to completely ignore (whitespace ignored by default)
ignoreable_chars = oneOf(", from starting beginning of", caseless=True)
# Final putting together of everything
daterange = (
Optional(date("start") + Optional(time).suppress() + separator.suppress()) +
date("end") + Optional(time).suppress() + stringEnd()
)
daterange.ignore(ignoreable_chars)
return daterange
class NodeParser:
def __init__(self):
self.num = Word(nums)
self.header = Regex(r"^UCLA.*")
self.comment = Regex(r"#.*")
self.bkid = Word(alphanums)
self.num_nodes = Literal("NumNodes") + Literal(":") + self.num("NumNodes")
self.num_terminals = Literal("NumTerminals") + Literal(":") + self.num("NumTerminals")
self.size = Group(self.num("width") + self.num("height"))
self.terminal = Optional(Literal("terminal"))
self.node = self.bkid("id") + self.size("size") + self.terminal
self.node_grammar = (
self.header + ZeroOrMore(self.comment) + self.num_nodes + self.num_terminals + OneOrMore(self.node)
)
self.coordinate = Group(self.num("x") + self.num("y"))
self.pl = (
self.bkid("id") + self.coordinate("coordinate") + Suppress(Literal(": N") + Optional(Literal(r"/FIXED")))
)
self.pl_grammar = self.header + ZeroOrMore(self.comment) + OneOrMore(self.pl)
def compute_chip_size(self, benchmark):
benchmark_path = pathlib.Path(os.environ["BENCHMARK"])
node_file = benchmark_path / "ispd2005/{0}/{0}.nodes".format(benchmark)
pl_file = benchmark_path / "ispd2005/{0}/{0}.pl".format(benchmark)
print(node_file.as_posix())
print(pl_file.as_posix())
x_max = 0
y_max = 0
sizes = []
coordinates = []
self.size.setParseAction(lambda tokens: sizes.append([tokens.width, tokens.height]))
self.coordinate.setParseAction(lambda tokens: coordinates.append((tokens.x, tokens.y)))
self.bkid.setParseAction(lambda tokens: print(tokens[0]))
self.node_grammar.parseFile(node_file.as_posix())
self.pl_grammar.parseFile(pl_file.as_posix())
for i in range(len(sizes)):
print(i)
if coordinates[i][0] + sizes[i][0] > x_max:
x_max = coordinates[i][0] + sizes[i][0]
if coordinates[i][1] + sizes[i][1] > y_max:
y_max = coordinates[i][1] + sizes[i][1]
return x_max, y_max
def numeric_literal(cls) -> Token:
numeral = Word(nums) + ZeroOrMore(Optional(Word("_")) + Word(nums))
numeral.setParseAction(lambda t:
(int("".join(t.asList()).replace("_", "")), 0))
extended_digit = Word(nums + "ABCDEF")
based_numeral = extended_digit + ZeroOrMore(
Optional("_") + extended_digit)
based_literal = numeral + Literal("#") - based_numeral - Literal("#")
based_literal.setParseAction(
lambda t: (int(t[2].replace("_", ""), int(t[0][0])), int(t[0][0])))
return ((based_literal | numeral).setParseAction(
lambda t: Number(t[0][0], t[0][1])).setName("Number"))
def init_parser(self):
sequence_name = Word(
alphas + nums + "!#$%&\'*+-./;?@[\\]^_`{|}~",
max=100)
sequence_name.setParseAction(self.set_name)
name_block = Suppress(">") + sequence_name("name") + Suppress(LineEnd())
# Take a copy and disallow line breaks in the bases
bases = self.BASES.copy()
seq = bases("sequence") + Suppress(LineEnd())
seq.setParseAction(self.set_sequence)
name_and_seq = name_block + seq
return OneOrMore(name_and_seq)
def get_facet_expression():
facet_logical_expression = get_nested_logical_expression()
single_facet_expression = Word(
srange("[a-zA-Z0-9_.]")) +\
Optional(
Word('(').suppress() +
OneOrMore(facet_logical_expression).setParseAction(parse_one_or_more_facets_expression) +
Word(')').suppress())
single_facet_expression.setParseAction(parse_single_facet_expression)
base_facets_expression = OneOrMore(single_facet_expression
+ Optional(',').suppress())
base_facets_expression.setParseAction(parse_base_facets_expression)
facets_expression = Word('facets:').suppress() \
+ Word('[').suppress() \
+ base_facets_expression + Word(']').suppress()
return facets_expression
def get_facet_expression():
facet_logical_expression = get_nested_logical_expression()
single_facet_expression = Word(
srange("[a-zA-Z0-9_.]")) +\
Optional(
Word('(').suppress() +
OneOrMore(facet_logical_expression).setParseAction(parse_one_or_more_facets_expression) +
Word(')').suppress())
single_facet_expression.setParseAction(parse_single_facet_expression)
base_facets_expression = OneOrMore(single_facet_expression +
Optional(',').suppress())
base_facets_expression.setParseAction(parse_base_facets_expression)
facets_expression = Word('facets:').suppress() \
+ Word('[').suppress() \
+ base_facets_expression + Word(']').suppress()
return facets_expression
def get_vrml_format_old():
name_float = pword + pfloat
name_float3 = pword + Group(pfloat * 3)
pminus1 = Word('-1')
list_open = Literal('[').suppress()
list_close = Literal(']').suppress()
int_list = list_open + delimitedList(
pint.setParseAction(cvt_int)
| pminus1.setParseAction(cvt_int)) + list_close
float_list = list_open + delimitedList(pfloat) + list_close
name_int_list = pword + Group(int_list)
name_float_list = pword + Group(float_list)
data_value = name_int_list | name_float_list | name_float + name_float3
name_class_dict = pword + pword + dict_open + data_value + dict_close
name_dict = Forward()
name_dict <<= pword + dict_open + data_value + dict_close
#list_item = real | integer # | quotedString
#pfloat = Combine(Optional(oneOf("+-")) + Word(nums) + '.' + Word(nums))
#triple = nums + White + nums + White + nums
#pfloat = pfloat_neg
#names_dict = Optional(pword) + name_dict
#name_dict.parseString(simple_shape)
#name_class_dict.parseString(simple_shape)
msg = """
def parse_line(txt):
# Pseudo BNF for config format
# <dow>,<start> <sectionPart> <end>
#
# dow := "M" | "T" | "W" | "R" | "F"
# start := digits ":" digits
# sectionPart := specSection | digits name | "c" | "C"
# specSection := "(" digits ")" digits
# end := digits ":" digits
# name := alpha
day_of_week = oneOf(list("MTWRF"))
time = Word(nums) + ":" + Word(nums)
start_block = day_of_week + "," + time
name = Word(alphas + "'" + " " + "/") # what's in a name?
section = Word(nums)
specSection = "(" + Word(nums) + ")" + section
sectionPart = specSection + name | section + name | oneOf(list("Cc"))
line = start_block + sectionPart + time
# Parse Actions (run when a parse succeeds)
# group the times and the start block as a single string, get rid of
# possible trailing blanks on names
time.setParseAction(lambda x: [i for i in x if i != ':'])
start_block.setParseAction(lambda x: [i for i in x if i != ','])
name.setParseAction(lambda x: x[0].strip())
# format the specSection better
specSection.setParseAction(lambda x: "(%s) %s" % (x[1], x[3]))
def if_c_then_uppercase(x):
"""If the sectionPart is a consultation block, always use an upper C"""
if x[0] == "c" or x[0] == "C":
return "C"
else:
return x
sectionPart.setParseAction(if_c_then_uppercase)
# parses to a list of items ['M', '12', '05', '301', 'Ojalvo', '14', '00'] or
# (multi) ['T', '14', '25', '(321) 301', 'Dhorkah', '17', '25'] or
# (consult) ['F', '8', '00', 'C', '15', '00']
items = line.parseString(txt)
if len(items) == 6:
# consultation (no TA name)
day, h1, m1, sect, h2, m2 = items
return Section(day, (int(h1), int(m1)), sect, "", (int(h2), int(m2)))
day, h1, m1, sect, ta, h2, m2 = items
return Section(day, (int(h1), int(m1)), sect, ta, (int(h2),int(m2)))
def __init__(self):
"""Parser for instruction.
Example:
{{<a>},{<a>},{<a>},{<a>}}
{{<!>},{<!>},{<!>},{<a>}}
<{o"i!a,<{i<a>
"""
debug = False
self.garbo_count = 0
LBRACK, RBRACK, LBRACE, RBRACE, BANG = map(Suppress, "<>{}!")
nonspecial = CharsNotIn('<>{}!')
ignored = Word('!', printables, exact=2)
enclosed_garbo = SkipTo(Literal('>'), ignore=ignored)
val_str = Forward()
garbo_str = Forward()
item = Forward()
# a parsed item
item = (ignored | garbo_str | val_str | nonspecial).setDebug(debug)
# stuff in {}s
val_str << nestedExpr('{', '}', content=item,
ignoreExpr=None).setDebug(debug)
# stuff in <>s (suppressed)
garbo_str << (LBRACK + Optional(enclosed_garbo) +
RBRACK).setDebug(debug)
def cvt_list(toks):
return toks.asList()
val_str.setParseAction(cvt_list)
def take_garbo(s, loc, toks):
m = toks[0]
ig_str = re.sub(r'!.', '', m)
ln = len(ig_str)
self.garbo_count += ln
return f"<GARBO: {ln}>"
enclosed_garbo.setParseAction(take_garbo)
ignored.setParseAction(lambda: '!IGNORED')
# pattern build
self._pattern = item
def build(parsers: dict):
comma = Literal(",")
rb = Literal(")")
lb = Literal("(")
srb = Literal("]")
slb = Literal("[")
number = Regex(r"0|[1-9][0-9]*")
string = quotedString()
name = Word(alphanums)
label = Keyword(STRONG) | Keyword(WEAK) | Literal(SHORT_WEAK)
param = Combine(Keyword(PARAM) + slb + number + srb)
marker = Keyword(RESULT) | Keyword(TRUE) | Keyword(FALSE) | Keyword(
THIS) | Keyword(_THIS) | param
function = Keyword(GET)
get = Literal(GETATTR)
operator1 = Literal(MUL) | Literal(DIV) | Literal(MOD)
operator2 = Literal(ADD) | Literal(SUB)
operator3 = Literal(EQUAL) | Literal(NOT_EQUAL)
operator3 |= And(Keyword(word) for word in IS_NOT.split(" ")) | Keyword(IS)
operator4 = Literal(GREATER_OR_EQUAL) | Literal(GREATER) | Literal(
LOWER_OR_EQUAL) | Literal(LOWER)
operator5 = Keyword(AND)
operator6 = Keyword(OR)
operator7 = Keyword(FOLLOW)
expression = Forward()
string_st = string.setParseAction(parsers[STRING])
name_st = name.setParseAction(parsers[STRING])
marker_st = marker.setParseAction(parsers[MARKER])
tuple_st = expression + ZeroOrMore(comma + expression)
round_invocation_st = (lb + Optional(tuple_st) + rb).setParseAction(
parsers[INVOCATION])
function_st = (function + Suppress(round_invocation_st)).setParseAction(
parsers[FUNCTION])
getattr_st = (marker_st | name_st) + OneOrMore(
(get + Suppress(name_st)).setParseAction(parsers[OPERATOR]))
atom_st = (lb + expression +
rb) | function_st | string_st | getattr_st | marker_st
operator_st = atom_st + ZeroOrMore(
(operator1 + Suppress(atom_st)).setParseAction(parsers[OPERATOR]))
operator_st = operator_st + ZeroOrMore(
(operator2 + Suppress(operator_st)).setParseAction(parsers[OPERATOR]))
operator_st = operator_st + ZeroOrMore(
(operator3 + Suppress(operator_st)).setParseAction(parsers[OPERATOR]))
operator_st = operator_st + ZeroOrMore(
(operator4 + Suppress(operator_st)).setParseAction(parsers[OPERATOR]))
operator_st = operator_st + ZeroOrMore(
(operator5 + Suppress(operator_st)).setParseAction(parsers[OPERATOR]))
operator_st = operator_st + ZeroOrMore(
(operator6 + Suppress(operator_st)).setParseAction(parsers[OPERATOR]))
operator_st = operator_st + ZeroOrMore(
(operator7 + Suppress(operator_st)).setParseAction(parsers[OPERATOR]))
expression << operator_st
getattr_st.enablePackrat()
statement = (Optional(label, STRONG) +
Suppress(expression)).setParseAction(parsers[LABEL])
return ZeroOrMore(statement) + StringEnd()
def _define_features_section (self, heading, name, parse_action):
"""Returns a parser object for a feature section.
:param heading: section heading
:type heading: `str`
:param name: name for results
:type name: `str`
:param parse_action: parse action for a feature
:type parse_action: `function`
"""
heading = Literal('[{}]'.format(heading))
feature = Word(alphas).setResultsName('feature')
feature.setParseAction(parse_action)
features = Group(OneOrMore(feature)).setResultsName(name)
section = Suppress(heading) + features
return section
def get_aggregations_expression():
aggs_logical_expression = get_nested_logical_expression()
single_aggs_expression = Word(
srange("[a-zA-Z0-9_.]")) +\
Optional(
Word('(').suppress() +
OneOrMore(aggs_logical_expression).setParseAction(
parse_one_or_more_aggs_expression) +
Word(')').suppress())
single_aggs_expression.setParseAction(parse_single_aggs_expression)
base_aggs_expression = OneOrMore(single_aggs_expression
+ Optional(',').suppress())
base_aggs_expression.setParseAction(parse_base_aggs_expression)
aggs_expression = Word('aggregations:').suppress() \
+ Word('[').suppress() \
+ base_aggs_expression + Word(']').suppress()
return aggs_expression
def makeBNFParser(self):
# Basic Components
point = Literal(".")
integer = Word(nums)
fNumber = Combine(integer + point + integer)
number = fNumber | integer
unaryStarts = self.makeCombo(self.UNARY_OPS_START, Literal)
unaryEnds = self.makeCombo(self.UNARY_OPS_END, Literal)
ops = self.makeCombo(self.BINARY_OPS, Literal)
kwds = self.makeCombo(self.SPECIAL_KWDS, CaselessKeyword)
kwdLits = self.makeCombo(self.SPECIAL_KWDS, CaselessLiteral)
functs = self.makeCombo(self.SPECIAL_FUNCT, CaselessKeyword)
functLits = self.makeCombo(self.SPECIAL_FUNCT, CaselessLiteral)
anyLits = Word(printables)
variable = Word(alphas)
startDelim = self.makeCombo(self.START_DELIMS, Literal, True)
endDelim = self.makeCombo(self.END_DELIMS, Literal, True)
# Expression
group = Forward()
factor = Forward()
expr = Forward()
val = (
startDelim |
#unaryStarts.setParseAction(self.pushUnarySt) |
kwds.setParseAction(self.pushKeyword)
| functs.setParseAction(self.pushFunct)
| number.setParseAction(self.pushNum)
| kwdLits.setParseAction(self.pushKeyword)
| functLits.setParseAction(self.pushFunct)
| unaryEnds.setParseAction(self.pushUnaryEnd) | endDelim
| variable.setParseAction(self.pushMultVars)
| ops.setParseAction(self.pushOperation)
| anyLits.setParseAction(self.pushFirst))
vals = val + ZeroOrMore(val)
group << (Optional(unaryStarts.setParseAction(self.pushUnarySt)) +
Optional(functs.setParseAction(self.pushFunct)) +
Optional(startDelim) + factor + Optional(endDelim) +
Optional(unaryEnds.setParseAction(self.pushUnaryEnd)))
factor << (vals | expr)
expr << (group +
ZeroOrMore(ops.setParseAction(self.pushOperation) + group))
closedExpr = expr + StringEnd()
return closedExpr
def makeBNFParser(self):
# Basic Components
point = Literal(".")
integer = Word(nums)
fNumber = Combine(integer + point + integer)
number = fNumber | integer
unaryStarts = self.makeCombo(self.UNARY_OPS_START, Literal)
unaryEnds = self.makeCombo(self.UNARY_OPS_END, Literal)
ops = self.makeCombo(self.BINARY_OPS, Literal)
kwds = self.makeCombo(self.SPECIAL_KWDS, CaselessKeyword)
kwdLits = self.makeCombo(self.SPECIAL_KWDS, CaselessLiteral)
functs = self.makeCombo(self.SPECIAL_FUNCT, CaselessKeyword)
functLits = self.makeCombo(self.SPECIAL_FUNCT, CaselessLiteral)
anyLits = Word(printables)
variable = Word(alphas)
startDelim = self.makeCombo(self.START_DELIMS, Literal, True)
endDelim = self.makeCombo(self.END_DELIMS, Literal, True)
# Expression
group = Forward()
factor = Forward()
expr = Forward()
val = (startDelim |
#unaryStarts.setParseAction(self.pushUnarySt) |
kwds.setParseAction(self.pushKeyword) |
functs.setParseAction(self.pushFunct) |
number.setParseAction(self.pushNum) |
kwdLits.setParseAction(self.pushKeyword) |
functLits.setParseAction(self.pushFunct) |
unaryEnds.setParseAction(self.pushUnaryEnd) |
endDelim |
variable.setParseAction(self.pushMultVars)|
ops.setParseAction(self.pushOperation) |
anyLits.setParseAction(self.pushFirst))
vals = val + ZeroOrMore(val)
group << (Optional(unaryStarts.setParseAction(self.pushUnarySt)) +
Optional(functs.setParseAction(self.pushFunct)) +
Optional(startDelim) + factor + Optional(endDelim) +
Optional(unaryEnds.setParseAction(self.pushUnaryEnd)))
factor << (vals | expr)
expr << (group + ZeroOrMore(ops.setParseAction(self.pushOperation) + group))
closedExpr = expr + StringEnd()
return closedExpr
def _build_grammar(self):
expr = Forward()
float_lit = Combine(Word(nums) + '.' + Word(nums))
float_lit.setName('float')
float_lit.setParseAction(lambda x: \
self.to_literal(float(x[0])))
int_lit = Word(nums)
int_lit.setName('int')
int_lit.setParseAction(lambda x: \
self.to_literal(int(x[0])))
num = (float_lit | int_lit)
num.setParseAction(lambda x: x[0])
tag_name = Word(alphas + "_", alphanums + "_")
tag_name.setName('tag_name')
tag_name.setParseAction(lambda t: tag_reference.TagReference(t[0]))
quoted_string = QuotedString("'")
quoted_string.setParseAction(lambda s: self.to_literal(s[0]))
oper = oneOf('+ * / -')
oper.setParseAction(lambda o: o[0])
lpar = Literal("(").suppress()
rpar = Literal(")").suppress()
arith = Group(lpar + expr + oper + expr + rpar)
arith.setParseAction(lambda t: \
self.to_arith(t[0][0], t[0][1], t[0][2]))
assign = tag_name + '=' + expr
assign.setName('assign')
assign.setParseAction(lambda x: self.to_assign(x[0],x[2]))
print_tags = Literal('?')
print_tags.setParseAction(lambda x: self.to_print_tags())
expr <<(arith|assign|tag_name|num|quoted_string|print_tags)
expr.setParseAction(lambda x: x[0])
return expr
def _getPattern(self):
arith_expr = Forward()
comp_expr = Forward()
logic_expr = Forward()
LPAR, RPAR, SEMI = map(Suppress, "();")
identifier = Word(alphas+"_", alphanums+"_")
multop = oneOf('* /')
plusop = oneOf('+ -')
expop = Literal( "^" )
compop = oneOf('> < >= <= != ==')
andop = Literal("AND")
orop = Literal("OR")
current_value = Literal( "." )
assign = Literal( "=" )
# notop = Literal('NOT')
function = oneOf(' '.join(self.FUNCTIONS))
function_call = Group(function.setResultsName('fn') + LPAR + Optional(delimitedList(arith_expr)) + RPAR)
aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}')
single_column = QuotedString(quoteChar='[', endQuoteChar=']')
integer = Regex(r"-?\d+")
real = Regex(r"-?\d+\.\d*")
# quotedString enables strings without quotes to pass
operand = \
function_call.setParseAction(self.__evalFunction) | \
aggregate_column.setParseAction(self.__evalAggregateColumn) | \
single_column.setParseAction(self.__evalSingleColumn) | \
((real | integer).setParseAction(self.__evalConstant)) | \
quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \
current_value.setParseAction(self.__evalCurrentValue) | \
identifier.setParseAction(self.__evalString)
arith_expr << operatorPrecedence(operand,
[
(expop, 2, opAssoc.LEFT, self.__expOp),
(multop, 2, opAssoc.LEFT, self.__multOp),
(plusop, 2, opAssoc.LEFT, self.__addOp),
])
# comp_expr = Group(arith_expr + compop + arith_expr)
comp_expr << operatorPrecedence(arith_expr,
[
(compop, 2, opAssoc.LEFT, self.__evalComparisonOp),
])
logic_expr << operatorPrecedence(comp_expr,
[
(andop, 2, opAssoc.LEFT, self.__evalLogicOp),
(orop, 2, opAssoc.LEFT, self.__evalLogicOp)
])
pattern = logic_expr + StringEnd()
return pattern
def genParseObject(self,cLocale):
# generate dictionary of months
self.monthdict = {}
x = 1
sym = DateFormatSymbols(cLocale)
for sm in sym.getShortMonths():
self.monthdict[sm.encode('utf-8').lower()] = x
x += 1
x = 1
for mm in sym.getMonths():
self.monthdict[mm.encode('utf-8').lower()] = x
x += 1
def parseHandler(s,loc,toks):
if toks[0].lower() not in self.monthdict:
raise 'f**k'
ret = Word(alphas + alphas8bit).setResultsName('month')
ret.setParseAction(parseHandler)
return ret
def __init__(self, max_=60):
# define the grammar structure
digits = "0123456789"
star = Literal('*')
number = Word(digits) | Word(alphas)
steps = number
range_ = number + Optional(Literal('-') + number)
numspec = star | range_
expr = Group(numspec) + Optional(Literal('/') + steps)
extra_groups = ZeroOrMore(Literal(',') + expr)
groups = expr + extra_groups + StringEnd()
# define parse actions
star.setParseAction(self._expand_star)
number.setParseAction(self._expand_number)
range_.setParseAction(self._expand_range)
expr.setParseAction(self._filter_steps)
extra_groups.setParseAction(self._ignore_comma)
groups.setParseAction(self._join_to_set)
self.max_ = max_
self.parser = groups
def get_sort_expression():
value_expression = Word(srange("[a-zA-Z0-9_.*]"))
value_expression.setParseAction(lambda tokens: tokens[0])
quoted_value_expression = Word('"').suppress() +\
value_expression + Word('"').suppress()
option_value = value_expression | quoted_value_expression
option_value.setParseAction(lambda tokens: tokens[0])
simple_option = Word(srange("[a-zA-Z0-9_.*]")) +\
Word(':').suppress() + option_value
simple_option.setParseAction(lambda tokens: (tokens[0], tokens[1]))
option = Forward()
option << (simple_option |
(Word(srange("[a-zA-Z0-9_.*]")) +
Word(':').suppress() +
nestedExpr(content=option)))
option.setParseAction(
lambda tokens: parse_sort_field_option(tokens.asList())
)
exp = option + ZeroOrMore(Word(',').suppress() + option)
field_expression = Optional('-') + Word(
srange("[a-zA-Z0-9_.*]")
) + Optional(nestedExpr(content=exp))
field_expression.setParseAction(parse_sort_field_expression)
fields_expression = field_expression + ZeroOrMore(
Word(',').suppress() + field_expression)
fields_expression.setParseAction(parse_sort_expression)
sort_expression = Word('sort:').suppress() \
+ Word('[').suppress() \
+ fields_expression + Word(']').suppress()
return sort_expression
def _define_grammar (self):
heading_open = Literal('[').suppress()
heading_close = Literal(']').suppress()
cluster = self._define_cluster()
base_feature_set = self._define_base_feature_set()
rule = self._define_rule(cluster, base_feature_set)
rules = OneOrMore(rule).setResultsName('rules')
date_number = Word('-'+nums, nums).setResultsName('date_number')
date_number.setParseAction(self._handle_date_number)
date_name = Combine(OneOrMore(Word(alphanums+'.')), adjacent=False,
joinString=' ').setResultsName('date_name')
date_heading = heading_open + Suppress(Keyword('Date')) + date_number \
+ date_name + heading_close
date_section = date_heading + rules
date_sections = Group(OneOrMore(date_section))
language_name = Combine(OneOrMore(Word(alphas)), adjacent=False,
joinString=' ').setResultsName('language_name')
language_heading = heading_open + Suppress(Keyword('Language')) + \
language_name + heading_close
language_section = Group(language_heading + date_sections)
ruleset = OneOrMore(language_section)
return ruleset
def dict_from_key_values_pyparsing(file, lowercase_keys=False):
"""
>>> filename = os.path.dirname(__file__)
>>> filename = os.path.join(filename, "data/iradio-initial.pls")
>>> with open(filename, "rt", encoding="utf8") as fh:
... d = dict_from_key_values_pyparsing(fh)
>>> for key in sorted(d.keys())[-4:]:
... print("{0}: {1}".format(key, d[key]))
title6: Virgin Xtreme (Broadband)
title7: Virgin Classic Rock (Modem)
title8: Virgin Classic Rock (Broadband)
title9: CBC Radio One (Canada)
>>> d["file13"]
'http://media.hiof.no/streams/m3u/nrk-petre-172.ogg.m3u'
>>> d["genre15"]
''
>>> len(d.keys())
54
"""
def accumulate(tokens):
key, value = tokens
key = key.lower() if lowercase_keys else key
key_values[key] = value
key_values = {}
left_bracket, right_bracket, equals = map(Suppress, "[]=")
ini_header = left_bracket + CharsNotIn("]") + right_bracket
key_value = Word(alphanums) + equals + restOfLine
key_value.setParseAction(accumulate)
comment = "#" + restOfLine
parser = OneOrMore(ini_header | key_value)
parser.ignore(comment)
try:
parser.parseFile(file)
except ParseException as err:
print("parse error: {0}".format(err))
return {}
return key_values
def _construct_grammar():
logical_operator = get_logical_operator()
logical_expression = get_logical_expression()
facets_expression = get_facet_expression()
nested_expression = get_nested_expression()
# The below line describes how the type expression should be.
type_expression = Word('type')\
+ Word(':').suppress()\
+ Word(alphanums)\
+ Optional(CaselessLiteral('AND')).suppress()
type_expression.setParseAction(parse_type_expression)
base_expression = Optional(type_expression)\
+ ZeroOrMore((facets_expression
| nested_expression
| logical_expression)
+ Optional(logical_operator)).setParseAction(
parse_one_or_more_logical_expressions)
base_expression.setParseAction(parse_type_logical_facets_expression)
return base_expression
def find_query_syntax(self):
operators = "+-*/%=&|><!"
find_keyword = CaselessLiteral("FIND").setParseAction(self.create_find_node_query_obj)
variable = Word(alphanums)
vtype = Word(alphanums)
node = Optional(variable, default=None) + Suppress(Literal(":")) + vtype
edge = Optional(variable, default=None) + Suppress(Literal(":")) + Optional(vtype, default=None)
query_graph = node + ZeroOrMore(Suppress(Literal("-")) + edge + Suppress(Literal(">")) + node)
query_graph.setParseAction(self.set_query_graph)
where_keyword = CaselessLiteral("WHERE")
expression = self.condition_expression_syntax()
expression.setParseAction(self.set_expression)
return_keyword = CaselessLiteral("RETURN")
variable_list = Word(alphanums + ".") + ZeroOrMore(Suppress(Literal(",")) + Word(alphanums + "."))
variable_list.setParseAction(self.set_variable_list)
find_node_query = (
find_keyword
+ OneOrMore(query_graph)
+ Optional(where_keyword + expression)
+ return_keyword
+ variable_list
)
return find_node_query
