def get_formula_parser(chemical_tokens):
LPAR, RPAR = map(Suppress, "[]")
index = Regex(r"\d+(\.\d*)?").setParseAction(lambda t: float(t[0]))
element = oneOf(chemical_tokens)
chemical_formula = Forward()
term = Group((element
| Group(LPAR + chemical_formula + RPAR)("subgroup")) +
Optional(index, default=1)("mult"))
chemical_formula << OneOrMore(term)
def multiplyContents(tokens):
t = tokens[0]
if t.subgroup:
mult = t.mult
for term in t.subgroup:
term[1] *= mult
return t.subgroup
term.setParseAction(multiplyContents)
def sumByElement(tokens):
elementsList = [t[0] for t in tokens]
duplicates = len(elementsList) > len(set(elementsList))
if duplicates:
ctr = defaultdict(int)
for t in tokens:
ctr[t[0]] += t[1]
return ParseResults([ParseResults([k, v]) for k, v in ctr.items()])
chemical_formula.setParseAction(sumByElement)
return chemical_formula
def get_parse_expression(parse_func, parse_var_id):
"""return a parse expression with for the
input parseActions
"""
var_id = Group(FUNC_NAME + COLON + IDENTIFIER) | REAL | INTEGER | IDENTIFIER
var_id.setParseAction(parse_var_id)
# Forward declaration for an Arithmetic Expression
arith_expr = Forward()
func_call = Group(FUNC_NAME + LPAREN + Optional(Group(delimitedList(arith_expr))) + RPAREN)
# An Arithmetic expression can have a var_id or
# a function call as an operand
# pylint: disable=expression-not-assigned
arith_expr << operatorPrecedence(
func_call | var_id,
[
(EXPONENTIATION_OPS, 2, opAssoc.LEFT, eval_binary_op),
(UNARY_OPS, 1, opAssoc.RIGHT, eval_unary_op),
(MULT_OPS, 2, opAssoc.LEFT, eval_binary_op),
(SUM_OPS, 2, opAssoc.LEFT, eval_binary_op),
(REL_OPS, 2, opAssoc.LEFT, eval_binary_op),
(LOGICAL_OPS, 2, opAssoc.LEFT, eval_binary_op),
],
)
# pylint: enable=expression-not-assigned
# Argument expression for a function call
# An argument to a function can be an
# IDENTIFIER, Arithmetic expression, REAL number, INTEGER or a
# Function call itself
func_call.setParseAction(parse_func)
return arith_expr
def __init__(self, ffilter, queue_out):
FuzzQueue.__init__(self, queue_out)
Thread.__init__(self)
self.setName('filter_thread')
self.queue_out = queue_out
if PYPARSING:
element = oneOf("c l w h")
digits = "XB0123456789"
integer = Word( digits )#.setParseAction( self.__convertIntegers )
elementRef = Group(element + oneOf("= != < > >= <=") + integer)
operator = oneOf("and or")
definition = elementRef + ZeroOrMore( operator + elementRef)
nestedformula = Group(Suppress(Optional(Literal("("))) + definition + Suppress(Optional(Literal(")"))))
self.finalformula = nestedformula + ZeroOrMore( operator + nestedformula)
elementRef.setParseAction(self.__compute_element)
nestedformula.setParseAction(self.__compute_formula)
self.finalformula.setParseAction(self.__myreduce)
self.res = None
self.hideparams = ffilter
if "XXX" in self.hideparams['codes']:
self.hideparams['codes'].append("0")
self.baseline = None
def _define_context_component (self, cluster, base_feature_set):
placeholder = Literal(SOURCE_PLACEHOLDER)
placeholder.setParseAction(self._handle_placeholder)
context_component = Group(ZeroOrMore(cluster ^ base_feature_set) + \
placeholder + ZeroOrMore(cluster ^ base_feature_set)).setResultsName('context_component')
context_component.setParseAction(self._handle_context_component)
return context_component
def get_parse_expression(parse_func, parse_var_id):
"""return a parse expression with for the
input parseActions
"""
var_id = Group(FUNC_NAME + COLON +
IDENTIFIER) | REAL | INTEGER | IDENTIFIER
var_id.setParseAction(parse_var_id)
# Forward declaration for an Arithmetic Expression
arith_expr = Forward()
func_call = Group(FUNC_NAME + LPAREN +
Optional(Group(delimitedList(arith_expr))) + RPAREN)
# An Arithmetic expression can have a var_id or
# a function call as an operand
# pylint: disable=expression-not-assigned
arith_expr << operatorPrecedence(func_call | var_id, [
(EXPONENTIATION_OPS, 2, opAssoc.LEFT, eval_binary_op),
(UNARY_OPS, 1, opAssoc.RIGHT, eval_unary_op),
(MULT_OPS, 2, opAssoc.LEFT, eval_binary_op),
(SUM_OPS, 2, opAssoc.LEFT, eval_binary_op),
(REL_OPS, 2, opAssoc.LEFT, eval_binary_op),
(LOGICAL_OPS, 2, opAssoc.LEFT, eval_binary_op)
])
# pylint: enable=expression-not-assigned
# Argument expression for a function call
# An argument to a function can be an
# IDENTIFIER, Arithmetic expression, REAL number, INTEGER or a
# Function call itself
func_call.setParseAction(parse_func)
return arith_expr
def ifParser():
comma = Literal(",").suppress()
hash = Literal("#").suppress()
equal = Literal("=").suppress()
# Rules and labels
rulename = Word (alphanums + "_")
rulecategory = oneOf("Protocol_Rules Invariant_Rules Decomposition_Rules Intruder_Rules Init Goal")
label = hash + Literal("lb") + equal + rulename + comma + Literal("type") + equal + rulecategory
labeledrule = Group(label) + Group(ruleParser())
def labeledruleAction(s,l,t):
if t[0][3] == "Protocol_Rules":
print "-----------------"
print "- Detected rule -"
print "-----------------"
print t[0]
print t[1]
print
labeledrule.setParseAction(labeledruleAction)
# A complete file
parser = OneOrMore(labeledrule)
parser.ignore("##" + restOfLine)
return parser
def message_type_definition() -> Token:
first_aspect = Keyword("First") - Keyword("=>") - mathematical_expression()
first_aspect.setParseAction(parse_aspect)
length_aspect = Keyword("Length") - Keyword(
"=>") - mathematical_expression()
length_aspect.setParseAction(parse_aspect)
component_aspects = Keyword("with") - delimitedList(first_aspect
| length_aspect)
component_aspects.setParseAction(parse_aspects)
then = locatedExpr(
Keyword("then") - (Keyword("null") | unqualified_identifier()) -
Group(Optional(component_aspects)) -
Group(Optional(value_constraint())))
then.setParseAction(parse_then)
then_list = then + ZeroOrMore(comma() - then)
then_list.setParseAction(lambda t: [t.asList()])
component_item = (~Keyword("end") + ~CaselessKeyword("Message") -
unqualified_identifier() + Literal(":") -
qualified_identifier() - Optional(then_list) -
semicolon())
component_item.setParseAction(lambda t: Component(t[0], t[2], t[3])
if len(t) >= 4 else Component(t[0], t[2]))
component_item.setName("Component")
null_component_item = Keyword("null") - then - semicolon()
null_component_item.setParseAction(lambda t: Component(None, None, [t[1]]))
null_component_item.setName("NullComponent")
component_list = Group(
Optional(null_component_item) - component_item -
ZeroOrMore(component_item))
component_list.setParseAction(lambda t: t.asList())
return (Keyword("message") - component_list - Keyword("end message")
| Keyword("null message")).setName("Message")
def ifParser():
comma = Literal(",").suppress()
hash = Literal("#").suppress()
equal = Literal("=").suppress()
# Rules and labels
rulename = Word(alphanums + "_")
rulecategory = oneOf(
"Protocol_Rules Invariant_Rules Decomposition_Rules Intruder_Rules Init Goal"
)
label = hash + Literal("lb") + equal + rulename + comma + Literal(
"type") + equal + rulecategory
labeledrule = Group(label) + Group(ruleParser())
def labeledruleAction(s, l, t):
if t[0][3] == "Protocol_Rules":
print "-----------------"
print "- Detected rule -"
print "-----------------"
print t[0]
print t[1]
print
labeledrule.setParseAction(labeledruleAction)
# A complete file
parser = OneOrMore(labeledrule)
parser.ignore("##" + restOfLine)
return parser
def list_dict(word=word_free):
"""
Return the pyparsing lexical element, that parses a string either as a list
or as a dictionary.
Parameters
----------
word : lexical element
A custom lexical element for word.
Returns
-------
ld : lexical element
The returned lexical element parses a string in the form
``..., ..., ...`` or ``key1:..., key2=..., key3: ...``
where ``...`` is a ``list_item`` from :func:`get_standard_type_defs()`
and interprets it as a list or a dictionary.
"""
defs = get_standard_type_defs(word)
i = defs['list_item']
arg = i.copy()
arg.setParseAction(lambda t: (t[0],))
narg = word_strict + (colon | equal_sign) + i
narg.setParseAction(lambda t: (t[0], t[1]))
ld = Group(list_of(narg | arg))
ld.setParseAction(lambda t: ([x[0] for x in t[0] if len(x) == 1],
dict([x for x in t[0] if len(x) > 1]))
)
return ld
def __init__(self, ffilter, queue_out):
FuzzQueue.__init__(self, queue_out)
Thread.__init__(self)
self.setName('filter_thread')
self.queue_out = queue_out
if PYPARSING:
element = oneOf("c l w h")
digits = "XB0123456789"
integer = Word( digits )#.setParseAction( self.__convertIntegers )
elementRef = Group(element + oneOf("= != < > >= <=") + integer)
operator = oneOf("and or")
definition = elementRef + ZeroOrMore( operator + elementRef)
nestedformula = Group(Suppress(Optional(Literal("("))) + definition + Suppress(Optional(Literal(")"))))
self.finalformula = nestedformula + ZeroOrMore( operator + nestedformula)
elementRef.setParseAction(self.__compute_element)
nestedformula.setParseAction(self.__compute_formula)
self.finalformula.setParseAction(self.__myreduce)
self.res = None
self.hideparams = ffilter
if "XXX" in self.hideparams['codes']:
self.hideparams['codes'].append("0")
self.baseline = None
def __init__(self, filter_string=None):
self.filter_string = filter_string
self.baseline = None
quoted_str_value = QuotedString("'", unquoteResults=True, escChar="\\")
int_values = Word("0123456789").setParseAction(
lambda s, l, t: [int(t[0])])
error_value = Literal("XXX").setParseAction(self.__compute_xxx_value)
operator_call = Regex(
r"\|(?P<operator>(m|d|e|un|u|r|l|sw|gre|gregex|unique|startswith|decode|encode|unquote|replace|lower|upper))"
r"\((?:(?P<param1>('.*?'|\d+))(?:,(?P<param2>('.*?'|\d+)))?)?\)",
asMatch=True,
).setParseAction(lambda s, l, t: [(l, t[0])])
fuzz_symbol = Regex(
r"FUZ(?P<index>\d)*Z(?:\[(?P<field>(\w|_|-|\.)+)\])?",
asMatch=True).setParseAction(self._compute_fuzz_symbol)
res_symbol = Regex(
r"(description|nres|code|chars|lines|words|md5|content|timer|url|plugins|l|w|c|(r|history)(\w|_|-|\.)*|h)"
).setParseAction(self._compute_res_symbol)
bbb_symbol = Regex(r"BBB(?:\[(?P<field>(\w|_|-|\.)+)\])?",
asMatch=True).setParseAction(
self.__compute_bbb_symbol)
fuzz_statement = Group((fuzz_symbol | res_symbol | bbb_symbol
| int_values | quoted_str_value) +
Optional(operator_call, None)).setParseAction(
self.__compute_res_value)
operator = oneOf("and or")
not_operator = Optional(oneOf("not"), "notpresent")
symbol_expr = Group(fuzz_statement +
oneOf("= == != < > >= <= =~ !~ ~ := =+ =-") +
(error_value | fuzz_statement)).setParseAction(
self.__compute_expr)
definition = symbol_expr ^ fuzz_statement
definition_not = not_operator + definition
definition_expr = definition_not + ZeroOrMore(operator +
definition_not)
nested_definition = Group(
Suppress("(") + definition_expr + Suppress(")"))
nested_definition_not = not_operator + nested_definition
self.finalformula = (nested_definition_not
| definition_expr) + ZeroOrMore(operator + (
nested_definition_not | definition_expr))
definition_not.setParseAction(self.__compute_not_operator)
nested_definition_not.setParseAction(self.__compute_not_operator)
nested_definition.setParseAction(self.__compute_formula)
self.finalformula.setParseAction(self.__myreduce)
self.res = None
self.stack = []
self._cache = collections.defaultdict(set)
def _formula_parser(formula, calculate_or_not, atoms):
''' parses the chemical formula and determine its mol weight. But only
if `calculate_or_not` equals `calculate`, otherwise, do nothing. Use
the molecular weight from atoms dict. '''
if (calculate_or_not != 'calculate') or (formula in [
'N_tot_k', 'PO4_ortho', 'SO4_ic', 'alkalinity'
]):
return calculate_or_not
# define some strings to use later, when describing valid lists
# of characters for chemical symbols and numbers
caps = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
lowers = caps.lower()
digits = "0123456789"
brackets = "[]{}()"
if any([_ in formula for _ in brackets]):
raise Exception('Not implemented with brackets in formula string')
# Version 1
# Define grammar for a chemical formula
# - an element is a Word, beginning with one of the characters in caps,
# followed by zero or more characters in lowers
# - an integer is a Word composed of digits
# - an elementRef is an element, optionally followed by an integer - if
# the integer is omitted, assume the value "1" as a default; these are
# enclosed in a Group to make it easier to walk the list of parsed
# chemical symbols, each with its associated number of atoms per
# molecule
# - a chemicalFormula is just one or more elementRef's
# Version 3 - Compute partial molecular weight per element, simplifying
# summing
# No need to redefine grammar, just define parse action function, and
# attach to elementRef
# Auto-convert integers, and add results names
def convert_integers(tokens):
return int(tokens[0])
element = Word(caps, lowers)
integer = Word(digits).setParseAction(convert_integers)
element_ref = Group(
element("symbol") + Optional(integer, default=1)("qty"))
chemical_formula = OneOrMore(element_ref)
def compute_element_weight(tokens):
element = tokens[0]
element["weight"] = atoms[element.symbol] * element.qty
element_ref.setParseAction(compute_element_weight)
formula_data = chemical_formula.parseString(formula)
# compute molecular weight
mw = sum([element.weight for element in formula_data])
return mw
def __init__(self):
# speed up infixNotation considerably at the price of some cache memory
ParserElement.enablePackrat()
boolean = Keyword('True') | Keyword('False')
none = Keyword('None')
integer = Word(nums)
real = Combine(Word(nums) + "." + Word(nums))
string = (QuotedString('"', escChar='\\')
| QuotedString("'", escChar='\\'))
regex = QuotedString('/', escChar='\\')
identifier = Word(alphas, alphanums + '_')
dereference = infixNotation(identifier, [
(Literal('.'), 2, opAssoc.LEFT, EvalArith),
])
result = (Keyword('bad') | Keyword('fail') | Keyword('good')
| Keyword('ignore') | Keyword('unknown'))
rval = boolean | none | real | integer | string | regex | result | dereference
rvallist = Group(
Suppress('[') + Optional(delimitedList(rval)) + Suppress(']'))
rvalset = Group(
Suppress('{') + Optional(delimitedList(rval)) + Suppress('}'))
operand = rval | rvallist | rvalset
# parse actions replace the parsed tokens with an instantiated object
# which we can later call into for evaluation of its content
boolean.setParseAction(EvalBoolean)
none.setParseAction(EvalNone)
integer.setParseAction(EvalInteger)
real.setParseAction(EvalReal)
string.setParseAction(EvalString)
regex.setParseAction(EvalRegex)
identifier.setParseAction(EvalIdentifier)
result.setParseAction(EvalResult)
rvallist.setParseAction(EvalList)
rvalset.setParseAction(EvalSet)
identity_test = Keyword('is') + ~Keyword('not') | Combine(
Keyword('is') + Keyword('not'), adjacent=False, joinString=' ')
membership_test = Keyword('in') | Combine(
Keyword('not') + Keyword('in'), adjacent=False, joinString=' ')
comparison_op = oneOf('< <= > >= != == isdisjoint')
comparison = identity_test | membership_test | comparison_op
self.parser = infixNotation(operand, [
(Literal('**'), 2, opAssoc.LEFT, EvalPower),
(oneOf('+ - ~'), 1, opAssoc.RIGHT, EvalModifier),
(oneOf('* / // %'), 2, opAssoc.LEFT, EvalArith),
(oneOf('+ -'), 2, opAssoc.LEFT, EvalArith),
(oneOf('<< >>'), 2, opAssoc.LEFT, EvalArith),
(Literal('&'), 2, opAssoc.LEFT, EvalArith),
(Literal('^'), 2, opAssoc.LEFT, EvalArith),
(Literal('|'), 2, opAssoc.LEFT, EvalArith),
(comparison, 2, opAssoc.LEFT, EvalLogic),
(Keyword('not'), 1, opAssoc.RIGHT, EvalModifier),
(Keyword('and'), 2, opAssoc.LEFT, EvalLogic),
(Keyword('or'), 2, opAssoc.LEFT, EvalLogic),
(Keyword('->'), 2, opAssoc.LEFT, EvalArith),
])
def split_chemical_formula(formula):
def is_number(s):
try:
float(s)
return True
except ValueError:
return False
def replace_things(stringg, listt, replacement):
for x in listt:
stringg = stringg.replace(x, replacement)
return stringg
bad_chars = ["(", ")", "-", "."]
formula = replace_things(formula, bad_chars, "|")
if is_number(formula):
return [["", 0]]
if len(formula) == 0:
return [["", 0]]
# define some strings to use later, when describing valid lists
# of characters for chemical symbols and numbers
caps = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
lowers = caps.lower()
digits = "0123456789"
# Version 1
# Version 2 - Auto-convert integers, and add results names
def convertIntegers(tokens):
return int(tokens[0])
element = Word(caps, lowers)
integer = Word(digits).setParseAction(convertIntegers)
elementRef = Group(element("symbol") + Optional(integer, default=1)("qty"))
# pre-1.4.7, use this:
# elementRef = Group( element.setResultsName("symbol") + Optional( integer, default=1 ).setResultsName("qty") )
chemicalFormula = OneOrMore(elementRef)
# Version 3 - Compute partial molecular weight per element, simplifying
# summing
# No need to redefine grammar, just define parse action function, and
# attach to elementRef
def computeElementWeight(tokens):
element = tokens[0]
element["weight"] = atomicWeight[element.symbol] * element.qty
elementRef.setParseAction(computeElementWeight)
formulaData = chemicalFormula.parseString(formula)
mw = sum([element.weight for element in formulaData])
return formulaData
def get_fragment_grammar():
# Match header [mapping]
header = Suppress("[") + Suppress("mapping") + Suppress("]")
# There are three possible patterns for mapping entries:
# obj:symbol (scheme)
# obj (scheme)
# * (scheme)
obj = Fragment.ENTITY.setResultsName("object")
symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol")
scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName(
"scheme") + Suppress(")")
pattern1 = Group(obj + symbol + scheme)
pattern2 = Group(obj + scheme)
pattern3 = Group(
Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") +
scheme)
mapping_entry = pattern1 | pattern2 | pattern3
# To simplify parsing, classify groups of condition-mapping entry into two types: normal and default
# A normal grouping is one with a non-default condition. The default grouping is one which contains the
# default condition
mapping_entries = Group(
ZeroOrMore(mapping_entry)).setResultsName("mappings")
normal_condition = Suppress(":") + originalTextFor(
SDKConfig.get_expression_grammar())
default_condition = Optional(
Suppress(":") + Literal(Mapping.DEFAULT_CONDITION))
normal_group = Group(
normal_condition.setResultsName("condition") + mapping_entries)
default_group = Group(default_condition +
mapping_entries).setResultsName("default_group")
normal_groups = Group(
ZeroOrMore(normal_group)).setResultsName("normal_groups")
# Any mapping fragment definition can have zero or more normal group and only one default group as a last entry.
archive = Suppress("archive") + Suppress(
":") + Fragment.ENTITY.setResultsName("archive")
entries = Suppress("entries") + Suppress(":") + (
normal_groups + default_group).setResultsName("entries")
mapping = Group(header + archive + entries)
mapping.setParseAction(lambda t: Mapping(t[0].archive, t[0].entries))
mapping.ignore("#" + restOfLine)
return mapping
def condition_term():
'''
Grammar for condition term
'IF' <predicate> {'AND' <predicate>}* 'THEN'
'''
predicate = predicate_term()
condition = Group(
Suppress(IF_KEYWORD) + delimitedList(predicate, AND_KEYWORD) +
Suppress(THEN_KEYWORD))
condition.setParseAction(lambda t: t[0].asList())
return condition
def GetHeaders(self):
title = Group(
Optional(Literal("OPTIMAL")) + Literal("POWER FLOW REPORT"))
title.setParseAction(self.process_header_title)
version = slit("P S A T 2.1.") + integer.suppress()
author = slit("Author: Federico Milano, (c) 2002-2009")
email = slit("e-mail: [email protected]")
website = slit("website: http://www.uclm.es/area/gsee/Web/Federico")
filename = slit("File:") + restOfLine.suppress()
date = slit("Date:") + restOfLine.suppress()
return title + version + author + email + website + filename + date
class NodeParser:
def __init__(self):
self.num = Word(nums)
self.header = Regex(r"^UCLA.*")
self.comment = Regex(r"#.*")
self.bkid = Word(alphanums)
self.num_nodes = Literal("NumNodes") + Literal(":") + self.num(
"NumNodes")
self.num_terminals = Literal("NumTerminals") + Literal(":") + self.num(
"NumTerminals")
self.size = Group(self.num("width") + self.num("height"))
self.terminal = Optional(Literal("terminal"))
self.node = self.bkid("id") + self.size("size") + self.terminal
self.node_grammar = self.header + ZeroOrMore(self.comment) + self.num_nodes + self.num_terminals + \
OneOrMore(self.node)
self.coordinate = Group(self.num("x") + self.num("y"))
self.pl = self.bkid("id") + self.coordinate("coordinate") + \
Suppress(Literal(": N") + Optional(Literal(r"/FIXED")))
self.pl_grammar = self.header + ZeroOrMore(self.comment) + OneOrMore(
self.pl)
def compute_chip_size(self, benchmark):
benchmark_path = pathlib.Path(os.environ["BENCHMARK"])
node_file = benchmark_path / "ispd2005/{0}/{0}.nodes".format(benchmark)
pl_file = benchmark_path / "ispd2005/{0}/{0}.pl".format(benchmark)
print(node_file.as_posix())
print(pl_file.as_posix())
x_max = 0
y_max = 0
sizes = []
coordinates = []
self.size.setParseAction(
lambda tokens: sizes.append([tokens.width, tokens.height]))
self.coordinate.setParseAction(lambda tokens: coordinates.append(
(tokens.x, tokens.y)))
self.bkid.setParseAction(lambda tokens: print(tokens[0]))
self.node_grammar.parseFile(node_file.as_posix())
self.pl_grammar.parseFile(pl_file.as_posix())
for i in range(len(sizes)):
print(i)
if coordinates[i][0] + sizes[i][0] > x_max:
x_max = coordinates[i][0] + sizes[i][0]
if coordinates[i][1] + sizes[i][1] > y_max:
y_max = coordinates[i][1] + sizes[i][1]
return x_max, y_max
def main(s):
lpar = Literal('(').suppress()
rpar = Literal(')').suppress()
integer = Word(nums)
element = Word(alphas, exact=1)
formula = Forward()
term = Group((element | Group(lpar + formula + rpar)('subgroup')) +
Optional(integer, default=1)('mult'))
formula << OneOrMore(term)
integer.setParseAction(process_integer)
term.setParseAction(process_term)
formula.setParseAction(process_formula)
return formula.parseString(s)[0]
def get_fragment_grammar():
name = Fragment.IDENTIFIER
header = Suppress("[") + Suppress("sections") + Suppress(":") + name.setResultsName("name") + Suppress("]")
entry = Word(alphanums + "+" + ".")
entries = Suppress("entries") + Suppress(":") + Group(OneOrMore(entry)).setResultsName("entries")
sections = Group(header + entries)
sections.setParseAction(lambda t: Sections(t[0].name, t[0].entries))
sections.ignore("#" + restOfLine)
return sections
def __init__(self):
plus, minus, mult, div, mod = map(Literal, '+-*/%')
lpar = Literal('(')
rpar = Literal(')')
comma = Literal(',')
powop = Literal( '^' )
productop = mult | div
modop = Literal( '%' )
sumop = plus | minus
tupl = Forward()
number = Regex(r'[+-]?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?')
number.setParseAction(number_action)
ident = Word(alphas, alphanums+'_')
ident.setParseAction(ident_action)
funccall = ident + tupl
funccall.setParseAction(funccall_action)
atom = funccall | ident | number
atom.setParseAction(atom_action)
patom = lpar + atom + rpar
patom.setParseAction(patom_action)
powexpr = Forward()
powexpr << Group( atom + ZeroOrMore( ( powop + powexpr ) ) )
powexpr.setParseAction(powexpr_action)
modexpr = Forward()
modexpr << Group( powexpr + ZeroOrMore( ( modop + modexpr ) ) )
modexpr.setParseAction(modexpr_action)
product = Group( modexpr + ZeroOrMore( ( productop + modexpr ) ) )
product.setParseAction(product_action)
sumexpr = Group( product + Group( ZeroOrMore( sumop + product ) ) )
sumexpr.setParseAction(sum_action)
tupl << lpar + Optional(sumexpr + ZeroOrMore( comma + sumexpr ) ) + rpar
tupl.setParseAction(tupl_action)
expr = sumexpr | tupl
expr.setParseAction(expr_action)
self.bnf = expr
def temporal_condition_term():
'''
Grammar for condition term
'IF' <predicate> {'AND' <predicate>}* 'THEN'
'''
from grammar.keywords import IF_KEYWORD, AND_KEYWORD, \
THEN_KEYWORD
predicate = temporal_predicate_term()
condition = Group(
Suppress(IF_KEYWORD) + delimitedList(predicate, AND_KEYWORD) +
Suppress(THEN_KEYWORD))
condition.setParseAction(lambda t: t[0].asList())
return condition
class NodeParser:
def __init__(self):
self.num = Word(nums)
self.header = Regex(r"^UCLA.*")
self.comment = Regex(r"#.*")
self.bkid = Word(alphanums)
self.num_nodes = Literal("NumNodes") + Literal(":") + self.num("NumNodes")
self.num_terminals = Literal("NumTerminals") + Literal(":") + self.num("NumTerminals")
self.size = Group(self.num("width") + self.num("height"))
self.terminal = Optional(Literal("terminal"))
self.node = self.bkid("id") + self.size("size") + self.terminal
self.node_grammar = (
self.header + ZeroOrMore(self.comment) + self.num_nodes + self.num_terminals + OneOrMore(self.node)
)
self.coordinate = Group(self.num("x") + self.num("y"))
self.pl = (
self.bkid("id") + self.coordinate("coordinate") + Suppress(Literal(": N") + Optional(Literal(r"/FIXED")))
)
self.pl_grammar = self.header + ZeroOrMore(self.comment) + OneOrMore(self.pl)
def compute_chip_size(self, benchmark):
benchmark_path = pathlib.Path(os.environ["BENCHMARK"])
node_file = benchmark_path / "ispd2005/{0}/{0}.nodes".format(benchmark)
pl_file = benchmark_path / "ispd2005/{0}/{0}.pl".format(benchmark)
print(node_file.as_posix())
print(pl_file.as_posix())
x_max = 0
y_max = 0
sizes = []
coordinates = []
self.size.setParseAction(lambda tokens: sizes.append([tokens.width, tokens.height]))
self.coordinate.setParseAction(lambda tokens: coordinates.append((tokens.x, tokens.y)))
self.bkid.setParseAction(lambda tokens: print(tokens[0]))
self.node_grammar.parseFile(node_file.as_posix())
self.pl_grammar.parseFile(pl_file.as_posix())
for i in range(len(sizes)):
print(i)
if coordinates[i][0] + sizes[i][0] > x_max:
x_max = coordinates[i][0] + sizes[i][0]
if coordinates[i][1] + sizes[i][1] > y_max:
y_max = coordinates[i][1] + sizes[i][1]
return x_max, y_max
def define_operand(self):
"""
Return the syntax definition for an operand.
An operand can be a variable, a string, a number or a set. A set
is made of other operands, including other sets.
**This method shouldn't be overridden**. Instead, override the syntax
definitions for variables, strings and/or numbers.
If you want to customize the sets, check :meth:`T_SET_START`,
:meth:`T_SET_END` and :meth:`T_ELEMENT_SEPARATOR`.
"""
identifier = self.define_identifier()
operand = Forward()
# Defining the sets:
set_start = Suppress(self._grammar.get_token("set_start"))
set_end = Suppress(self._grammar.get_token("set_end"))
element_separator = self._grammar.get_token("element_separator")
elements = delimitedList(operand, delim=element_separator)
set_ = Group(set_start + Optional(elements) + set_end)
set_.setParseAction(self.make_set)
set_.setName("set")
# Defining the variables:
variable = identifier.copy()
variable.setName("variable")
variable.addParseAction(self.make_variable)
# Defining the functions:
function_name = identifier.setResultsName("function_name")
function_name.setName("function_name")
args_start = Suppress(self._grammar.get_token("arguments_start"))
args_end = Suppress(self._grammar.get_token("arguments_end"))
args_sep = self._grammar.get_token("arguments_separator")
arguments = Optional(Group(delimitedList(operand, delim=args_sep)),
default=())
arguments = arguments.setResultsName("arguments")
arguments.setParseAction(lambda tokens: tokens[0])
function = function_name + args_start + arguments + args_end
function.setName("function")
function.setParseAction(self.make_function)
operand << (function | variable | self.define_number() | \
self.define_string() | set_)
return operand
def define_operand(self):
"""
Return the syntax definition for an operand.
An operand can be a variable, a string, a number or a set. A set
is made of other operands, including other sets.
**This method shouldn't be overridden**. Instead, override the syntax
definitions for variables, strings and/or numbers.
If you want to customize the sets, check :meth:`T_SET_START`,
:meth:`T_SET_END` and :meth:`T_ELEMENT_SEPARATOR`.
"""
identifier = self.define_identifier()
operand = Forward()
# Defining the sets:
set_start = Suppress(self._grammar.get_token("set_start"))
set_end = Suppress(self._grammar.get_token("set_end"))
element_separator = self._grammar.get_token("element_separator")
elements = delimitedList(operand, delim=element_separator)
set_ = Group(set_start + Optional(elements) + set_end)
set_.setParseAction(self.make_set)
set_.setName("set")
# Defining the variables:
variable = identifier.copy()
variable.setName("variable")
variable.addParseAction(self.make_variable)
# Defining the functions:
function_name = identifier.setResultsName("function_name")
function_name.setName("function_name")
args_start = Suppress(self._grammar.get_token("arguments_start"))
args_end = Suppress(self._grammar.get_token("arguments_end"))
args_sep = self._grammar.get_token("arguments_separator")
arguments = Optional(Group(delimitedList(operand, delim=args_sep)),
default=())
arguments = arguments.setResultsName("arguments")
arguments.setParseAction(lambda tokens: tokens[0])
function = function_name + args_start + arguments + args_end
function.setName("function")
function.setParseAction(self.make_function)
operand << (function | variable | self.define_number() | \
self.define_string() | set_)
return operand
def __init__(self):
if PYPARSING:
category = Word( alphas + "_-*", alphanums + "_-*" )
operator = oneOf("and or ,")
neg_operator = "not"
elementRef = category
definition = elementRef + ZeroOrMore( operator + elementRef)
nestedformula = Group(Suppress(Optional(Literal("("))) + definition + Suppress(Optional(Literal(")"))))
neg_nestedformula = Optional(neg_operator) + nestedformula
self.finalformula = neg_nestedformula + ZeroOrMore( operator + neg_nestedformula)
elementRef.setParseAction(self.__compute_element)
neg_nestedformula.setParseAction(self.__compute_neg_formula)
nestedformula.setParseAction(self.__compute_formula)
self.finalformula.setParseAction(self.__myreduce)
def _build_filter_parser(field_names):
field = _build_field_expr(field_names)
negation = CaselessKeyword('not')
negation.setParseAction(lambda x: Negation(x[0]))
comparison_operator = Group(
Keyword('=') ^ Keyword('!=') ^ Keyword('>=') ^ Keyword('<=')
^ Keyword('<') ^ Keyword('>') ^ Keyword('lte') # match before lt
^ Keyword('lt') ^ Keyword('gte') # match before gt
^ Keyword('gt')
^ (Optional(negation) +
(CaselessKeyword('contains') ^ CaselessKeyword('icontains')
^ CaselessKeyword('startswith') ^ CaselessKeyword('istartswith')
^ CaselessKeyword('endswith') ^ CaselessKeyword('iendswith')
^ CaselessKeyword('eq'))))
comparison_operator.setParseAction(lambda x: Operator(x))
single_value_operator = Group(
CaselessKeyword('isnull')
^ (Optional(negation) + CaselessKeyword('isnull')))
single_value_operator.setParseAction(lambda x: Operator(x))
plusorminus = Literal('+') | Literal('-')
num_integer = Combine(Optional(plusorminus) + Word(nums))
num_integer.setParseAction(lambda x: Integer(x[0]))
num_float = Combine(Optional(plusorminus) + Word(nums) + '.' + Word(nums))
num_float.setParseAction(lambda x: Float(x[0]))
quoted_string = (QuotedString("'") ^ QuotedString('"'))
quoted_string.setParseAction(lambda x: String(x[0]))
boolean = Or([
CaselessKeyword(v) for v in BOOLEAN_TRUE_VALUES + BOOLEAN_FALSE_VALUES
])
boolean.setParseAction(lambda x: Boolean(x[0]))
value = (quoted_string ^ num_integer ^ boolean)
comparison = Group((field + comparison_operator + value)
^ (value + comparison_operator + field)
^ (field + comparison_operator + field)
^ (field + single_value_operator))
comparison.setParseAction(lambda x: Comparison(x))
invalid_comparison = Group(
(value + comparison_operator + value).setParseAction(lambda x: fail(
"Value may not be compared with values: {0}".format(' '.join(x)))))
logical_op = Group(CaselessKeyword("and") | CaselessKeyword("or"))
logical_op.setParseAction(lambda x: LogicalOp(x[0][0]))
statement = Optional(comparison | invalid_comparison) + ZeroOrMore(
logical_op + (comparison | invalid_comparison))
return statement
def indifferent_list():
'''
Grammar for list of indifferent attributes
'[' <attribute> (, <attribute>)* ']'
'(' <attribute> (, <attribute>)* ')'
'''
att_term = identifier_token()
indiff_par = Suppress(LEFT_PAR) + \
delimitedList(att_term, COMMA) + \
Suppress(RIGHT_PAR)
indiff_bra = Suppress(LEFT_BRA) + \
delimitedList(att_term, COMMA) + \
Suppress(RIGHT_BRA)
indiff_list = Group(indiff_par | indiff_bra)
indiff_list.setParseAction(lambda t: t[0].asList())
return indiff_list
def get_fragment_grammar():
name = Fragment.IDENTIFIER
header = Suppress("[") + Suppress("scheme") + Suppress(":") + name.setResultsName("name") + Suppress("]")
# Scheme entry in the form 'sections -> target'
sections = Fragment.IDENTIFIER
target = Fragment.IDENTIFIER
entry = Group(sections.setResultsName("sections") + Suppress("->") + target.setResultsName("target"))
entries = Suppress("entries") + Suppress(":") + Group(OneOrMore(entry)).setResultsName("entries")
scheme = Group(header + entries)
scheme.setParseAction(lambda t: Scheme(t[0].name, t[0].entries))
scheme.ignore("#" + restOfLine)
return scheme
def _initialize(self):
def keywords(klass):
kws = sorted(klass.keyword_aliases.keys())
return MatchFirst([Keyword(kw) for kw in kws])
def infix(klass):
kws = sorted(klass.keyword_aliases.keys())
return [(kw, klass.n_terms, getattr(opAssoc, klass.assoc), klass)
for kw in kws]
# literals include words made of alphanumerics, numbers,
# or quoted strings but we exclude any of the logical
# operands (e.g. 'or') from being parsed literals
literal = (
~(keywords(BinaryInfixOperand) | keywords(UnaryInfixOperand)) +
(Word(NUMS) | quotedString | Word(alphas, alphanums)))
literal.setParseAction(Literal)
# These are the other 'root' expressions,
# the selection keywords (resname, resid, mass, etc)
selection_keyword = keywords(SelectionKeyword)
selection_keyword.setParseAction(SelectionKeyword)
base_expression = MatchFirst([selection_keyword, literal])
# range condition matches expressions such as 'mass 1 to 20'
range_condition = Group(selection_keyword + literal + Keyword('to') +
literal)
range_condition.setParseAction(RangeCondition)
# matches expression such as `resname GLU ASP ARG` and also
# handles implicit equality `resname ALA`
in_list_condition = Group(selection_keyword + OneOrMore(literal))
in_list_condition.setParseAction(InListCondition)
expression = range_condition | in_list_condition | base_expression
logical_expr = infixNotation(
expression,
infix(UnaryInfixOperand) + infix(BinaryInfixOperand) +
infix(RegexInfixOperand))
self.expression = logical_expr
self.is_initialized = True
self.transformer = _RewriteNames()
def get_fragment_grammar():
# Match header [mapping]
header = Suppress("[") + Suppress("mapping") + Suppress("]")
# There are three possible patterns for mapping entries:
# obj:symbol (scheme)
# obj (scheme)
# * (scheme)
obj = Fragment.ENTITY.setResultsName("object")
symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol")
scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName("scheme") + Suppress(")")
pattern1 = Group(obj + symbol + scheme)
pattern2 = Group(obj + scheme)
pattern3 = Group(Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") + scheme)
mapping_entry = pattern1 | pattern2 | pattern3
# To simplify parsing, classify groups of condition-mapping entry into two types: normal and default
# A normal grouping is one with a non-default condition. The default grouping is one which contains the
# default condition
mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName("mappings")
normal_condition = Suppress(":") + originalTextFor(SDKConfig.get_expression_grammar())
default_condition = Optional(Suppress(":") + Literal(Mapping.DEFAULT_CONDITION))
normal_group = Group(normal_condition.setResultsName("condition") + mapping_entries)
default_group = Group(default_condition + mapping_entries).setResultsName("default_group")
normal_groups = Group(ZeroOrMore(normal_group)).setResultsName("normal_groups")
# Any mapping fragment definition can have zero or more normal group and only one default group as a last entry.
archive = Suppress("archive") + Suppress(":") + Fragment.ENTITY.setResultsName("archive")
entries = Suppress("entries") + Suppress(":") + (normal_groups + default_group).setResultsName("entries")
mapping = Group(header + archive + entries)
mapping.setParseAction(lambda t: Mapping(t[0].archive, t[0].entries))
mapping.ignore("#" + restOfLine)
return mapping
def _getPattern(self):
arith_expr = Forward()
comp_expr = Forward()
logic_expr = Forward()
LPAR, RPAR, SEMI = map(Suppress, "();")
identifier = Word(alphas+"_", alphanums+"_")
multop = oneOf('* /')
plusop = oneOf('+ -')
expop = Literal( "^" )
compop = oneOf('> < >= <= != ==')
andop = Literal("AND")
orop = Literal("OR")
current_value = Literal( "." )
assign = Literal( "=" )
# notop = Literal('NOT')
function = oneOf(' '.join(self.FUNCTIONS))
function_call = Group(function.setResultsName('fn') + LPAR + Optional(delimitedList(arith_expr)) + RPAR)
aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}')
single_column = QuotedString(quoteChar='[', endQuoteChar=']')
integer = Regex(r"-?\d+")
real = Regex(r"-?\d+\.\d*")
# quotedString enables strings without quotes to pass
operand = \
function_call.setParseAction(self.__evalFunction) | \
aggregate_column.setParseAction(self.__evalAggregateColumn) | \
single_column.setParseAction(self.__evalSingleColumn) | \
((real | integer).setParseAction(self.__evalConstant)) | \
quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \
current_value.setParseAction(self.__evalCurrentValue) | \
identifier.setParseAction(self.__evalString)
arith_expr << operatorPrecedence(operand,
[
(expop, 2, opAssoc.LEFT, self.__expOp),
(multop, 2, opAssoc.LEFT, self.__multOp),
(plusop, 2, opAssoc.LEFT, self.__addOp),
])
# comp_expr = Group(arith_expr + compop + arith_expr)
comp_expr << operatorPrecedence(arith_expr,
[
(compop, 2, opAssoc.LEFT, self.__evalComparisonOp),
])
logic_expr << operatorPrecedence(comp_expr,
[
(andop, 2, opAssoc.LEFT, self.__evalLogicOp),
(orop, 2, opAssoc.LEFT, self.__evalLogicOp)
])
pattern = logic_expr + StringEnd()
return pattern
def indifferent_list():
'''
Grammar for list of indifferent attributes
'[' <attribute> (, <attribute>)* ']'
'(' <attribute> (, <attribute>)* ')'
'''
from grammar.symbols import LEFT_BRA, RIGHT_BRA, LEFT_PAR, \
RIGHT_PAR, COMMA
from grammar.basic import attribute_term
att_term = attribute_term()
indiff_par = Suppress(LEFT_PAR) + \
delimitedList(att_term, COMMA) + \
Suppress(RIGHT_PAR)
indiff_bra = Suppress(LEFT_BRA) + \
delimitedList(att_term, COMMA) + \
Suppress(RIGHT_BRA)
indiff_list = Group(indiff_par | indiff_bra)
indiff_list.setParseAction(lambda t: t[0].asList())
return indiff_list
def numeric_literal() -> Token:
numeral = Combine(
Word(nums) + ZeroOrMore(Optional(Word("_")) + Word(nums)))
numeral.setParseAction(lambda t: t[0].replace("_", ""))
decimal_literal = Group(numeral)
decimal_literal.setParseAction(lambda t: (int(t[0][0]), 0))
extended_digit = Word(nums + "ABCDEF")
based_numeral = Combine(extended_digit +
ZeroOrMore(Optional("_") + extended_digit))
based_numeral.setParseAction(lambda t: t[0].replace("_", ""))
based_literal = numeral + Literal("#") - based_numeral - Literal("#")
based_literal.setParseAction(lambda t: (int(t[2], int(t[0])), int(t[0])))
num_literal = based_literal | decimal_literal
num_literal.setName("Number")
return locatedExpr(num_literal).setParseAction(lambda s, l, t: Number(
t[0][1][0], t[0][1][1], parser_location(t[0][0], t[0][2], s)))
def _getPattern(self):
arith_expr = Forward()
comp_expr = Forward()
logic_expr = Forward()
LPAR, RPAR, SEMI = map(Suppress, "();")
identifier = Word(alphas + "_", alphanums + "_")
multop = oneOf('* /')
plusop = oneOf('+ -')
expop = Literal("^")
compop = oneOf('> < >= <= != ==')
andop = Literal("AND")
orop = Literal("OR")
current_value = Literal(".")
assign = Literal("=")
# notop = Literal('NOT')
function = oneOf(' '.join(self.FUNCTIONS))
function_call = Group(
function.setResultsName('fn') + LPAR +
Optional(delimitedList(arith_expr)) + RPAR)
aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}')
single_column = QuotedString(quoteChar='[', endQuoteChar=']')
integer = Regex(r"-?\d+")
real = Regex(r"-?\d+\.\d*")
# quotedString enables strings without quotes to pass
operand = \
function_call.setParseAction(self.__evalFunction) | \
aggregate_column.setParseAction(self.__evalAggregateColumn) | \
single_column.setParseAction(self.__evalSingleColumn) | \
((real | integer).setParseAction(self.__evalConstant)) | \
quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \
current_value.setParseAction(self.__evalCurrentValue) | \
identifier.setParseAction(self.__evalString)
arith_expr << operatorPrecedence(operand, [
(expop, 2, opAssoc.LEFT, self.__expOp),
(multop, 2, opAssoc.LEFT, self.__multOp),
(plusop, 2, opAssoc.LEFT, self.__addOp),
])
# comp_expr = Group(arith_expr + compop + arith_expr)
comp_expr << operatorPrecedence(arith_expr, [
(compop, 2, opAssoc.LEFT, self.__evalComparisonOp),
])
logic_expr << operatorPrecedence(
comp_expr, [(andop, 2, opAssoc.LEFT, self.__evalLogicOp),
(orop, 2, opAssoc.LEFT, self.__evalLogicOp)])
pattern = logic_expr + StringEnd()
return pattern
def __init__(self, max_=60):
# define the grammar structure
digits = "0123456789"
star = Literal('*')
number = Word(digits) | Word(alphas)
steps = number
range_ = number + Optional(Literal('-') + number)
numspec = star | range_
expr = Group(numspec) + Optional(Literal('/') + steps)
extra_groups = ZeroOrMore(Literal(',') + expr)
groups = expr + extra_groups + StringEnd()
# define parse actions
star.setParseAction(self._expand_star)
number.setParseAction(self._expand_number)
range_.setParseAction(self._expand_range)
expr.setParseAction(self._filter_steps)
extra_groups.setParseAction(self._ignore_comma)
groups.setParseAction(self._join_to_set)
self.max_ = max_
self.parser = groups
def __init__(self, max_=60):
# define the grammar structure
digits = "0123456789"
star = Literal('*')
number = Word(digits) | Word(alphas)
steps = number
range_ = number + Optional(Literal('-') + number)
numspec = star | range_
expr = Group(numspec) + Optional(Literal('/') + steps)
extra_groups = ZeroOrMore(Literal(',') + expr)
groups = expr + extra_groups + StringEnd()
# define parse actions
star.setParseAction(self._expand_star)
number.setParseAction(self._expand_number)
range_.setParseAction(self._expand_range)
expr.setParseAction(self._filter_steps)
extra_groups.setParseAction(self._ignore_comma)
groups.setParseAction(self._join_to_set)
self.max_ = max_
self.parser = groups
def arithmetic_term():
'''
Grammar for arithmetic expressions
'''
from grammar.keywords import CURRENT_KEYWORD
from grammar.symbols import MULTI_OP_SET, PLUS_OP_SET
from grammar.basic import attribute_term, value_term
from grammar.symbols import LEFT_PAR, RIGHT_PAR,\
AGGREGATE_FUNCTIONS_SET
# Arithmetic operators
multiply_op = oneOf(list(MULTI_OP_SET))
addition_op = oneOf(list(PLUS_OP_SET))
operand_token = CURRENT_KEYWORD | attribute_term() | value_term()
# Arithmetic expression
precedence_list = [(multiply_op, 2, opAssoc.LEFT),
(addition_op, 2, opAssoc.LEFT)]
expression_t = operatorPrecedence(operand_token, precedence_list)
# Aggregate function
aggregate_function = oneOf(list(AGGREGATE_FUNCTIONS_SET), caseless=True)
arithmetic_t = Group(aggregate_function + Suppress(LEFT_PAR) +
expression_t + Suppress(RIGHT_PAR)) | \
expression_t
arithmetic_t.setParseAction(lambda t: t.asList())
return arithmetic_t
def parse_create_view(string):
"""Parse the create view sql query and return metadata
Args:
string(str): Input sql string that should be parsed
Returns:
view_data(dict): view_data dictionary for instantiating a view object
"""
string = rreplace(string, ')', ' )')
end = ')' + StringEnd()
select = Group(ZeroOrMore(~end + Word(printables)))
parser = _create + replace_check.setResultsName('replace') + _view
parser += _db_name.setResultsName('view_name') + _as + '('
parser += select.setParseAction(merge).setResultsName('select_statement')
parser += end
# Parse the base table definitions
view_data = to_dict(parser.parseString(string))
return view_data
def parse_create_view(string):
"""Parse the create view sql query and return metadata
Args:
string(str): Input sql string that should be parsed
Returns:
view_data(dict): view_data dictionary for instantiating a view object
"""
string = rreplace(string, ')', ' )')
end = ')' + StringEnd()
select = Group(ZeroOrMore(~end + Word(printables)))
parser = _create + replace_check.setResultsName('replace') + _view
parser += _db_name.setResultsName('view_name') + _as + '('
parser += select.setParseAction(merge).setResultsName('select_statement')
parser += end
# Parse the base table definitions
view_data = to_dict(parser.parseString(string))
return view_data
(VerbPath | VerbSimple) + ObjectList))
# [82] PropertyListPath ::= Optional(PropertyListPathNotEmpty)
PropertyListPath = Optional(PropertyListPathNotEmpty)
# [77] PropertyListNotEmpty ::= Verb ObjectList ( ';' ( Verb ObjectList )? )*
PropertyListNotEmpty = Verb + ObjectList + ZeroOrMore(
';' + Optional(Verb + ObjectList))
# [76] PropertyList ::= Optional(PropertyListNotEmpty)
PropertyList = Optional(PropertyListNotEmpty)
# [99] BlankNodePropertyList ::= '[' PropertyListNotEmpty ']'
BlankNodePropertyList = Group(
Suppress('[') + PropertyListNotEmpty + Suppress(']'))
BlankNodePropertyList.setParseAction(expandBNodeTriples)
# [101] BlankNodePropertyListPath ::= '[' PropertyListPathNotEmpty ']'
BlankNodePropertyListPath = Group(
Suppress('[') + PropertyListPathNotEmpty + Suppress(']'))
BlankNodePropertyListPath.setParseAction(expandBNodeTriples)
# [98] TriplesNode ::= Collection | BlankNodePropertyList
TriplesNode << (Collection | BlankNodePropertyList)
# [100] TriplesNodePath ::= CollectionPath | BlankNodePropertyListPath
TriplesNodePath << (CollectionPath | BlankNodePropertyListPath)
# [75] TriplesSameSubject ::= VarOrTerm PropertyListNotEmpty | TriplesNode PropertyList
TriplesSameSubject = VarOrTerm + PropertyListNotEmpty | TriplesNode + \
PropertyList
def parse_algebra(self):
"""
Parse an algebraic expression into a tree.
Store a `pyparsing.ParseResult` in `self.tree` with proper groupings to
reflect parenthesis and order of operations. Leave all operators in the
tree and do not parse any strings of numbers into their float versions.
Adding the groups and result names makes the `repr()` of the result
really gross. For debugging, use something like
print OBJ.tree.asXML()
"""
# 0.33 or 7 or .34 or 16.
number_part = Word(nums)
inner_number = (number_part + Optional("." + Optional(number_part))) | ("." + number_part)
# pyparsing allows spaces between tokens--`Combine` prevents that.
inner_number = Combine(inner_number)
# SI suffixes and percent.
number_suffix = MatchFirst(Literal(k) for k in SUFFIXES.keys())
# 0.33k or 17
plus_minus = Literal('+') | Literal('-')
number = Group(
Optional(plus_minus) +
inner_number +
Optional(CaselessLiteral("E") + Optional(plus_minus) + number_part) +
Optional(number_suffix)
)
number = number("number")
# Predefine recursive variables.
expr = Forward()
# Handle variables passed in. They must start with a letter
# and may contain numbers and underscores afterward.
inner_varname = Combine(Word(alphas, alphanums + "_") + ZeroOrMore("'"))
# Alternative variable name in tensor format
# Tensor name must start with a letter, continue with alphanums
# Indices may be alphanumeric
# e.g., U_{ijk}^{123}
upper_indices = Literal("^{") + Word(alphanums) + Literal("}")
lower_indices = Literal("_{") + Word(alphanums) + Literal("}")
tensor_lower = Combine(Word(alphas, alphanums) + lower_indices + ZeroOrMore("'"))
tensor_mixed = Combine(Word(alphas, alphanums) + Optional(lower_indices) + upper_indices + ZeroOrMore("'"))
# Test for mixed tensor first, then lower tensor alone, then generic variable name
varname = Group(tensor_mixed | tensor_lower | inner_varname)("variable")
varname.setParseAction(self.variable_parse_action)
# Same thing for functions.
function = Group(inner_varname + Suppress("(") + expr + Suppress(")"))("function")
function.setParseAction(self.function_parse_action)
atom = number | function | varname | "(" + expr + ")"
atom = Group(atom)("atom")
# Do the following in the correct order to preserve order of operation.
pow_term = atom + ZeroOrMore("^" + atom)
pow_term = Group(pow_term)("power")
par_term = pow_term + ZeroOrMore('||' + pow_term) # 5k || 4k
par_term = Group(par_term)("parallel")
prod_term = par_term + ZeroOrMore((Literal('*') | Literal('/')) + par_term) # 7 * 5 / 4
prod_term = Group(prod_term)("product")
sum_term = Optional(plus_minus) + prod_term + ZeroOrMore(plus_minus + prod_term) # -5 + 4 - 3
sum_term = Group(sum_term)("sum")
# Finish the recursion.
expr << sum_term # pylint: disable=pointless-statement
self.tree = (expr + stringEnd).parseString(self.math_expr)[0]
def _parse_study_search_string(self, searchstr,
only_with_processed_data=False):
"""parses string into SQL query for study search
Parameters
----------
searchstr : str
The string to parse
only_with_processed_data : bool
Whether or not to return studies with processed data.
Returns
-------
study_sql : str
SQL query for selecting studies with the required metadata columns
sample_sql : str
SQL query for each study to get the sample ids that mach the query
meta_headers : list
metadata categories in the query string in alphabetical order
Notes
-----
All searches are case-sensitive
References
----------
.. [1] McGuire P (2007) Getting started with pyparsing.
"""
# build the parse grammar
category = Word(alphas + nums + "_")
seperator = oneOf("> < = >= <= !=") | CaselessLiteral("includes") | \
CaselessLiteral("startswith")
value = Word(alphas + nums + "_" + ":" + ".") | \
dblQuotedString().setParseAction(removeQuotes)
criterion = Group(category + seperator + value)
criterion.setParseAction(SearchTerm)
and_ = CaselessLiteral("and")
or_ = CaselessLiteral("or")
not_ = CaselessLiteral("not")
optional_seps = Optional(and_ | or_ | not_)
# create the grammar for parsing operators AND, OR, NOT
search_expr = operatorPrecedence(
criterion, [
(not_, 1, opAssoc.RIGHT, SearchNot),
(and_, 2, opAssoc.LEFT, SearchAnd),
(or_, 2, opAssoc.LEFT, SearchOr)])
# parse the search string to get out the SQL WHERE formatted query
eval_stack = (search_expr + stringEnd).parseString(searchstr)[0]
sql_where = eval_stack.generate_sql()
# this lookup will be used to select only studies with columns
# of the correct type
type_lookup = {int: 'integer', float: 'float8', str: 'varchar'}
# parse out all metadata headers we need to have in a study, and
# their corresponding types
all_headers = [c[0][0].term[0] for c in
(criterion + optional_seps).scanString(searchstr)]
meta_headers = set(all_headers)
all_types = [c[0][0].term[2] for c in
(criterion + optional_seps).scanString(searchstr)]
all_types = [type_lookup[type(typecast_string(s))] for s in all_types]
# sort headers and types so they return in same order every time.
# Should be a relatively short list so very quick
# argsort implementation taken from
# http://stackoverflow.com/questions/3382352/
# equivalent-of-numpy-argsort-in-basic-python
sort_order = sorted(range(len(all_headers)),
key=all_headers.__getitem__)
all_types = [all_types[x] for x in sort_order]
all_headers.sort()
# At this point it is possible that a metadata header has been
# reference more than once in the query. If the types agree, then we
# do not need to do anything. If the types do not agree (specifically,
# if it appears to be numerical in one case and string in another),
# then we need to give varchar the precedence.
meta_header_type_lookup = dict()
for header, header_type in zip(all_headers, all_types):
if header not in meta_header_type_lookup:
meta_header_type_lookup[header] = header_type
else:
if header_type == 'varchar' or \
meta_header_type_lookup[header] == 'varchar':
meta_header_type_lookup[header] = 'varchar'
# create the study finding SQL
# remove metadata headers that are in required_sample_info table
meta_headers = meta_headers.difference(self.required_cols).difference(
self.study_cols)
# get all study ids that contain all metadata categories searched for
sql = []
if meta_headers:
# have study-specific metadata, so need to find specific studies
for meta in meta_headers:
if meta_header_type_lookup[meta] in ('integer', 'float8'):
allowable_types = "('integer', 'float8')"
else:
allowable_types = "('varchar')"
sql.append("SELECT study_id FROM qiita.study_sample_columns "
"WHERE lower(column_name) = lower('%s') and "
"column_type in %s" %
(scrub_data(meta), allowable_types))
else:
# no study-specific metadata, so need all studies
sql.append("SELECT study_id FROM qiita.study_sample_columns")
# combine the query
if only_with_processed_data:
sql.append('SELECT study_id FROM qiita.study_processed_data')
study_sql = ' INTERSECT '.join(sql)
# create the sample finding SQL, getting both sample id and values
# build the sql formatted list of metadata headers
header_info = []
for meta in meta_header_type_lookup:
if meta in self.required_cols:
header_info.append("r.%s" % meta)
elif meta in self.study_cols:
header_info.append("st.%s" % meta)
else:
header_info.append("sa.%s" % meta)
# build the SQL query
sample_sql = ("SELECT r.sample_id,%s FROM qiita.required_sample_info "
"r JOIN qiita.sample_{0} sa ON sa.sample_id = "
"r.sample_id JOIN qiita.study st ON st.study_id = "
"r.study_id WHERE %s" %
(','.join(header_info), sql_where))
return study_sql, sample_sql, meta_header_type_lookup.keys()
ternaryop = ('?', ':')
current_scope_operand.setParseAction(EvalCurrentScope)
variable_operand.setParseAction(EvalVariable)
explicit_variable_operand.setParseAction(EvalExplicitVariable)
integer_operand.setParseAction(EvalInteger)
real_operand.setParseAction(EvalReal)
string_operand.setParseAction(EvalString)
constant.setParseAction(EvalConstant)
regexp.setParseAction(EvalRegExp)
timespan.setParseAction(EvalTimespan)
modifier = Regex(r'([a-zA-Z][a-zA-Z0-9_]*)\:')
simple_list_operand = Group(delimitedList(expr))
simple_list_operand.setParseAction(EvalSimpleList)
list_operand = (Suppress('[') + delimitedList(expr) + Suppress(']'))
list_operand.setParseAction(EvalList)
empty_list_operand = Literal('[]')
empty_list_operand.setParseAction(EvalEmptyList)
dict_item = Group(expr + Suppress(Literal(':')) + expr)
dict_operand = Group(Suppress('{') + delimitedList(dict_item) + Suppress('}'))
dict_operand.setParseAction(EvalDict)
empty_dict_operand = Literal('{}')
empty_dict_operand.setParseAction(EvalEmptyDict)
key_pair = Group(Regex(r'([a-zA-Z0-9_]+)') + Suppress(Literal('=') + WordEnd('=!+-*/')) + expr)
>>> print join(['a', 'long', 'long', 'road'])
a~long long~road
>>> print join(['very', 'long', 'phrase'])
very long~phrase
"""
if len(words) <= 2:
return tie.join(words)
else:
return (words[0] + tie_or_space(words[0], tie, space) +
space.join(words[1:-1]) +
tie + words[-1])
def format(name, format):
return NameFormat(format).format(name)
lbrace = Literal('{')
rbrace = Literal('}')
format_chars = Word(alphas)
braced_string = Forward()
braced_string << Combine(lbrace + ZeroOrMore(CharsNotIn('{}')| braced_string) + rbrace)
verbatim = Combine(ZeroOrMore(CharsNotIn(alphas + '{}') | braced_string))
delimiter = braced_string.copy().setParseAction(removeQuotes)
group = Group(Suppress(lbrace) + verbatim + format_chars + Optional(delimiter, None) +
verbatim + Suppress(rbrace))
group.setParseAction(lambda toks: NamePart(toks))
toplevel_text = CharsNotIn('{}').setParseAction(lambda toks: Text(toks))
name_format_grammar = ZeroOrMore(toplevel_text | group) + StringEnd()
name_format_grammar.leaveWhitespace()
ft = float(t)
it = int(ft)
return it if it == ft else ft
nreal.setParseAction(make_int)
def multiply_contents(tokens):
t = tokens[0]
if t.subgroup:
mult = t.mult
sg = next(subGroupCount) # is needed to calculate wt%
for term in t.subgroup:
term[1] *= mult
term.append(sg)
return t.subgroup
term.setParseAction(multiply_contents)
def sum_by_element(tokens):
nt = tokens
dopes = []
for t in nt:
if isinstance(t[1], complex):
sg = t[2] if len(t) > 2 else next(subGroupCount)
dopes.append([t, sg])
if dopes:
matrix = [[t[0], t[1]] for t in nt if isinstance(t[1], (float, int))]
matrixMass = sum([read_atomic_data(e[0])*e[1] for e in matrix])
dopeGroups = []
for e in dopes:
if not (e[1] in dopeGroups):
def graph_definition():
global graphparser
if not graphparser:
# punctuation
colon = Literal(":")
lbrace = Literal("{")
rbrace = Literal("}")
lbrack = Literal("[")
rbrack = Literal("]")
lparen = Literal("(")
rparen = Literal(")")
equals = Literal("=")
comma = Literal(",")
dot = Literal(".")
slash = Literal("/")
bslash = Literal("\\")
star = Literal("*")
semi = Literal(";")
at = Literal("@")
minus = Literal("-")
# keywords
strict_ = Literal("strict")
graph_ = Literal("graph")
digraph_ = Literal("digraph")
subgraph_ = Literal("subgraph")
node_ = Literal("node")
edge_ = Literal("edge")
# token definitions
identifier = Word(alphanums + "_" ).setName("identifier")
double_quoted_string = dblQuotedString
alphastring_ = OneOrMore(CharsNotIn(_noncomma))
ID = (identifier | double_quoted_string.setParseAction(strip_quotes) |\
alphastring_).setName("ID")
html_text = Combine(Literal("<<") + OneOrMore(CharsNotIn(",]")))
float_number = Combine(Optional(minus) + \
OneOrMore(Word(nums + "."))).setName("float_number")
righthand_id = (float_number | ID | html_text).setName("righthand_id")
port_angle = (at + ID).setName("port_angle")
port_location = (Group(colon + ID) | \
Group(colon + lparen + ID + comma + ID + rparen)).setName("port_location")
port = (Group(port_location + Optional(port_angle)) | \
Group(port_angle + Optional(port_location))).setName("port")
node_id = (ID + Optional(port))
a_list = OneOrMore(ID + Optional(equals.suppress() + righthand_id) + \
Optional(comma.suppress())).setName("a_list")
attr_list = OneOrMore(lbrack.suppress() + Optional(a_list) + \
rbrack.suppress()).setName("attr_list")
attr_stmt = (Group(graph_ | node_ | edge_) + attr_list).setName("attr_stmt")
edgeop = (Literal("--") | Literal("->")).setName("edgeop")
stmt_list = Forward()
graph_stmt = Group(lbrace.suppress() + Optional(stmt_list) + \
rbrace.suppress()).setName("graph_stmt")
subgraph = (Group(Optional(subgraph_ + Optional(ID)) + graph_stmt) | \
Group(subgraph_ + ID)).setName("subgraph")
edgeRHS = OneOrMore(edgeop + Group(node_id | subgraph))
edge_stmt = Group(node_id | subgraph) + edgeRHS + Optional(attr_list)
node_stmt = (node_id + Optional(attr_list) + Optional(semi.suppress())).setName("node_stmt")
assignment = (ID + equals.suppress() + righthand_id).setName("assignment")
stmt = (assignment | edge_stmt | attr_stmt | subgraph | node_stmt).setName("stmt")
stmt_list << OneOrMore(stmt + Optional(semi.suppress()))
graphparser = (Optional(strict_) + Group((graph_ | digraph_)) + \
Optional(ID) + graph_stmt).setResultsName("graph")
singleLineComment = "//" + restOfLine
# actions
graphparser.ignore(singleLineComment)
graphparser.ignore(cStyleComment)
assignment.setParseAction(push_attr_list)
a_list.setParseAction(push_attr_list)
edge_stmt.setParseAction(push_edge_stmt)
node_stmt.setParseAction(push_node_stmt)
attr_stmt.setParseAction(push_default_stmt)
subgraph.setParseAction(push_subgraph_stmt)
graph_stmt.setParseAction(push_graph_stmt)
graphparser.setParseAction(push_top_graph_stmt)
return graphparser
L("~=") |
L(">") |
L("<")
)
MARKER_OP = VERSION_CMP | L("not in") | L("in")
MARKER_VALUE = QuotedString("'") | QuotedString('"')
MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0]))
BOOLOP = L("and") | L("or")
MARKER_VAR = VARIABLE | MARKER_VALUE
MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR)
MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0]))
LPAREN = L("(").suppress()
RPAREN = L(")").suppress()
MARKER_EXPR = Forward()
MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN)
MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR)
MARKER = stringStart + MARKER_EXPR + stringEnd
def _coerce_parse_result(results):
if isinstance(results, ParseResults):
return [_coerce_parse_result(i) for i in results]
else:
(VerbPath | VerbSimple) + ObjectListPath + ZeroOrMore(";" + Optional((VerbPath | VerbSimple) + ObjectList))
)
# [82] PropertyListPath ::= Optional(PropertyListPathNotEmpty)
PropertyListPath = Optional(PropertyListPathNotEmpty)
# [77] PropertyListNotEmpty ::= Verb ObjectList ( ';' ( Verb ObjectList )? )*
PropertyListNotEmpty = Verb + ObjectList + ZeroOrMore(";" + Optional(Verb + ObjectList))
# [76] PropertyList ::= Optional(PropertyListNotEmpty)
PropertyList = Optional(PropertyListNotEmpty)
# [99] BlankNodePropertyList ::= '[' PropertyListNotEmpty ']'
BlankNodePropertyList = Group(Suppress("[") + PropertyListNotEmpty + Suppress("]"))
BlankNodePropertyList.setParseAction(expandBNodeTriples)
# [101] BlankNodePropertyListPath ::= '[' PropertyListPathNotEmpty ']'
BlankNodePropertyListPath = Group(Suppress("[") + PropertyListPathNotEmpty + Suppress("]"))
BlankNodePropertyListPath.setParseAction(expandBNodeTriples)
# [98] TriplesNode ::= Collection | BlankNodePropertyList
TriplesNode << (Collection | BlankNodePropertyList)
# [100] TriplesNodePath ::= CollectionPath | BlankNodePropertyListPath
TriplesNodePath << (CollectionPath | BlankNodePropertyListPath)
# [75] TriplesSameSubject ::= VarOrTerm PropertyListNotEmpty | TriplesNode PropertyList
TriplesSameSubject = VarOrTerm + PropertyListNotEmpty | TriplesNode + PropertyList
TriplesSameSubject.setParseAction(expandTriples)
ident = Word(alphas, alphanums + "_$" ) ^ QuotedString('"') ^ QuotedString("`")
ident.setParseAction(lambda toks: ['"%s"' % toks[0]])
string = QuotedString("'",multiline=True)
string.setParseAction(lambda toks: ["'%s'" % toks[0]])
columnName = delimitedList( ident, ".",combine=True)
tableName = delimitedList( ident, ".",combine=True)
dataType = Word(alphas) + Combine(Optional(Literal("(") + (Word(nums) ^ delimitedList(string,combine=True)) + Literal(")"))) + ZeroOrMore(nnToken ^ autoincrementToken ^ (defaultToken + (string ^ nullToken)) ^ unsignedToken.suppress() )
dataType.setParseAction(convert_datatypes)
columnDescription = Group(ident + dataType)
keyDescription = Optional(primaryToken ^ uniqueToken) + keyToken + Optional(ident) + Literal("(") + delimitedList(ident + Optional(Literal("(") + Word(nums) + Literal(")"))) + Literal(")")
createTableStmt = Group(createToken + tableToken + ifneToken + ident + Literal("(")) + delimitedList(columnDescription ^ keyDescription.suppress()) + Group(Literal(")")) + Optional(autoincrementToken + Literal("=") + Word(nums)).suppress()
createTableStmt.setParseAction(rebuild_createtable)
createDataBaseStmt = Group(createToken + databaseToken + ident + dcsToken + Word(alphanums)+ collateToken + ident)
useStmt = Group(useToken + ident)
comment = LineStart() + CharsNotIn("\n") + LineEnd()
value = Group(Literal("(") + delimitedList(Word(nums) ^ string) + Literal(")"))
insertPrefix = Group(insertToken + intoToken + ident + Literal("(") + delimitedList(ident) + Literal(")") + valuesToken)
insertStmt = insertPrefix + delimitedList(value)
insertStmt.setParseAction(rebuild_insert)
def get_grammar(self):
"""
Defines our grammar for mathematical expressions.
Possibly helpful:
- BNF form of context-free grammar https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form
- Some pyparsing docs http://infohost.nmt.edu/~shipman/soft/pyparsing/web/index.html
"""
# Define + and -
plus = Literal("+")
# Also accept unicode emdash
emdash = Literal("\u2014")
emdash.setParseAction(lambda: "-")
minus = Literal("-") | emdash
plus_minus = plus | minus
# 1 or 1.0 or .1
number_part = Word(nums)
inner_number = Combine((number_part +
Optional("." + Optional(number_part)))
| ("." + number_part))
# Combine() joints the matching parts together in a single token,
# and requires that the matching parts be contiguous (no spaces)
# Define our suffixes
suffix = Word(alphas + '%')
suffix.setParseAction(self.suffix_parse_action)
# Construct number as a group consisting of a text string ("num") and an optional suffix.
# num can include a decimal number and numerical exponent, and can be
# converted to a number using float()
# suffix may contain alphas or %
# Spaces are ignored inside numbers
# Group wraps everything up into its own ParseResults object when parsing
number = Group(
Combine(
inner_number + Optional(
CaselessLiteral("E") + Optional(plus_minus) +
number_part), )("num") +
Optional(suffix)("suffix"))("number")
# Note that calling ("name") on the end of a parser is equivalent to calling
# parser.setResultsName, which is used to pull that result out of a parsed
# expression like a dictionary.
# Construct variable and function names
front = Word(alphas, alphanums) # must start with alpha
subscripts = Word(alphanums + '_') + ~FollowedBy('{') # ~ = not
lower_indices = Literal("_{") + Optional("-") + Word(
alphanums) + Literal("}")
upper_indices = Literal("^{") + Optional("-") + Word(
alphanums) + Literal("}")
# Construct an object name in either of two forms:
# 1. front + subscripts + tail
# 2. front + lower_indices + upper_indices + tail
# where:
# front (required):
# starts with alpha, followed by alphanumeric
# subscripts (optional):
# any combination of alphanumeric and underscores
# lower_indices (optional):
# Of form "_{(-)<alphanumeric>}"
# upper_indices (optional):
# Of form "^{(-)<alphanumeric>}"
# tail (optional):
# any number of primes
name = Combine(front +
Optional(subscripts | (Optional(lower_indices) +
Optional(upper_indices))) +
ZeroOrMore("'"))
# Define a variable as a pyparsing result that contains one object name
variable = Group(name("varname"))("variable")
variable.setParseAction(self.variable_parse_action)
# initialize recursive grammar
expression = Forward()
# Construct functions as consisting of funcname and arguments as
# funcname(arguments)
# where arguments is a comma-separated list of arguments, returned as a list
# Must have at least 1 argument
function = Group(
name("funcname") + Suppress("(") +
Group(delimitedList(expression))("arguments") +
Suppress(")"))("function")
function.setParseAction(self.function_parse_action)
# Define parentheses
parentheses = Group(Suppress("(") + expression +
Suppress(")"))('parentheses')
# Define arrays
array = Group(
Suppress("[") + delimitedList(expression) + Suppress("]"))("array")
# atomic units evaluate directly to number or array without binary operations
atom = number | function | variable | parentheses | array
# Define operations in order of precedence
# Define exponentiation, possibly including negative powers
power = atom + ZeroOrMore(Suppress("^") + Optional(minus)("op") + atom)
power.addParseAction(self.group_if_multiple('power'))
# Define negation (e.g., in 5*-3 --> we need to evaluate the -3 first)
# Negation in powers is handled separately
# This has been arbitrarily assigned a higher precedence than parallel
negation = Optional(minus)("op") + power
negation.addParseAction(self.group_if_multiple('negation'))
# Define the parallel operator 1 || 5 == 1/(1/1 + 1/5)
pipes = Literal('|') + Literal('|')
parallel = negation + ZeroOrMore(Suppress(pipes) + negation)
parallel.addParseAction(self.group_if_multiple('parallel'))
# Define multiplication and division
product = parallel + ZeroOrMore((Literal('*') | Literal('/'))("op") +
parallel)
product.addParseAction(self.group_if_multiple('product'))
# Define sums and differences
# Note that leading - signs are treated by negation
sumdiff = Optional(plus) + product + ZeroOrMore(
plus_minus("op") + product)
sumdiff.addParseAction(self.group_if_multiple('sum'))
# Close the recursion
expression << sumdiff
return expression + stringEnd
fIdent = Word("f", alphanums)
fIdent.setParseAction(mkIdent)
gIdent = Word("g", alphanums)
gIdent.setParseAction(mkIdent)
LPAR = Suppress("(")
RPAR = Suppress(")")
EQ = Suppress("=")
SC = Suppress(";")
exp = Forward();
variable = Group(lIdent)
variable.setParseAction(mkVar)
ctrArgList = Group(Optional(LPAR + Optional(delimitedList(exp)) + RPAR))
constructor = uIdent + ctrArgList
constructor.setParseAction(lambda t: Ctr(t[0], list(t[1])))
fArgList = Group(LPAR + Optional(delimitedList(exp)) + RPAR)
fCall = fIdent + fArgList
fCall.setParseAction(lambda t: FCall(t[0], list(t[1])))
gArgList = Group(LPAR + delimitedList(exp) + RPAR)
gCall = gIdent + gArgList
RETURNS + LPAR + Optional(identifier("response")) + RPAR))("method_definition")
method_definition.setParseAction(method_definition_fn)
service_definition= (SERVICE - identifier("service") + LBRACE + ZeroOrMore(Group(method_definition)) + RBRACE)("service_definition")
service_definition.setParseAction(service_definition_fn)
package_directive = (Group(PACKAGE - delimitedList(identifier, '.', combine=True) + SEMI))("package_directive")
package_directive.setParseAction(package_directive_fn)
import_directive = IMPORT - quotedString("import") + SEMI
import_directive.setParseAction(import_directive_fn)
option_directive = OPTION - identifier("option") + EQ + (integer | quotedString)("value") + SEMI
top_level_statement = Group(message_definition| enum_definition| option_directive | import_directive | service_definition)("top_level_statement")
top_level_statement.setParseAction(top_level_statement_fn)
syntax_directive = (SYNTAX + EQ + quotedString("syntax_version") + SEMI)("syntax_directive")
parser = (Optional(syntax_directive) + Optional(package_directive) + ZeroOrMore(top_level_statement))("parser")
parser.setParseAction(parser_fn)
comment = '//' + restOfLine
parser.ignore(comment)
test2 = """message Person {
required uint32 id = 1;
required string name = 2;
optional string email = 3;
optional Person person = 4;
}"""
for formula in ( "H2O", "NaCl", "C6H5OH", "H2SO4" ):
formulaData = chemicalFormula.parseString(formula)
# compute molecular weight
mw = sum( [ atomicWeight[element.symbol]*element.qty for element in formulaData ] )
# print the results
print formula, "->", formulaData, "(", mw, ")"
print
# Version 3 - Compute partial molecular weight per element, simplifying
# summing
# No need to redefine grammar, just define parse action function, and
# attach to elementRef
def computeElementWeight(tokens):
element = tokens[0]
element["weight"] = atomicWeight[element.symbol] * element.qty
elementRef.setParseAction(computeElementWeight)
for formula in ( "H2O", "NaCl", "C6H5OH", "H2SO4" ):
formulaData = chemicalFormula.parseString(formula)
# compute molecular weight
mw = sum( [ element.weight for element in formulaData ] )
# print the results
print formula, "->", formulaData, "(", mw, ")"
def graph_definition():
global graphparser
if not graphparser:
# punctuation
colon = Literal(":")
lbrace = Literal("{")
rbrace = Literal("}")
lbrack = Literal("[")
rbrack = Literal("]")
lparen = Literal("(")
rparen = Literal(")")
equals = Literal("=")
comma = Literal(",")
dot = Literal(".")
slash = Literal("/")
bslash = Literal("\\")
star = Literal("*")
semi = Literal(";")
at = Literal("@")
minus = Literal("-")
# keywords
strict_ = CaselessLiteral("strict")
graph_ = CaselessLiteral("graph")
digraph_ = CaselessLiteral("digraph")
subgraph_ = CaselessLiteral("subgraph")
node_ = CaselessLiteral("node")
edge_ = CaselessLiteral("edge")
# token definitions
identifier = Word(alphanums + "_." ).setName("identifier")
double_quoted_string = QuotedString('"', multiline=True, unquoteResults=False) # dblQuotedString
_noncomma = "".join([c for c in printables if c != ","])
alphastring_ = OneOrMore(CharsNotIn(_noncomma + ' '))
def parse_html(s, loc, toks):
return '<%s>' % ''.join(toks[0])
opener = '<'
closer = '>'
html_text = nestedExpr( opener, closer,
( CharsNotIn( opener + closer ) )
).setParseAction(parse_html).leaveWhitespace()
ID = ( identifier | html_text |
double_quoted_string | #.setParseAction(strip_quotes) |
alphastring_ ).setName("ID")
float_number = Combine(Optional(minus) +
OneOrMore(Word(nums + "."))).setName("float_number")
righthand_id = (float_number | ID ).setName("righthand_id")
port_angle = (at + ID).setName("port_angle")
port_location = (OneOrMore(Group(colon + ID)) |
Group(colon + lparen + ID + comma + ID + rparen)).setName("port_location")
port = (Group(port_location + Optional(port_angle)) |
Group(port_angle + Optional(port_location))).setName("port")
node_id = (ID + Optional(port))
a_list = OneOrMore(ID + Optional(equals + righthand_id) +
Optional(comma.suppress())).setName("a_list")
attr_list = OneOrMore(lbrack.suppress() + Optional(a_list) +
rbrack.suppress()).setName("attr_list")
attr_stmt = (Group(graph_ | node_ | edge_) + attr_list).setName("attr_stmt")
edgeop = (Literal("--") | Literal("->")).setName("edgeop")
stmt_list = Forward()
graph_stmt = Group(lbrace.suppress() + Optional(stmt_list) +
rbrace.suppress() + Optional(semi.suppress()) ).setName("graph_stmt")
edge_point = Forward()
edgeRHS = OneOrMore(edgeop + edge_point)
edge_stmt = edge_point + edgeRHS + Optional(attr_list)
subgraph = Group(subgraph_ + Optional(ID) + graph_stmt).setName("subgraph")
edge_point << Group(subgraph | graph_stmt | node_id ).setName('edge_point')
node_stmt = (node_id + Optional(attr_list) + Optional(semi.suppress())).setName("node_stmt")
assignment = (ID + equals + righthand_id).setName("assignment")
stmt = (assignment | edge_stmt | attr_stmt | subgraph | graph_stmt | node_stmt).setName("stmt")
stmt_list << OneOrMore(stmt + Optional(semi.suppress()))
graphparser = OneOrMore( (Optional(strict_) + Group((graph_ | digraph_)) +
Optional(ID) + graph_stmt).setResultsName("graph") )
singleLineComment = Group("//" + restOfLine) | Group("#" + restOfLine)
# actions
graphparser.ignore(singleLineComment)
graphparser.ignore(cStyleComment)
assignment.setParseAction(push_attr_list)
a_list.setParseAction(push_attr_list)
edge_stmt.setParseAction(push_edge_stmt)
node_stmt.setParseAction(push_node_stmt)
attr_stmt.setParseAction(push_default_stmt)
subgraph.setParseAction(push_subgraph_stmt)
graph_stmt.setParseAction(push_graph_stmt)
graphparser.setParseAction(push_top_graph_stmt)
return graphparser
def __init__(self, ffilter = None, filter_string = None):
if PYPARSING:
quoted_str_value = QuotedString('\'', unquoteResults=True, escChar='\\')
int_values = Word("0123456789")
error_value = Literal("XXX").setParseAction(self.__compute_xxx_value)
bbb_value = Literal("BBB").setParseAction(self.__compute_bbb_value)
field_value = Word(alphas + "." + "_" + "-")
basic_primitives = int_values | quoted_str_value
operator_names = oneOf("m d e un u r l sw unique startswith decode encode unquote replace lower upper").setParseAction(lambda s,l,t: [ (l,t[0])])
fuzz_symbol = (Suppress("FUZ") + Optional(Word("23456789"), 1).setParseAction( lambda s,l,t: [ int(t[0])-1 ] ) + Suppress("Z")).setParseAction(self.__compute_fuzz_symbol)
operator_call = Group(Suppress("|") + operator_names + Suppress("(") + Optional(basic_primitives, None) + Optional(Suppress(",") + basic_primitives, None) + Suppress(")"))
fuzz_value = (fuzz_symbol + Optional(Suppress("[") + field_value + Suppress("]"), None)).setParseAction(self.__compute_fuzz_value)
fuzz_value_op = ((fuzz_symbol + Suppress("[") + Optional(field_value)).setParseAction(self.__compute_fuzz_value) + operator_call + Suppress("]")).setParseAction(self.__compute_perl_value)
fuzz_value_op2 = ((fuzz_symbol + operator_call).setParseAction(self.__compute_perl_value))
res_value_op = (Word(alphas + "." + "_" + "-").setParseAction(self.__compute_res_value) + Optional(operator_call, None)).setParseAction(self.__compute_perl_value)
basic_primitives_op = (basic_primitives + Optional(operator_call, None)).setParseAction(self.__compute_perl_value)
fuzz_statement = fuzz_value ^ fuzz_value_op ^ fuzz_value_op2 ^ res_value_op ^ basic_primitives_op
operator = oneOf("and or")
not_operator = Optional(oneOf("not"), "notpresent")
symbol_expr = Group(fuzz_statement + oneOf("= != < > >= <= =~ !~ ~") + (bbb_value ^ error_value ^ fuzz_statement ^ basic_primitives)).setParseAction(self.__compute_expr)
definition = fuzz_statement ^ symbol_expr
definition_not = not_operator + definition
definition_expr = definition_not + ZeroOrMore( operator + definition_not)
nested_definition = Group(Suppress("(") + definition_expr + Suppress(")"))
nested_definition_not = not_operator + nested_definition
self.finalformula = (nested_definition_not ^ definition_expr) + ZeroOrMore( operator + (nested_definition_not ^ definition_expr))
definition_not.setParseAction(self.__compute_not_operator)
nested_definition_not.setParseAction(self.__compute_not_operator)
nested_definition.setParseAction(self.__compute_formula)
self.finalformula.setParseAction(self.__myreduce)
if ffilter is not None and filter_string is not None:
raise FuzzExceptInternalError(FuzzException.FATAL, "A filter must be initilized with a filter string or an object, not both")
self.res = None
if ffilter:
self.hideparams = ffilter
else:
self.hideparams = dict(
regex_show = None,
codes_show = None,
codes = [],
words = [],
lines = [],
chars = [],
regex = None,
filter_string = ""
)
if filter_string:
self.hideparams['filter_string'] = filter_string
self.baseline = None
self.stack = {}
self._cache = collections.defaultdict(set)
