def get_formula_parser(chemical_tokens): LPAR, RPAR = map(Suppress, "[]") index = Regex(r"\d+(\.\d*)?").setParseAction(lambda t: float(t[0])) element = oneOf(chemical_tokens) chemical_formula = Forward() term = Group((element | Group(LPAR + chemical_formula + RPAR)("subgroup")) + Optional(index, default=1)("mult")) chemical_formula << OneOrMore(term) def multiplyContents(tokens): t = tokens[0] if t.subgroup: mult = t.mult for term in t.subgroup: term[1] *= mult return t.subgroup term.setParseAction(multiplyContents) def sumByElement(tokens): elementsList = [t[0] for t in tokens] duplicates = len(elementsList) > len(set(elementsList)) if duplicates: ctr = defaultdict(int) for t in tokens: ctr[t[0]] += t[1] return ParseResults([ParseResults([k, v]) for k, v in ctr.items()]) chemical_formula.setParseAction(sumByElement) return chemical_formula
def get_parse_expression(parse_func, parse_var_id): """return a parse expression with for the input parseActions """ var_id = Group(FUNC_NAME + COLON + IDENTIFIER) | REAL | INTEGER | IDENTIFIER var_id.setParseAction(parse_var_id) # Forward declaration for an Arithmetic Expression arith_expr = Forward() func_call = Group(FUNC_NAME + LPAREN + Optional(Group(delimitedList(arith_expr))) + RPAREN) # An Arithmetic expression can have a var_id or # a function call as an operand # pylint: disable=expression-not-assigned arith_expr << operatorPrecedence( func_call | var_id, [ (EXPONENTIATION_OPS, 2, opAssoc.LEFT, eval_binary_op), (UNARY_OPS, 1, opAssoc.RIGHT, eval_unary_op), (MULT_OPS, 2, opAssoc.LEFT, eval_binary_op), (SUM_OPS, 2, opAssoc.LEFT, eval_binary_op), (REL_OPS, 2, opAssoc.LEFT, eval_binary_op), (LOGICAL_OPS, 2, opAssoc.LEFT, eval_binary_op), ], ) # pylint: enable=expression-not-assigned # Argument expression for a function call # An argument to a function can be an # IDENTIFIER, Arithmetic expression, REAL number, INTEGER or a # Function call itself func_call.setParseAction(parse_func) return arith_expr
def __init__(self, ffilter, queue_out): FuzzQueue.__init__(self, queue_out) Thread.__init__(self) self.setName('filter_thread') self.queue_out = queue_out if PYPARSING: element = oneOf("c l w h") digits = "XB0123456789" integer = Word( digits )#.setParseAction( self.__convertIntegers ) elementRef = Group(element + oneOf("= != < > >= <=") + integer) operator = oneOf("and or") definition = elementRef + ZeroOrMore( operator + elementRef) nestedformula = Group(Suppress(Optional(Literal("("))) + definition + Suppress(Optional(Literal(")")))) self.finalformula = nestedformula + ZeroOrMore( operator + nestedformula) elementRef.setParseAction(self.__compute_element) nestedformula.setParseAction(self.__compute_formula) self.finalformula.setParseAction(self.__myreduce) self.res = None self.hideparams = ffilter if "XXX" in self.hideparams['codes']: self.hideparams['codes'].append("0") self.baseline = None
def _define_context_component (self, cluster, base_feature_set): placeholder = Literal(SOURCE_PLACEHOLDER) placeholder.setParseAction(self._handle_placeholder) context_component = Group(ZeroOrMore(cluster ^ base_feature_set) + \ placeholder + ZeroOrMore(cluster ^ base_feature_set)).setResultsName('context_component') context_component.setParseAction(self._handle_context_component) return context_component
def get_parse_expression(parse_func, parse_var_id): """return a parse expression with for the input parseActions """ var_id = Group(FUNC_NAME + COLON + IDENTIFIER) | REAL | INTEGER | IDENTIFIER var_id.setParseAction(parse_var_id) # Forward declaration for an Arithmetic Expression arith_expr = Forward() func_call = Group(FUNC_NAME + LPAREN + Optional(Group(delimitedList(arith_expr))) + RPAREN) # An Arithmetic expression can have a var_id or # a function call as an operand # pylint: disable=expression-not-assigned arith_expr << operatorPrecedence(func_call | var_id, [ (EXPONENTIATION_OPS, 2, opAssoc.LEFT, eval_binary_op), (UNARY_OPS, 1, opAssoc.RIGHT, eval_unary_op), (MULT_OPS, 2, opAssoc.LEFT, eval_binary_op), (SUM_OPS, 2, opAssoc.LEFT, eval_binary_op), (REL_OPS, 2, opAssoc.LEFT, eval_binary_op), (LOGICAL_OPS, 2, opAssoc.LEFT, eval_binary_op) ]) # pylint: enable=expression-not-assigned # Argument expression for a function call # An argument to a function can be an # IDENTIFIER, Arithmetic expression, REAL number, INTEGER or a # Function call itself func_call.setParseAction(parse_func) return arith_expr
def ifParser(): comma = Literal(",").suppress() hash = Literal("#").suppress() equal = Literal("=").suppress() # Rules and labels rulename = Word (alphanums + "_") rulecategory = oneOf("Protocol_Rules Invariant_Rules Decomposition_Rules Intruder_Rules Init Goal") label = hash + Literal("lb") + equal + rulename + comma + Literal("type") + equal + rulecategory labeledrule = Group(label) + Group(ruleParser()) def labeledruleAction(s,l,t): if t[0][3] == "Protocol_Rules": print "-----------------" print "- Detected rule -" print "-----------------" print t[0] print t[1] print labeledrule.setParseAction(labeledruleAction) # A complete file parser = OneOrMore(labeledrule) parser.ignore("##" + restOfLine) return parser
def message_type_definition() -> Token: first_aspect = Keyword("First") - Keyword("=>") - mathematical_expression() first_aspect.setParseAction(parse_aspect) length_aspect = Keyword("Length") - Keyword( "=>") - mathematical_expression() length_aspect.setParseAction(parse_aspect) component_aspects = Keyword("with") - delimitedList(first_aspect | length_aspect) component_aspects.setParseAction(parse_aspects) then = locatedExpr( Keyword("then") - (Keyword("null") | unqualified_identifier()) - Group(Optional(component_aspects)) - Group(Optional(value_constraint()))) then.setParseAction(parse_then) then_list = then + ZeroOrMore(comma() - then) then_list.setParseAction(lambda t: [t.asList()]) component_item = (~Keyword("end") + ~CaselessKeyword("Message") - unqualified_identifier() + Literal(":") - qualified_identifier() - Optional(then_list) - semicolon()) component_item.setParseAction(lambda t: Component(t[0], t[2], t[3]) if len(t) >= 4 else Component(t[0], t[2])) component_item.setName("Component") null_component_item = Keyword("null") - then - semicolon() null_component_item.setParseAction(lambda t: Component(None, None, [t[1]])) null_component_item.setName("NullComponent") component_list = Group( Optional(null_component_item) - component_item - ZeroOrMore(component_item)) component_list.setParseAction(lambda t: t.asList()) return (Keyword("message") - component_list - Keyword("end message") | Keyword("null message")).setName("Message")
def ifParser(): comma = Literal(",").suppress() hash = Literal("#").suppress() equal = Literal("=").suppress() # Rules and labels rulename = Word(alphanums + "_") rulecategory = oneOf( "Protocol_Rules Invariant_Rules Decomposition_Rules Intruder_Rules Init Goal" ) label = hash + Literal("lb") + equal + rulename + comma + Literal( "type") + equal + rulecategory labeledrule = Group(label) + Group(ruleParser()) def labeledruleAction(s, l, t): if t[0][3] == "Protocol_Rules": print "-----------------" print "- Detected rule -" print "-----------------" print t[0] print t[1] print labeledrule.setParseAction(labeledruleAction) # A complete file parser = OneOrMore(labeledrule) parser.ignore("##" + restOfLine) return parser
def list_dict(word=word_free): """ Return the pyparsing lexical element, that parses a string either as a list or as a dictionary. Parameters ---------- word : lexical element A custom lexical element for word. Returns ------- ld : lexical element The returned lexical element parses a string in the form ``..., ..., ...`` or ``key1:..., key2=..., key3: ...`` where ``...`` is a ``list_item`` from :func:`get_standard_type_defs()` and interprets it as a list or a dictionary. """ defs = get_standard_type_defs(word) i = defs['list_item'] arg = i.copy() arg.setParseAction(lambda t: (t[0],)) narg = word_strict + (colon | equal_sign) + i narg.setParseAction(lambda t: (t[0], t[1])) ld = Group(list_of(narg | arg)) ld.setParseAction(lambda t: ([x[0] for x in t[0] if len(x) == 1], dict([x for x in t[0] if len(x) > 1])) ) return ld
def __init__(self, filter_string=None): self.filter_string = filter_string self.baseline = None quoted_str_value = QuotedString("'", unquoteResults=True, escChar="\\") int_values = Word("0123456789").setParseAction( lambda s, l, t: [int(t[0])]) error_value = Literal("XXX").setParseAction(self.__compute_xxx_value) operator_call = Regex( r"\|(?P<operator>(m|d|e|un|u|r|l|sw|gre|gregex|unique|startswith|decode|encode|unquote|replace|lower|upper))" r"\((?:(?P<param1>('.*?'|\d+))(?:,(?P<param2>('.*?'|\d+)))?)?\)", asMatch=True, ).setParseAction(lambda s, l, t: [(l, t[0])]) fuzz_symbol = Regex( r"FUZ(?P<index>\d)*Z(?:\[(?P<field>(\w|_|-|\.)+)\])?", asMatch=True).setParseAction(self._compute_fuzz_symbol) res_symbol = Regex( r"(description|nres|code|chars|lines|words|md5|content|timer|url|plugins|l|w|c|(r|history)(\w|_|-|\.)*|h)" ).setParseAction(self._compute_res_symbol) bbb_symbol = Regex(r"BBB(?:\[(?P<field>(\w|_|-|\.)+)\])?", asMatch=True).setParseAction( self.__compute_bbb_symbol) fuzz_statement = Group((fuzz_symbol | res_symbol | bbb_symbol | int_values | quoted_str_value) + Optional(operator_call, None)).setParseAction( self.__compute_res_value) operator = oneOf("and or") not_operator = Optional(oneOf("not"), "notpresent") symbol_expr = Group(fuzz_statement + oneOf("= == != < > >= <= =~ !~ ~ := =+ =-") + (error_value | fuzz_statement)).setParseAction( self.__compute_expr) definition = symbol_expr ^ fuzz_statement definition_not = not_operator + definition definition_expr = definition_not + ZeroOrMore(operator + definition_not) nested_definition = Group( Suppress("(") + definition_expr + Suppress(")")) nested_definition_not = not_operator + nested_definition self.finalformula = (nested_definition_not | definition_expr) + ZeroOrMore(operator + ( nested_definition_not | definition_expr)) definition_not.setParseAction(self.__compute_not_operator) nested_definition_not.setParseAction(self.__compute_not_operator) nested_definition.setParseAction(self.__compute_formula) self.finalformula.setParseAction(self.__myreduce) self.res = None self.stack = [] self._cache = collections.defaultdict(set)
def _formula_parser(formula, calculate_or_not, atoms): ''' parses the chemical formula and determine its mol weight. But only if `calculate_or_not` equals `calculate`, otherwise, do nothing. Use the molecular weight from atoms dict. ''' if (calculate_or_not != 'calculate') or (formula in [ 'N_tot_k', 'PO4_ortho', 'SO4_ic', 'alkalinity' ]): return calculate_or_not # define some strings to use later, when describing valid lists # of characters for chemical symbols and numbers caps = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" lowers = caps.lower() digits = "0123456789" brackets = "[]{}()" if any([_ in formula for _ in brackets]): raise Exception('Not implemented with brackets in formula string') # Version 1 # Define grammar for a chemical formula # - an element is a Word, beginning with one of the characters in caps, # followed by zero or more characters in lowers # - an integer is a Word composed of digits # - an elementRef is an element, optionally followed by an integer - if # the integer is omitted, assume the value "1" as a default; these are # enclosed in a Group to make it easier to walk the list of parsed # chemical symbols, each with its associated number of atoms per # molecule # - a chemicalFormula is just one or more elementRef's # Version 3 - Compute partial molecular weight per element, simplifying # summing # No need to redefine grammar, just define parse action function, and # attach to elementRef # Auto-convert integers, and add results names def convert_integers(tokens): return int(tokens[0]) element = Word(caps, lowers) integer = Word(digits).setParseAction(convert_integers) element_ref = Group( element("symbol") + Optional(integer, default=1)("qty")) chemical_formula = OneOrMore(element_ref) def compute_element_weight(tokens): element = tokens[0] element["weight"] = atoms[element.symbol] * element.qty element_ref.setParseAction(compute_element_weight) formula_data = chemical_formula.parseString(formula) # compute molecular weight mw = sum([element.weight for element in formula_data]) return mw
def __init__(self): # speed up infixNotation considerably at the price of some cache memory ParserElement.enablePackrat() boolean = Keyword('True') | Keyword('False') none = Keyword('None') integer = Word(nums) real = Combine(Word(nums) + "." + Word(nums)) string = (QuotedString('"', escChar='\\') | QuotedString("'", escChar='\\')) regex = QuotedString('/', escChar='\\') identifier = Word(alphas, alphanums + '_') dereference = infixNotation(identifier, [ (Literal('.'), 2, opAssoc.LEFT, EvalArith), ]) result = (Keyword('bad') | Keyword('fail') | Keyword('good') | Keyword('ignore') | Keyword('unknown')) rval = boolean | none | real | integer | string | regex | result | dereference rvallist = Group( Suppress('[') + Optional(delimitedList(rval)) + Suppress(']')) rvalset = Group( Suppress('{') + Optional(delimitedList(rval)) + Suppress('}')) operand = rval | rvallist | rvalset # parse actions replace the parsed tokens with an instantiated object # which we can later call into for evaluation of its content boolean.setParseAction(EvalBoolean) none.setParseAction(EvalNone) integer.setParseAction(EvalInteger) real.setParseAction(EvalReal) string.setParseAction(EvalString) regex.setParseAction(EvalRegex) identifier.setParseAction(EvalIdentifier) result.setParseAction(EvalResult) rvallist.setParseAction(EvalList) rvalset.setParseAction(EvalSet) identity_test = Keyword('is') + ~Keyword('not') | Combine( Keyword('is') + Keyword('not'), adjacent=False, joinString=' ') membership_test = Keyword('in') | Combine( Keyword('not') + Keyword('in'), adjacent=False, joinString=' ') comparison_op = oneOf('< <= > >= != == isdisjoint') comparison = identity_test | membership_test | comparison_op self.parser = infixNotation(operand, [ (Literal('**'), 2, opAssoc.LEFT, EvalPower), (oneOf('+ - ~'), 1, opAssoc.RIGHT, EvalModifier), (oneOf('* / // %'), 2, opAssoc.LEFT, EvalArith), (oneOf('+ -'), 2, opAssoc.LEFT, EvalArith), (oneOf('<< >>'), 2, opAssoc.LEFT, EvalArith), (Literal('&'), 2, opAssoc.LEFT, EvalArith), (Literal('^'), 2, opAssoc.LEFT, EvalArith), (Literal('|'), 2, opAssoc.LEFT, EvalArith), (comparison, 2, opAssoc.LEFT, EvalLogic), (Keyword('not'), 1, opAssoc.RIGHT, EvalModifier), (Keyword('and'), 2, opAssoc.LEFT, EvalLogic), (Keyword('or'), 2, opAssoc.LEFT, EvalLogic), (Keyword('->'), 2, opAssoc.LEFT, EvalArith), ])
def split_chemical_formula(formula): def is_number(s): try: float(s) return True except ValueError: return False def replace_things(stringg, listt, replacement): for x in listt: stringg = stringg.replace(x, replacement) return stringg bad_chars = ["(", ")", "-", "."] formula = replace_things(formula, bad_chars, "|") if is_number(formula): return [["", 0]] if len(formula) == 0: return [["", 0]] # define some strings to use later, when describing valid lists # of characters for chemical symbols and numbers caps = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" lowers = caps.lower() digits = "0123456789" # Version 1 # Version 2 - Auto-convert integers, and add results names def convertIntegers(tokens): return int(tokens[0]) element = Word(caps, lowers) integer = Word(digits).setParseAction(convertIntegers) elementRef = Group(element("symbol") + Optional(integer, default=1)("qty")) # pre-1.4.7, use this: # elementRef = Group( element.setResultsName("symbol") + Optional( integer, default=1 ).setResultsName("qty") ) chemicalFormula = OneOrMore(elementRef) # Version 3 - Compute partial molecular weight per element, simplifying # summing # No need to redefine grammar, just define parse action function, and # attach to elementRef def computeElementWeight(tokens): element = tokens[0] element["weight"] = atomicWeight[element.symbol] * element.qty elementRef.setParseAction(computeElementWeight) formulaData = chemicalFormula.parseString(formula) mw = sum([element.weight for element in formulaData]) return formulaData
def get_fragment_grammar(): # Match header [mapping] header = Suppress("[") + Suppress("mapping") + Suppress("]") # There are three possible patterns for mapping entries: # obj:symbol (scheme) # obj (scheme) # * (scheme) obj = Fragment.ENTITY.setResultsName("object") symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol") scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName( "scheme") + Suppress(")") pattern1 = Group(obj + symbol + scheme) pattern2 = Group(obj + scheme) pattern3 = Group( Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") + scheme) mapping_entry = pattern1 | pattern2 | pattern3 # To simplify parsing, classify groups of condition-mapping entry into two types: normal and default # A normal grouping is one with a non-default condition. The default grouping is one which contains the # default condition mapping_entries = Group( ZeroOrMore(mapping_entry)).setResultsName("mappings") normal_condition = Suppress(":") + originalTextFor( SDKConfig.get_expression_grammar()) default_condition = Optional( Suppress(":") + Literal(Mapping.DEFAULT_CONDITION)) normal_group = Group( normal_condition.setResultsName("condition") + mapping_entries) default_group = Group(default_condition + mapping_entries).setResultsName("default_group") normal_groups = Group( ZeroOrMore(normal_group)).setResultsName("normal_groups") # Any mapping fragment definition can have zero or more normal group and only one default group as a last entry. archive = Suppress("archive") + Suppress( ":") + Fragment.ENTITY.setResultsName("archive") entries = Suppress("entries") + Suppress(":") + ( normal_groups + default_group).setResultsName("entries") mapping = Group(header + archive + entries) mapping.setParseAction(lambda t: Mapping(t[0].archive, t[0].entries)) mapping.ignore("#" + restOfLine) return mapping
def condition_term(): ''' Grammar for condition term 'IF' <predicate> {'AND' <predicate>}* 'THEN' ''' predicate = predicate_term() condition = Group( Suppress(IF_KEYWORD) + delimitedList(predicate, AND_KEYWORD) + Suppress(THEN_KEYWORD)) condition.setParseAction(lambda t: t[0].asList()) return condition
def GetHeaders(self): title = Group( Optional(Literal("OPTIMAL")) + Literal("POWER FLOW REPORT")) title.setParseAction(self.process_header_title) version = slit("P S A T 2.1.") + integer.suppress() author = slit("Author: Federico Milano, (c) 2002-2009") email = slit("e-mail: [email protected]") website = slit("website: http://www.uclm.es/area/gsee/Web/Federico") filename = slit("File:") + restOfLine.suppress() date = slit("Date:") + restOfLine.suppress() return title + version + author + email + website + filename + date
class NodeParser: def __init__(self): self.num = Word(nums) self.header = Regex(r"^UCLA.*") self.comment = Regex(r"#.*") self.bkid = Word(alphanums) self.num_nodes = Literal("NumNodes") + Literal(":") + self.num( "NumNodes") self.num_terminals = Literal("NumTerminals") + Literal(":") + self.num( "NumTerminals") self.size = Group(self.num("width") + self.num("height")) self.terminal = Optional(Literal("terminal")) self.node = self.bkid("id") + self.size("size") + self.terminal self.node_grammar = self.header + ZeroOrMore(self.comment) + self.num_nodes + self.num_terminals + \ OneOrMore(self.node) self.coordinate = Group(self.num("x") + self.num("y")) self.pl = self.bkid("id") + self.coordinate("coordinate") + \ Suppress(Literal(": N") + Optional(Literal(r"/FIXED"))) self.pl_grammar = self.header + ZeroOrMore(self.comment) + OneOrMore( self.pl) def compute_chip_size(self, benchmark): benchmark_path = pathlib.Path(os.environ["BENCHMARK"]) node_file = benchmark_path / "ispd2005/{0}/{0}.nodes".format(benchmark) pl_file = benchmark_path / "ispd2005/{0}/{0}.pl".format(benchmark) print(node_file.as_posix()) print(pl_file.as_posix()) x_max = 0 y_max = 0 sizes = [] coordinates = [] self.size.setParseAction( lambda tokens: sizes.append([tokens.width, tokens.height])) self.coordinate.setParseAction(lambda tokens: coordinates.append( (tokens.x, tokens.y))) self.bkid.setParseAction(lambda tokens: print(tokens[0])) self.node_grammar.parseFile(node_file.as_posix()) self.pl_grammar.parseFile(pl_file.as_posix()) for i in range(len(sizes)): print(i) if coordinates[i][0] + sizes[i][0] > x_max: x_max = coordinates[i][0] + sizes[i][0] if coordinates[i][1] + sizes[i][1] > y_max: y_max = coordinates[i][1] + sizes[i][1] return x_max, y_max
def main(s): lpar = Literal('(').suppress() rpar = Literal(')').suppress() integer = Word(nums) element = Word(alphas, exact=1) formula = Forward() term = Group((element | Group(lpar + formula + rpar)('subgroup')) + Optional(integer, default=1)('mult')) formula << OneOrMore(term) integer.setParseAction(process_integer) term.setParseAction(process_term) formula.setParseAction(process_formula) return formula.parseString(s)[0]
def get_fragment_grammar(): name = Fragment.IDENTIFIER header = Suppress("[") + Suppress("sections") + Suppress(":") + name.setResultsName("name") + Suppress("]") entry = Word(alphanums + "+" + ".") entries = Suppress("entries") + Suppress(":") + Group(OneOrMore(entry)).setResultsName("entries") sections = Group(header + entries) sections.setParseAction(lambda t: Sections(t[0].name, t[0].entries)) sections.ignore("#" + restOfLine) return sections
def __init__(self): plus, minus, mult, div, mod = map(Literal, '+-*/%') lpar = Literal('(') rpar = Literal(')') comma = Literal(',') powop = Literal( '^' ) productop = mult | div modop = Literal( '%' ) sumop = plus | minus tupl = Forward() number = Regex(r'[+-]?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?') number.setParseAction(number_action) ident = Word(alphas, alphanums+'_') ident.setParseAction(ident_action) funccall = ident + tupl funccall.setParseAction(funccall_action) atom = funccall | ident | number atom.setParseAction(atom_action) patom = lpar + atom + rpar patom.setParseAction(patom_action) powexpr = Forward() powexpr << Group( atom + ZeroOrMore( ( powop + powexpr ) ) ) powexpr.setParseAction(powexpr_action) modexpr = Forward() modexpr << Group( powexpr + ZeroOrMore( ( modop + modexpr ) ) ) modexpr.setParseAction(modexpr_action) product = Group( modexpr + ZeroOrMore( ( productop + modexpr ) ) ) product.setParseAction(product_action) sumexpr = Group( product + Group( ZeroOrMore( sumop + product ) ) ) sumexpr.setParseAction(sum_action) tupl << lpar + Optional(sumexpr + ZeroOrMore( comma + sumexpr ) ) + rpar tupl.setParseAction(tupl_action) expr = sumexpr | tupl expr.setParseAction(expr_action) self.bnf = expr
def temporal_condition_term(): ''' Grammar for condition term 'IF' <predicate> {'AND' <predicate>}* 'THEN' ''' from grammar.keywords import IF_KEYWORD, AND_KEYWORD, \ THEN_KEYWORD predicate = temporal_predicate_term() condition = Group( Suppress(IF_KEYWORD) + delimitedList(predicate, AND_KEYWORD) + Suppress(THEN_KEYWORD)) condition.setParseAction(lambda t: t[0].asList()) return condition
class NodeParser: def __init__(self): self.num = Word(nums) self.header = Regex(r"^UCLA.*") self.comment = Regex(r"#.*") self.bkid = Word(alphanums) self.num_nodes = Literal("NumNodes") + Literal(":") + self.num("NumNodes") self.num_terminals = Literal("NumTerminals") + Literal(":") + self.num("NumTerminals") self.size = Group(self.num("width") + self.num("height")) self.terminal = Optional(Literal("terminal")) self.node = self.bkid("id") + self.size("size") + self.terminal self.node_grammar = ( self.header + ZeroOrMore(self.comment) + self.num_nodes + self.num_terminals + OneOrMore(self.node) ) self.coordinate = Group(self.num("x") + self.num("y")) self.pl = ( self.bkid("id") + self.coordinate("coordinate") + Suppress(Literal(": N") + Optional(Literal(r"/FIXED"))) ) self.pl_grammar = self.header + ZeroOrMore(self.comment) + OneOrMore(self.pl) def compute_chip_size(self, benchmark): benchmark_path = pathlib.Path(os.environ["BENCHMARK"]) node_file = benchmark_path / "ispd2005/{0}/{0}.nodes".format(benchmark) pl_file = benchmark_path / "ispd2005/{0}/{0}.pl".format(benchmark) print(node_file.as_posix()) print(pl_file.as_posix()) x_max = 0 y_max = 0 sizes = [] coordinates = [] self.size.setParseAction(lambda tokens: sizes.append([tokens.width, tokens.height])) self.coordinate.setParseAction(lambda tokens: coordinates.append((tokens.x, tokens.y))) self.bkid.setParseAction(lambda tokens: print(tokens[0])) self.node_grammar.parseFile(node_file.as_posix()) self.pl_grammar.parseFile(pl_file.as_posix()) for i in range(len(sizes)): print(i) if coordinates[i][0] + sizes[i][0] > x_max: x_max = coordinates[i][0] + sizes[i][0] if coordinates[i][1] + sizes[i][1] > y_max: y_max = coordinates[i][1] + sizes[i][1] return x_max, y_max
def define_operand(self): """ Return the syntax definition for an operand. An operand can be a variable, a string, a number or a set. A set is made of other operands, including other sets. **This method shouldn't be overridden**. Instead, override the syntax definitions for variables, strings and/or numbers. If you want to customize the sets, check :meth:`T_SET_START`, :meth:`T_SET_END` and :meth:`T_ELEMENT_SEPARATOR`. """ identifier = self.define_identifier() operand = Forward() # Defining the sets: set_start = Suppress(self._grammar.get_token("set_start")) set_end = Suppress(self._grammar.get_token("set_end")) element_separator = self._grammar.get_token("element_separator") elements = delimitedList(operand, delim=element_separator) set_ = Group(set_start + Optional(elements) + set_end) set_.setParseAction(self.make_set) set_.setName("set") # Defining the variables: variable = identifier.copy() variable.setName("variable") variable.addParseAction(self.make_variable) # Defining the functions: function_name = identifier.setResultsName("function_name") function_name.setName("function_name") args_start = Suppress(self._grammar.get_token("arguments_start")) args_end = Suppress(self._grammar.get_token("arguments_end")) args_sep = self._grammar.get_token("arguments_separator") arguments = Optional(Group(delimitedList(operand, delim=args_sep)), default=()) arguments = arguments.setResultsName("arguments") arguments.setParseAction(lambda tokens: tokens[0]) function = function_name + args_start + arguments + args_end function.setName("function") function.setParseAction(self.make_function) operand << (function | variable | self.define_number() | \ self.define_string() | set_) return operand
def __init__(self): if PYPARSING: category = Word( alphas + "_-*", alphanums + "_-*" ) operator = oneOf("and or ,") neg_operator = "not" elementRef = category definition = elementRef + ZeroOrMore( operator + elementRef) nestedformula = Group(Suppress(Optional(Literal("("))) + definition + Suppress(Optional(Literal(")")))) neg_nestedformula = Optional(neg_operator) + nestedformula self.finalformula = neg_nestedformula + ZeroOrMore( operator + neg_nestedformula) elementRef.setParseAction(self.__compute_element) neg_nestedformula.setParseAction(self.__compute_neg_formula) nestedformula.setParseAction(self.__compute_formula) self.finalformula.setParseAction(self.__myreduce)
def _build_filter_parser(field_names): field = _build_field_expr(field_names) negation = CaselessKeyword('not') negation.setParseAction(lambda x: Negation(x[0])) comparison_operator = Group( Keyword('=') ^ Keyword('!=') ^ Keyword('>=') ^ Keyword('<=') ^ Keyword('<') ^ Keyword('>') ^ Keyword('lte') # match before lt ^ Keyword('lt') ^ Keyword('gte') # match before gt ^ Keyword('gt') ^ (Optional(negation) + (CaselessKeyword('contains') ^ CaselessKeyword('icontains') ^ CaselessKeyword('startswith') ^ CaselessKeyword('istartswith') ^ CaselessKeyword('endswith') ^ CaselessKeyword('iendswith') ^ CaselessKeyword('eq')))) comparison_operator.setParseAction(lambda x: Operator(x)) single_value_operator = Group( CaselessKeyword('isnull') ^ (Optional(negation) + CaselessKeyword('isnull'))) single_value_operator.setParseAction(lambda x: Operator(x)) plusorminus = Literal('+') | Literal('-') num_integer = Combine(Optional(plusorminus) + Word(nums)) num_integer.setParseAction(lambda x: Integer(x[0])) num_float = Combine(Optional(plusorminus) + Word(nums) + '.' + Word(nums)) num_float.setParseAction(lambda x: Float(x[0])) quoted_string = (QuotedString("'") ^ QuotedString('"')) quoted_string.setParseAction(lambda x: String(x[0])) boolean = Or([ CaselessKeyword(v) for v in BOOLEAN_TRUE_VALUES + BOOLEAN_FALSE_VALUES ]) boolean.setParseAction(lambda x: Boolean(x[0])) value = (quoted_string ^ num_integer ^ boolean) comparison = Group((field + comparison_operator + value) ^ (value + comparison_operator + field) ^ (field + comparison_operator + field) ^ (field + single_value_operator)) comparison.setParseAction(lambda x: Comparison(x)) invalid_comparison = Group( (value + comparison_operator + value).setParseAction(lambda x: fail( "Value may not be compared with values: {0}".format(' '.join(x))))) logical_op = Group(CaselessKeyword("and") | CaselessKeyword("or")) logical_op.setParseAction(lambda x: LogicalOp(x[0][0])) statement = Optional(comparison | invalid_comparison) + ZeroOrMore( logical_op + (comparison | invalid_comparison)) return statement
def indifferent_list(): ''' Grammar for list of indifferent attributes '[' <attribute> (, <attribute>)* ']' '(' <attribute> (, <attribute>)* ')' ''' att_term = identifier_token() indiff_par = Suppress(LEFT_PAR) + \ delimitedList(att_term, COMMA) + \ Suppress(RIGHT_PAR) indiff_bra = Suppress(LEFT_BRA) + \ delimitedList(att_term, COMMA) + \ Suppress(RIGHT_BRA) indiff_list = Group(indiff_par | indiff_bra) indiff_list.setParseAction(lambda t: t[0].asList()) return indiff_list
def get_fragment_grammar(): name = Fragment.IDENTIFIER header = Suppress("[") + Suppress("scheme") + Suppress(":") + name.setResultsName("name") + Suppress("]") # Scheme entry in the form 'sections -> target' sections = Fragment.IDENTIFIER target = Fragment.IDENTIFIER entry = Group(sections.setResultsName("sections") + Suppress("->") + target.setResultsName("target")) entries = Suppress("entries") + Suppress(":") + Group(OneOrMore(entry)).setResultsName("entries") scheme = Group(header + entries) scheme.setParseAction(lambda t: Scheme(t[0].name, t[0].entries)) scheme.ignore("#" + restOfLine) return scheme
def _initialize(self): def keywords(klass): kws = sorted(klass.keyword_aliases.keys()) return MatchFirst([Keyword(kw) for kw in kws]) def infix(klass): kws = sorted(klass.keyword_aliases.keys()) return [(kw, klass.n_terms, getattr(opAssoc, klass.assoc), klass) for kw in kws] # literals include words made of alphanumerics, numbers, # or quoted strings but we exclude any of the logical # operands (e.g. 'or') from being parsed literals literal = ( ~(keywords(BinaryInfixOperand) | keywords(UnaryInfixOperand)) + (Word(NUMS) | quotedString | Word(alphas, alphanums))) literal.setParseAction(Literal) # These are the other 'root' expressions, # the selection keywords (resname, resid, mass, etc) selection_keyword = keywords(SelectionKeyword) selection_keyword.setParseAction(SelectionKeyword) base_expression = MatchFirst([selection_keyword, literal]) # range condition matches expressions such as 'mass 1 to 20' range_condition = Group(selection_keyword + literal + Keyword('to') + literal) range_condition.setParseAction(RangeCondition) # matches expression such as `resname GLU ASP ARG` and also # handles implicit equality `resname ALA` in_list_condition = Group(selection_keyword + OneOrMore(literal)) in_list_condition.setParseAction(InListCondition) expression = range_condition | in_list_condition | base_expression logical_expr = infixNotation( expression, infix(UnaryInfixOperand) + infix(BinaryInfixOperand) + infix(RegexInfixOperand)) self.expression = logical_expr self.is_initialized = True self.transformer = _RewriteNames()
def get_fragment_grammar(): # Match header [mapping] header = Suppress("[") + Suppress("mapping") + Suppress("]") # There are three possible patterns for mapping entries: # obj:symbol (scheme) # obj (scheme) # * (scheme) obj = Fragment.ENTITY.setResultsName("object") symbol = Suppress(":") + Fragment.IDENTIFIER.setResultsName("symbol") scheme = Suppress("(") + Fragment.IDENTIFIER.setResultsName("scheme") + Suppress(")") pattern1 = Group(obj + symbol + scheme) pattern2 = Group(obj + scheme) pattern3 = Group(Literal(Mapping.MAPPING_ALL_OBJECTS).setResultsName("object") + scheme) mapping_entry = pattern1 | pattern2 | pattern3 # To simplify parsing, classify groups of condition-mapping entry into two types: normal and default # A normal grouping is one with a non-default condition. The default grouping is one which contains the # default condition mapping_entries = Group(ZeroOrMore(mapping_entry)).setResultsName("mappings") normal_condition = Suppress(":") + originalTextFor(SDKConfig.get_expression_grammar()) default_condition = Optional(Suppress(":") + Literal(Mapping.DEFAULT_CONDITION)) normal_group = Group(normal_condition.setResultsName("condition") + mapping_entries) default_group = Group(default_condition + mapping_entries).setResultsName("default_group") normal_groups = Group(ZeroOrMore(normal_group)).setResultsName("normal_groups") # Any mapping fragment definition can have zero or more normal group and only one default group as a last entry. archive = Suppress("archive") + Suppress(":") + Fragment.ENTITY.setResultsName("archive") entries = Suppress("entries") + Suppress(":") + (normal_groups + default_group).setResultsName("entries") mapping = Group(header + archive + entries) mapping.setParseAction(lambda t: Mapping(t[0].archive, t[0].entries)) mapping.ignore("#" + restOfLine) return mapping
def _getPattern(self): arith_expr = Forward() comp_expr = Forward() logic_expr = Forward() LPAR, RPAR, SEMI = map(Suppress, "();") identifier = Word(alphas+"_", alphanums+"_") multop = oneOf('* /') plusop = oneOf('+ -') expop = Literal( "^" ) compop = oneOf('> < >= <= != ==') andop = Literal("AND") orop = Literal("OR") current_value = Literal( "." ) assign = Literal( "=" ) # notop = Literal('NOT') function = oneOf(' '.join(self.FUNCTIONS)) function_call = Group(function.setResultsName('fn') + LPAR + Optional(delimitedList(arith_expr)) + RPAR) aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}') single_column = QuotedString(quoteChar='[', endQuoteChar=']') integer = Regex(r"-?\d+") real = Regex(r"-?\d+\.\d*") # quotedString enables strings without quotes to pass operand = \ function_call.setParseAction(self.__evalFunction) | \ aggregate_column.setParseAction(self.__evalAggregateColumn) | \ single_column.setParseAction(self.__evalSingleColumn) | \ ((real | integer).setParseAction(self.__evalConstant)) | \ quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \ current_value.setParseAction(self.__evalCurrentValue) | \ identifier.setParseAction(self.__evalString) arith_expr << operatorPrecedence(operand, [ (expop, 2, opAssoc.LEFT, self.__expOp), (multop, 2, opAssoc.LEFT, self.__multOp), (plusop, 2, opAssoc.LEFT, self.__addOp), ]) # comp_expr = Group(arith_expr + compop + arith_expr) comp_expr << operatorPrecedence(arith_expr, [ (compop, 2, opAssoc.LEFT, self.__evalComparisonOp), ]) logic_expr << operatorPrecedence(comp_expr, [ (andop, 2, opAssoc.LEFT, self.__evalLogicOp), (orop, 2, opAssoc.LEFT, self.__evalLogicOp) ]) pattern = logic_expr + StringEnd() return pattern
def indifferent_list(): ''' Grammar for list of indifferent attributes '[' <attribute> (, <attribute>)* ']' '(' <attribute> (, <attribute>)* ')' ''' from grammar.symbols import LEFT_BRA, RIGHT_BRA, LEFT_PAR, \ RIGHT_PAR, COMMA from grammar.basic import attribute_term att_term = attribute_term() indiff_par = Suppress(LEFT_PAR) + \ delimitedList(att_term, COMMA) + \ Suppress(RIGHT_PAR) indiff_bra = Suppress(LEFT_BRA) + \ delimitedList(att_term, COMMA) + \ Suppress(RIGHT_BRA) indiff_list = Group(indiff_par | indiff_bra) indiff_list.setParseAction(lambda t: t[0].asList()) return indiff_list
def numeric_literal() -> Token: numeral = Combine( Word(nums) + ZeroOrMore(Optional(Word("_")) + Word(nums))) numeral.setParseAction(lambda t: t[0].replace("_", "")) decimal_literal = Group(numeral) decimal_literal.setParseAction(lambda t: (int(t[0][0]), 0)) extended_digit = Word(nums + "ABCDEF") based_numeral = Combine(extended_digit + ZeroOrMore(Optional("_") + extended_digit)) based_numeral.setParseAction(lambda t: t[0].replace("_", "")) based_literal = numeral + Literal("#") - based_numeral - Literal("#") based_literal.setParseAction(lambda t: (int(t[2], int(t[0])), int(t[0]))) num_literal = based_literal | decimal_literal num_literal.setName("Number") return locatedExpr(num_literal).setParseAction(lambda s, l, t: Number( t[0][1][0], t[0][1][1], parser_location(t[0][0], t[0][2], s)))
def _getPattern(self): arith_expr = Forward() comp_expr = Forward() logic_expr = Forward() LPAR, RPAR, SEMI = map(Suppress, "();") identifier = Word(alphas + "_", alphanums + "_") multop = oneOf('* /') plusop = oneOf('+ -') expop = Literal("^") compop = oneOf('> < >= <= != ==') andop = Literal("AND") orop = Literal("OR") current_value = Literal(".") assign = Literal("=") # notop = Literal('NOT') function = oneOf(' '.join(self.FUNCTIONS)) function_call = Group( function.setResultsName('fn') + LPAR + Optional(delimitedList(arith_expr)) + RPAR) aggregate_column = QuotedString(quoteChar='{', endQuoteChar='}') single_column = QuotedString(quoteChar='[', endQuoteChar=']') integer = Regex(r"-?\d+") real = Regex(r"-?\d+\.\d*") # quotedString enables strings without quotes to pass operand = \ function_call.setParseAction(self.__evalFunction) | \ aggregate_column.setParseAction(self.__evalAggregateColumn) | \ single_column.setParseAction(self.__evalSingleColumn) | \ ((real | integer).setParseAction(self.__evalConstant)) | \ quotedString.setParseAction(self.__evalString).addParseAction(removeQuotes) | \ current_value.setParseAction(self.__evalCurrentValue) | \ identifier.setParseAction(self.__evalString) arith_expr << operatorPrecedence(operand, [ (expop, 2, opAssoc.LEFT, self.__expOp), (multop, 2, opAssoc.LEFT, self.__multOp), (plusop, 2, opAssoc.LEFT, self.__addOp), ]) # comp_expr = Group(arith_expr + compop + arith_expr) comp_expr << operatorPrecedence(arith_expr, [ (compop, 2, opAssoc.LEFT, self.__evalComparisonOp), ]) logic_expr << operatorPrecedence( comp_expr, [(andop, 2, opAssoc.LEFT, self.__evalLogicOp), (orop, 2, opAssoc.LEFT, self.__evalLogicOp)]) pattern = logic_expr + StringEnd() return pattern
def __init__(self, max_=60): # define the grammar structure digits = "0123456789" star = Literal('*') number = Word(digits) | Word(alphas) steps = number range_ = number + Optional(Literal('-') + number) numspec = star | range_ expr = Group(numspec) + Optional(Literal('/') + steps) extra_groups = ZeroOrMore(Literal(',') + expr) groups = expr + extra_groups + StringEnd() # define parse actions star.setParseAction(self._expand_star) number.setParseAction(self._expand_number) range_.setParseAction(self._expand_range) expr.setParseAction(self._filter_steps) extra_groups.setParseAction(self._ignore_comma) groups.setParseAction(self._join_to_set) self.max_ = max_ self.parser = groups
def arithmetic_term(): ''' Grammar for arithmetic expressions ''' from grammar.keywords import CURRENT_KEYWORD from grammar.symbols import MULTI_OP_SET, PLUS_OP_SET from grammar.basic import attribute_term, value_term from grammar.symbols import LEFT_PAR, RIGHT_PAR,\ AGGREGATE_FUNCTIONS_SET # Arithmetic operators multiply_op = oneOf(list(MULTI_OP_SET)) addition_op = oneOf(list(PLUS_OP_SET)) operand_token = CURRENT_KEYWORD | attribute_term() | value_term() # Arithmetic expression precedence_list = [(multiply_op, 2, opAssoc.LEFT), (addition_op, 2, opAssoc.LEFT)] expression_t = operatorPrecedence(operand_token, precedence_list) # Aggregate function aggregate_function = oneOf(list(AGGREGATE_FUNCTIONS_SET), caseless=True) arithmetic_t = Group(aggregate_function + Suppress(LEFT_PAR) + expression_t + Suppress(RIGHT_PAR)) | \ expression_t arithmetic_t.setParseAction(lambda t: t.asList()) return arithmetic_t
def parse_create_view(string): """Parse the create view sql query and return metadata Args: string(str): Input sql string that should be parsed Returns: view_data(dict): view_data dictionary for instantiating a view object """ string = rreplace(string, ')', ' )') end = ')' + StringEnd() select = Group(ZeroOrMore(~end + Word(printables))) parser = _create + replace_check.setResultsName('replace') + _view parser += _db_name.setResultsName('view_name') + _as + '(' parser += select.setParseAction(merge).setResultsName('select_statement') parser += end # Parse the base table definitions view_data = to_dict(parser.parseString(string)) return view_data
(VerbPath | VerbSimple) + ObjectList)) # [82] PropertyListPath ::= Optional(PropertyListPathNotEmpty) PropertyListPath = Optional(PropertyListPathNotEmpty) # [77] PropertyListNotEmpty ::= Verb ObjectList ( ';' ( Verb ObjectList )? )* PropertyListNotEmpty = Verb + ObjectList + ZeroOrMore( ';' + Optional(Verb + ObjectList)) # [76] PropertyList ::= Optional(PropertyListNotEmpty) PropertyList = Optional(PropertyListNotEmpty) # [99] BlankNodePropertyList ::= '[' PropertyListNotEmpty ']' BlankNodePropertyList = Group( Suppress('[') + PropertyListNotEmpty + Suppress(']')) BlankNodePropertyList.setParseAction(expandBNodeTriples) # [101] BlankNodePropertyListPath ::= '[' PropertyListPathNotEmpty ']' BlankNodePropertyListPath = Group( Suppress('[') + PropertyListPathNotEmpty + Suppress(']')) BlankNodePropertyListPath.setParseAction(expandBNodeTriples) # [98] TriplesNode ::= Collection | BlankNodePropertyList TriplesNode << (Collection | BlankNodePropertyList) # [100] TriplesNodePath ::= CollectionPath | BlankNodePropertyListPath TriplesNodePath << (CollectionPath | BlankNodePropertyListPath) # [75] TriplesSameSubject ::= VarOrTerm PropertyListNotEmpty | TriplesNode PropertyList TriplesSameSubject = VarOrTerm + PropertyListNotEmpty | TriplesNode + \ PropertyList
def parse_algebra(self): """ Parse an algebraic expression into a tree. Store a `pyparsing.ParseResult` in `self.tree` with proper groupings to reflect parenthesis and order of operations. Leave all operators in the tree and do not parse any strings of numbers into their float versions. Adding the groups and result names makes the `repr()` of the result really gross. For debugging, use something like print OBJ.tree.asXML() """ # 0.33 or 7 or .34 or 16. number_part = Word(nums) inner_number = (number_part + Optional("." + Optional(number_part))) | ("." + number_part) # pyparsing allows spaces between tokens--`Combine` prevents that. inner_number = Combine(inner_number) # SI suffixes and percent. number_suffix = MatchFirst(Literal(k) for k in SUFFIXES.keys()) # 0.33k or 17 plus_minus = Literal('+') | Literal('-') number = Group( Optional(plus_minus) + inner_number + Optional(CaselessLiteral("E") + Optional(plus_minus) + number_part) + Optional(number_suffix) ) number = number("number") # Predefine recursive variables. expr = Forward() # Handle variables passed in. They must start with a letter # and may contain numbers and underscores afterward. inner_varname = Combine(Word(alphas, alphanums + "_") + ZeroOrMore("'")) # Alternative variable name in tensor format # Tensor name must start with a letter, continue with alphanums # Indices may be alphanumeric # e.g., U_{ijk}^{123} upper_indices = Literal("^{") + Word(alphanums) + Literal("}") lower_indices = Literal("_{") + Word(alphanums) + Literal("}") tensor_lower = Combine(Word(alphas, alphanums) + lower_indices + ZeroOrMore("'")) tensor_mixed = Combine(Word(alphas, alphanums) + Optional(lower_indices) + upper_indices + ZeroOrMore("'")) # Test for mixed tensor first, then lower tensor alone, then generic variable name varname = Group(tensor_mixed | tensor_lower | inner_varname)("variable") varname.setParseAction(self.variable_parse_action) # Same thing for functions. function = Group(inner_varname + Suppress("(") + expr + Suppress(")"))("function") function.setParseAction(self.function_parse_action) atom = number | function | varname | "(" + expr + ")" atom = Group(atom)("atom") # Do the following in the correct order to preserve order of operation. pow_term = atom + ZeroOrMore("^" + atom) pow_term = Group(pow_term)("power") par_term = pow_term + ZeroOrMore('||' + pow_term) # 5k || 4k par_term = Group(par_term)("parallel") prod_term = par_term + ZeroOrMore((Literal('*') | Literal('/')) + par_term) # 7 * 5 / 4 prod_term = Group(prod_term)("product") sum_term = Optional(plus_minus) + prod_term + ZeroOrMore(plus_minus + prod_term) # -5 + 4 - 3 sum_term = Group(sum_term)("sum") # Finish the recursion. expr << sum_term # pylint: disable=pointless-statement self.tree = (expr + stringEnd).parseString(self.math_expr)[0]
def _parse_study_search_string(self, searchstr, only_with_processed_data=False): """parses string into SQL query for study search Parameters ---------- searchstr : str The string to parse only_with_processed_data : bool Whether or not to return studies with processed data. Returns ------- study_sql : str SQL query for selecting studies with the required metadata columns sample_sql : str SQL query for each study to get the sample ids that mach the query meta_headers : list metadata categories in the query string in alphabetical order Notes ----- All searches are case-sensitive References ---------- .. [1] McGuire P (2007) Getting started with pyparsing. """ # build the parse grammar category = Word(alphas + nums + "_") seperator = oneOf("> < = >= <= !=") | CaselessLiteral("includes") | \ CaselessLiteral("startswith") value = Word(alphas + nums + "_" + ":" + ".") | \ dblQuotedString().setParseAction(removeQuotes) criterion = Group(category + seperator + value) criterion.setParseAction(SearchTerm) and_ = CaselessLiteral("and") or_ = CaselessLiteral("or") not_ = CaselessLiteral("not") optional_seps = Optional(and_ | or_ | not_) # create the grammar for parsing operators AND, OR, NOT search_expr = operatorPrecedence( criterion, [ (not_, 1, opAssoc.RIGHT, SearchNot), (and_, 2, opAssoc.LEFT, SearchAnd), (or_, 2, opAssoc.LEFT, SearchOr)]) # parse the search string to get out the SQL WHERE formatted query eval_stack = (search_expr + stringEnd).parseString(searchstr)[0] sql_where = eval_stack.generate_sql() # this lookup will be used to select only studies with columns # of the correct type type_lookup = {int: 'integer', float: 'float8', str: 'varchar'} # parse out all metadata headers we need to have in a study, and # their corresponding types all_headers = [c[0][0].term[0] for c in (criterion + optional_seps).scanString(searchstr)] meta_headers = set(all_headers) all_types = [c[0][0].term[2] for c in (criterion + optional_seps).scanString(searchstr)] all_types = [type_lookup[type(typecast_string(s))] for s in all_types] # sort headers and types so they return in same order every time. # Should be a relatively short list so very quick # argsort implementation taken from # http://stackoverflow.com/questions/3382352/ # equivalent-of-numpy-argsort-in-basic-python sort_order = sorted(range(len(all_headers)), key=all_headers.__getitem__) all_types = [all_types[x] for x in sort_order] all_headers.sort() # At this point it is possible that a metadata header has been # reference more than once in the query. If the types agree, then we # do not need to do anything. If the types do not agree (specifically, # if it appears to be numerical in one case and string in another), # then we need to give varchar the precedence. meta_header_type_lookup = dict() for header, header_type in zip(all_headers, all_types): if header not in meta_header_type_lookup: meta_header_type_lookup[header] = header_type else: if header_type == 'varchar' or \ meta_header_type_lookup[header] == 'varchar': meta_header_type_lookup[header] = 'varchar' # create the study finding SQL # remove metadata headers that are in required_sample_info table meta_headers = meta_headers.difference(self.required_cols).difference( self.study_cols) # get all study ids that contain all metadata categories searched for sql = [] if meta_headers: # have study-specific metadata, so need to find specific studies for meta in meta_headers: if meta_header_type_lookup[meta] in ('integer', 'float8'): allowable_types = "('integer', 'float8')" else: allowable_types = "('varchar')" sql.append("SELECT study_id FROM qiita.study_sample_columns " "WHERE lower(column_name) = lower('%s') and " "column_type in %s" % (scrub_data(meta), allowable_types)) else: # no study-specific metadata, so need all studies sql.append("SELECT study_id FROM qiita.study_sample_columns") # combine the query if only_with_processed_data: sql.append('SELECT study_id FROM qiita.study_processed_data') study_sql = ' INTERSECT '.join(sql) # create the sample finding SQL, getting both sample id and values # build the sql formatted list of metadata headers header_info = [] for meta in meta_header_type_lookup: if meta in self.required_cols: header_info.append("r.%s" % meta) elif meta in self.study_cols: header_info.append("st.%s" % meta) else: header_info.append("sa.%s" % meta) # build the SQL query sample_sql = ("SELECT r.sample_id,%s FROM qiita.required_sample_info " "r JOIN qiita.sample_{0} sa ON sa.sample_id = " "r.sample_id JOIN qiita.study st ON st.study_id = " "r.study_id WHERE %s" % (','.join(header_info), sql_where)) return study_sql, sample_sql, meta_header_type_lookup.keys()
ternaryop = ('?', ':') current_scope_operand.setParseAction(EvalCurrentScope) variable_operand.setParseAction(EvalVariable) explicit_variable_operand.setParseAction(EvalExplicitVariable) integer_operand.setParseAction(EvalInteger) real_operand.setParseAction(EvalReal) string_operand.setParseAction(EvalString) constant.setParseAction(EvalConstant) regexp.setParseAction(EvalRegExp) timespan.setParseAction(EvalTimespan) modifier = Regex(r'([a-zA-Z][a-zA-Z0-9_]*)\:') simple_list_operand = Group(delimitedList(expr)) simple_list_operand.setParseAction(EvalSimpleList) list_operand = (Suppress('[') + delimitedList(expr) + Suppress(']')) list_operand.setParseAction(EvalList) empty_list_operand = Literal('[]') empty_list_operand.setParseAction(EvalEmptyList) dict_item = Group(expr + Suppress(Literal(':')) + expr) dict_operand = Group(Suppress('{') + delimitedList(dict_item) + Suppress('}')) dict_operand.setParseAction(EvalDict) empty_dict_operand = Literal('{}') empty_dict_operand.setParseAction(EvalEmptyDict) key_pair = Group(Regex(r'([a-zA-Z0-9_]+)') + Suppress(Literal('=') + WordEnd('=!+-*/')) + expr)
>>> print join(['a', 'long', 'long', 'road']) a~long long~road >>> print join(['very', 'long', 'phrase']) very long~phrase """ if len(words) <= 2: return tie.join(words) else: return (words[0] + tie_or_space(words[0], tie, space) + space.join(words[1:-1]) + tie + words[-1]) def format(name, format): return NameFormat(format).format(name) lbrace = Literal('{') rbrace = Literal('}') format_chars = Word(alphas) braced_string = Forward() braced_string << Combine(lbrace + ZeroOrMore(CharsNotIn('{}')| braced_string) + rbrace) verbatim = Combine(ZeroOrMore(CharsNotIn(alphas + '{}') | braced_string)) delimiter = braced_string.copy().setParseAction(removeQuotes) group = Group(Suppress(lbrace) + verbatim + format_chars + Optional(delimiter, None) + verbatim + Suppress(rbrace)) group.setParseAction(lambda toks: NamePart(toks)) toplevel_text = CharsNotIn('{}').setParseAction(lambda toks: Text(toks)) name_format_grammar = ZeroOrMore(toplevel_text | group) + StringEnd() name_format_grammar.leaveWhitespace()
ft = float(t) it = int(ft) return it if it == ft else ft nreal.setParseAction(make_int) def multiply_contents(tokens): t = tokens[0] if t.subgroup: mult = t.mult sg = next(subGroupCount) # is needed to calculate wt% for term in t.subgroup: term[1] *= mult term.append(sg) return t.subgroup term.setParseAction(multiply_contents) def sum_by_element(tokens): nt = tokens dopes = [] for t in nt: if isinstance(t[1], complex): sg = t[2] if len(t) > 2 else next(subGroupCount) dopes.append([t, sg]) if dopes: matrix = [[t[0], t[1]] for t in nt if isinstance(t[1], (float, int))] matrixMass = sum([read_atomic_data(e[0])*e[1] for e in matrix]) dopeGroups = [] for e in dopes: if not (e[1] in dopeGroups):
def graph_definition(): global graphparser if not graphparser: # punctuation colon = Literal(":") lbrace = Literal("{") rbrace = Literal("}") lbrack = Literal("[") rbrack = Literal("]") lparen = Literal("(") rparen = Literal(")") equals = Literal("=") comma = Literal(",") dot = Literal(".") slash = Literal("/") bslash = Literal("\\") star = Literal("*") semi = Literal(";") at = Literal("@") minus = Literal("-") # keywords strict_ = Literal("strict") graph_ = Literal("graph") digraph_ = Literal("digraph") subgraph_ = Literal("subgraph") node_ = Literal("node") edge_ = Literal("edge") # token definitions identifier = Word(alphanums + "_" ).setName("identifier") double_quoted_string = dblQuotedString alphastring_ = OneOrMore(CharsNotIn(_noncomma)) ID = (identifier | double_quoted_string.setParseAction(strip_quotes) |\ alphastring_).setName("ID") html_text = Combine(Literal("<<") + OneOrMore(CharsNotIn(",]"))) float_number = Combine(Optional(minus) + \ OneOrMore(Word(nums + "."))).setName("float_number") righthand_id = (float_number | ID | html_text).setName("righthand_id") port_angle = (at + ID).setName("port_angle") port_location = (Group(colon + ID) | \ Group(colon + lparen + ID + comma + ID + rparen)).setName("port_location") port = (Group(port_location + Optional(port_angle)) | \ Group(port_angle + Optional(port_location))).setName("port") node_id = (ID + Optional(port)) a_list = OneOrMore(ID + Optional(equals.suppress() + righthand_id) + \ Optional(comma.suppress())).setName("a_list") attr_list = OneOrMore(lbrack.suppress() + Optional(a_list) + \ rbrack.suppress()).setName("attr_list") attr_stmt = (Group(graph_ | node_ | edge_) + attr_list).setName("attr_stmt") edgeop = (Literal("--") | Literal("->")).setName("edgeop") stmt_list = Forward() graph_stmt = Group(lbrace.suppress() + Optional(stmt_list) + \ rbrace.suppress()).setName("graph_stmt") subgraph = (Group(Optional(subgraph_ + Optional(ID)) + graph_stmt) | \ Group(subgraph_ + ID)).setName("subgraph") edgeRHS = OneOrMore(edgeop + Group(node_id | subgraph)) edge_stmt = Group(node_id | subgraph) + edgeRHS + Optional(attr_list) node_stmt = (node_id + Optional(attr_list) + Optional(semi.suppress())).setName("node_stmt") assignment = (ID + equals.suppress() + righthand_id).setName("assignment") stmt = (assignment | edge_stmt | attr_stmt | subgraph | node_stmt).setName("stmt") stmt_list << OneOrMore(stmt + Optional(semi.suppress())) graphparser = (Optional(strict_) + Group((graph_ | digraph_)) + \ Optional(ID) + graph_stmt).setResultsName("graph") singleLineComment = "//" + restOfLine # actions graphparser.ignore(singleLineComment) graphparser.ignore(cStyleComment) assignment.setParseAction(push_attr_list) a_list.setParseAction(push_attr_list) edge_stmt.setParseAction(push_edge_stmt) node_stmt.setParseAction(push_node_stmt) attr_stmt.setParseAction(push_default_stmt) subgraph.setParseAction(push_subgraph_stmt) graph_stmt.setParseAction(push_graph_stmt) graphparser.setParseAction(push_top_graph_stmt) return graphparser
L("~=") | L(">") | L("<") ) MARKER_OP = VERSION_CMP | L("not in") | L("in") MARKER_VALUE = QuotedString("'") | QuotedString('"') MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0])) BOOLOP = L("and") | L("or") MARKER_VAR = VARIABLE | MARKER_VALUE MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) LPAREN = L("(").suppress() RPAREN = L(")").suppress() MARKER_EXPR = Forward() MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN) MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR) MARKER = stringStart + MARKER_EXPR + stringEnd def _coerce_parse_result(results): if isinstance(results, ParseResults): return [_coerce_parse_result(i) for i in results] else:
(VerbPath | VerbSimple) + ObjectListPath + ZeroOrMore(";" + Optional((VerbPath | VerbSimple) + ObjectList)) ) # [82] PropertyListPath ::= Optional(PropertyListPathNotEmpty) PropertyListPath = Optional(PropertyListPathNotEmpty) # [77] PropertyListNotEmpty ::= Verb ObjectList ( ';' ( Verb ObjectList )? )* PropertyListNotEmpty = Verb + ObjectList + ZeroOrMore(";" + Optional(Verb + ObjectList)) # [76] PropertyList ::= Optional(PropertyListNotEmpty) PropertyList = Optional(PropertyListNotEmpty) # [99] BlankNodePropertyList ::= '[' PropertyListNotEmpty ']' BlankNodePropertyList = Group(Suppress("[") + PropertyListNotEmpty + Suppress("]")) BlankNodePropertyList.setParseAction(expandBNodeTriples) # [101] BlankNodePropertyListPath ::= '[' PropertyListPathNotEmpty ']' BlankNodePropertyListPath = Group(Suppress("[") + PropertyListPathNotEmpty + Suppress("]")) BlankNodePropertyListPath.setParseAction(expandBNodeTriples) # [98] TriplesNode ::= Collection | BlankNodePropertyList TriplesNode << (Collection | BlankNodePropertyList) # [100] TriplesNodePath ::= CollectionPath | BlankNodePropertyListPath TriplesNodePath << (CollectionPath | BlankNodePropertyListPath) # [75] TriplesSameSubject ::= VarOrTerm PropertyListNotEmpty | TriplesNode PropertyList TriplesSameSubject = VarOrTerm + PropertyListNotEmpty | TriplesNode + PropertyList TriplesSameSubject.setParseAction(expandTriples)
ident = Word(alphas, alphanums + "_$" ) ^ QuotedString('"') ^ QuotedString("`") ident.setParseAction(lambda toks: ['"%s"' % toks[0]]) string = QuotedString("'",multiline=True) string.setParseAction(lambda toks: ["'%s'" % toks[0]]) columnName = delimitedList( ident, ".",combine=True) tableName = delimitedList( ident, ".",combine=True) dataType = Word(alphas) + Combine(Optional(Literal("(") + (Word(nums) ^ delimitedList(string,combine=True)) + Literal(")"))) + ZeroOrMore(nnToken ^ autoincrementToken ^ (defaultToken + (string ^ nullToken)) ^ unsignedToken.suppress() ) dataType.setParseAction(convert_datatypes) columnDescription = Group(ident + dataType) keyDescription = Optional(primaryToken ^ uniqueToken) + keyToken + Optional(ident) + Literal("(") + delimitedList(ident + Optional(Literal("(") + Word(nums) + Literal(")"))) + Literal(")") createTableStmt = Group(createToken + tableToken + ifneToken + ident + Literal("(")) + delimitedList(columnDescription ^ keyDescription.suppress()) + Group(Literal(")")) + Optional(autoincrementToken + Literal("=") + Word(nums)).suppress() createTableStmt.setParseAction(rebuild_createtable) createDataBaseStmt = Group(createToken + databaseToken + ident + dcsToken + Word(alphanums)+ collateToken + ident) useStmt = Group(useToken + ident) comment = LineStart() + CharsNotIn("\n") + LineEnd() value = Group(Literal("(") + delimitedList(Word(nums) ^ string) + Literal(")")) insertPrefix = Group(insertToken + intoToken + ident + Literal("(") + delimitedList(ident) + Literal(")") + valuesToken) insertStmt = insertPrefix + delimitedList(value) insertStmt.setParseAction(rebuild_insert)
def get_grammar(self): """ Defines our grammar for mathematical expressions. Possibly helpful: - BNF form of context-free grammar https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_form - Some pyparsing docs http://infohost.nmt.edu/~shipman/soft/pyparsing/web/index.html """ # Define + and - plus = Literal("+") # Also accept unicode emdash emdash = Literal("\u2014") emdash.setParseAction(lambda: "-") minus = Literal("-") | emdash plus_minus = plus | minus # 1 or 1.0 or .1 number_part = Word(nums) inner_number = Combine((number_part + Optional("." + Optional(number_part))) | ("." + number_part)) # Combine() joints the matching parts together in a single token, # and requires that the matching parts be contiguous (no spaces) # Define our suffixes suffix = Word(alphas + '%') suffix.setParseAction(self.suffix_parse_action) # Construct number as a group consisting of a text string ("num") and an optional suffix. # num can include a decimal number and numerical exponent, and can be # converted to a number using float() # suffix may contain alphas or % # Spaces are ignored inside numbers # Group wraps everything up into its own ParseResults object when parsing number = Group( Combine( inner_number + Optional( CaselessLiteral("E") + Optional(plus_minus) + number_part), )("num") + Optional(suffix)("suffix"))("number") # Note that calling ("name") on the end of a parser is equivalent to calling # parser.setResultsName, which is used to pull that result out of a parsed # expression like a dictionary. # Construct variable and function names front = Word(alphas, alphanums) # must start with alpha subscripts = Word(alphanums + '_') + ~FollowedBy('{') # ~ = not lower_indices = Literal("_{") + Optional("-") + Word( alphanums) + Literal("}") upper_indices = Literal("^{") + Optional("-") + Word( alphanums) + Literal("}") # Construct an object name in either of two forms: # 1. front + subscripts + tail # 2. front + lower_indices + upper_indices + tail # where: # front (required): # starts with alpha, followed by alphanumeric # subscripts (optional): # any combination of alphanumeric and underscores # lower_indices (optional): # Of form "_{(-)<alphanumeric>}" # upper_indices (optional): # Of form "^{(-)<alphanumeric>}" # tail (optional): # any number of primes name = Combine(front + Optional(subscripts | (Optional(lower_indices) + Optional(upper_indices))) + ZeroOrMore("'")) # Define a variable as a pyparsing result that contains one object name variable = Group(name("varname"))("variable") variable.setParseAction(self.variable_parse_action) # initialize recursive grammar expression = Forward() # Construct functions as consisting of funcname and arguments as # funcname(arguments) # where arguments is a comma-separated list of arguments, returned as a list # Must have at least 1 argument function = Group( name("funcname") + Suppress("(") + Group(delimitedList(expression))("arguments") + Suppress(")"))("function") function.setParseAction(self.function_parse_action) # Define parentheses parentheses = Group(Suppress("(") + expression + Suppress(")"))('parentheses') # Define arrays array = Group( Suppress("[") + delimitedList(expression) + Suppress("]"))("array") # atomic units evaluate directly to number or array without binary operations atom = number | function | variable | parentheses | array # Define operations in order of precedence # Define exponentiation, possibly including negative powers power = atom + ZeroOrMore(Suppress("^") + Optional(minus)("op") + atom) power.addParseAction(self.group_if_multiple('power')) # Define negation (e.g., in 5*-3 --> we need to evaluate the -3 first) # Negation in powers is handled separately # This has been arbitrarily assigned a higher precedence than parallel negation = Optional(minus)("op") + power negation.addParseAction(self.group_if_multiple('negation')) # Define the parallel operator 1 || 5 == 1/(1/1 + 1/5) pipes = Literal('|') + Literal('|') parallel = negation + ZeroOrMore(Suppress(pipes) + negation) parallel.addParseAction(self.group_if_multiple('parallel')) # Define multiplication and division product = parallel + ZeroOrMore((Literal('*') | Literal('/'))("op") + parallel) product.addParseAction(self.group_if_multiple('product')) # Define sums and differences # Note that leading - signs are treated by negation sumdiff = Optional(plus) + product + ZeroOrMore( plus_minus("op") + product) sumdiff.addParseAction(self.group_if_multiple('sum')) # Close the recursion expression << sumdiff return expression + stringEnd
fIdent = Word("f", alphanums) fIdent.setParseAction(mkIdent) gIdent = Word("g", alphanums) gIdent.setParseAction(mkIdent) LPAR = Suppress("(") RPAR = Suppress(")") EQ = Suppress("=") SC = Suppress(";") exp = Forward(); variable = Group(lIdent) variable.setParseAction(mkVar) ctrArgList = Group(Optional(LPAR + Optional(delimitedList(exp)) + RPAR)) constructor = uIdent + ctrArgList constructor.setParseAction(lambda t: Ctr(t[0], list(t[1]))) fArgList = Group(LPAR + Optional(delimitedList(exp)) + RPAR) fCall = fIdent + fArgList fCall.setParseAction(lambda t: FCall(t[0], list(t[1]))) gArgList = Group(LPAR + delimitedList(exp) + RPAR) gCall = gIdent + gArgList
RETURNS + LPAR + Optional(identifier("response")) + RPAR))("method_definition") method_definition.setParseAction(method_definition_fn) service_definition= (SERVICE - identifier("service") + LBRACE + ZeroOrMore(Group(method_definition)) + RBRACE)("service_definition") service_definition.setParseAction(service_definition_fn) package_directive = (Group(PACKAGE - delimitedList(identifier, '.', combine=True) + SEMI))("package_directive") package_directive.setParseAction(package_directive_fn) import_directive = IMPORT - quotedString("import") + SEMI import_directive.setParseAction(import_directive_fn) option_directive = OPTION - identifier("option") + EQ + (integer | quotedString)("value") + SEMI top_level_statement = Group(message_definition| enum_definition| option_directive | import_directive | service_definition)("top_level_statement") top_level_statement.setParseAction(top_level_statement_fn) syntax_directive = (SYNTAX + EQ + quotedString("syntax_version") + SEMI)("syntax_directive") parser = (Optional(syntax_directive) + Optional(package_directive) + ZeroOrMore(top_level_statement))("parser") parser.setParseAction(parser_fn) comment = '//' + restOfLine parser.ignore(comment) test2 = """message Person { required uint32 id = 1; required string name = 2; optional string email = 3; optional Person person = 4; }"""
for formula in ( "H2O", "NaCl", "C6H5OH", "H2SO4" ): formulaData = chemicalFormula.parseString(formula) # compute molecular weight mw = sum( [ atomicWeight[element.symbol]*element.qty for element in formulaData ] ) # print the results print formula, "->", formulaData, "(", mw, ")" print # Version 3 - Compute partial molecular weight per element, simplifying # summing # No need to redefine grammar, just define parse action function, and # attach to elementRef def computeElementWeight(tokens): element = tokens[0] element["weight"] = atomicWeight[element.symbol] * element.qty elementRef.setParseAction(computeElementWeight) for formula in ( "H2O", "NaCl", "C6H5OH", "H2SO4" ): formulaData = chemicalFormula.parseString(formula) # compute molecular weight mw = sum( [ element.weight for element in formulaData ] ) # print the results print formula, "->", formulaData, "(", mw, ")"
def graph_definition(): global graphparser if not graphparser: # punctuation colon = Literal(":") lbrace = Literal("{") rbrace = Literal("}") lbrack = Literal("[") rbrack = Literal("]") lparen = Literal("(") rparen = Literal(")") equals = Literal("=") comma = Literal(",") dot = Literal(".") slash = Literal("/") bslash = Literal("\\") star = Literal("*") semi = Literal(";") at = Literal("@") minus = Literal("-") # keywords strict_ = CaselessLiteral("strict") graph_ = CaselessLiteral("graph") digraph_ = CaselessLiteral("digraph") subgraph_ = CaselessLiteral("subgraph") node_ = CaselessLiteral("node") edge_ = CaselessLiteral("edge") # token definitions identifier = Word(alphanums + "_." ).setName("identifier") double_quoted_string = QuotedString('"', multiline=True, unquoteResults=False) # dblQuotedString _noncomma = "".join([c for c in printables if c != ","]) alphastring_ = OneOrMore(CharsNotIn(_noncomma + ' ')) def parse_html(s, loc, toks): return '<%s>' % ''.join(toks[0]) opener = '<' closer = '>' html_text = nestedExpr( opener, closer, ( CharsNotIn( opener + closer ) ) ).setParseAction(parse_html).leaveWhitespace() ID = ( identifier | html_text | double_quoted_string | #.setParseAction(strip_quotes) | alphastring_ ).setName("ID") float_number = Combine(Optional(minus) + OneOrMore(Word(nums + "."))).setName("float_number") righthand_id = (float_number | ID ).setName("righthand_id") port_angle = (at + ID).setName("port_angle") port_location = (OneOrMore(Group(colon + ID)) | Group(colon + lparen + ID + comma + ID + rparen)).setName("port_location") port = (Group(port_location + Optional(port_angle)) | Group(port_angle + Optional(port_location))).setName("port") node_id = (ID + Optional(port)) a_list = OneOrMore(ID + Optional(equals + righthand_id) + Optional(comma.suppress())).setName("a_list") attr_list = OneOrMore(lbrack.suppress() + Optional(a_list) + rbrack.suppress()).setName("attr_list") attr_stmt = (Group(graph_ | node_ | edge_) + attr_list).setName("attr_stmt") edgeop = (Literal("--") | Literal("->")).setName("edgeop") stmt_list = Forward() graph_stmt = Group(lbrace.suppress() + Optional(stmt_list) + rbrace.suppress() + Optional(semi.suppress()) ).setName("graph_stmt") edge_point = Forward() edgeRHS = OneOrMore(edgeop + edge_point) edge_stmt = edge_point + edgeRHS + Optional(attr_list) subgraph = Group(subgraph_ + Optional(ID) + graph_stmt).setName("subgraph") edge_point << Group(subgraph | graph_stmt | node_id ).setName('edge_point') node_stmt = (node_id + Optional(attr_list) + Optional(semi.suppress())).setName("node_stmt") assignment = (ID + equals + righthand_id).setName("assignment") stmt = (assignment | edge_stmt | attr_stmt | subgraph | graph_stmt | node_stmt).setName("stmt") stmt_list << OneOrMore(stmt + Optional(semi.suppress())) graphparser = OneOrMore( (Optional(strict_) + Group((graph_ | digraph_)) + Optional(ID) + graph_stmt).setResultsName("graph") ) singleLineComment = Group("//" + restOfLine) | Group("#" + restOfLine) # actions graphparser.ignore(singleLineComment) graphparser.ignore(cStyleComment) assignment.setParseAction(push_attr_list) a_list.setParseAction(push_attr_list) edge_stmt.setParseAction(push_edge_stmt) node_stmt.setParseAction(push_node_stmt) attr_stmt.setParseAction(push_default_stmt) subgraph.setParseAction(push_subgraph_stmt) graph_stmt.setParseAction(push_graph_stmt) graphparser.setParseAction(push_top_graph_stmt) return graphparser
def __init__(self, ffilter = None, filter_string = None): if PYPARSING: quoted_str_value = QuotedString('\'', unquoteResults=True, escChar='\\') int_values = Word("0123456789") error_value = Literal("XXX").setParseAction(self.__compute_xxx_value) bbb_value = Literal("BBB").setParseAction(self.__compute_bbb_value) field_value = Word(alphas + "." + "_" + "-") basic_primitives = int_values | quoted_str_value operator_names = oneOf("m d e un u r l sw unique startswith decode encode unquote replace lower upper").setParseAction(lambda s,l,t: [ (l,t[0])]) fuzz_symbol = (Suppress("FUZ") + Optional(Word("23456789"), 1).setParseAction( lambda s,l,t: [ int(t[0])-1 ] ) + Suppress("Z")).setParseAction(self.__compute_fuzz_symbol) operator_call = Group(Suppress("|") + operator_names + Suppress("(") + Optional(basic_primitives, None) + Optional(Suppress(",") + basic_primitives, None) + Suppress(")")) fuzz_value = (fuzz_symbol + Optional(Suppress("[") + field_value + Suppress("]"), None)).setParseAction(self.__compute_fuzz_value) fuzz_value_op = ((fuzz_symbol + Suppress("[") + Optional(field_value)).setParseAction(self.__compute_fuzz_value) + operator_call + Suppress("]")).setParseAction(self.__compute_perl_value) fuzz_value_op2 = ((fuzz_symbol + operator_call).setParseAction(self.__compute_perl_value)) res_value_op = (Word(alphas + "." + "_" + "-").setParseAction(self.__compute_res_value) + Optional(operator_call, None)).setParseAction(self.__compute_perl_value) basic_primitives_op = (basic_primitives + Optional(operator_call, None)).setParseAction(self.__compute_perl_value) fuzz_statement = fuzz_value ^ fuzz_value_op ^ fuzz_value_op2 ^ res_value_op ^ basic_primitives_op operator = oneOf("and or") not_operator = Optional(oneOf("not"), "notpresent") symbol_expr = Group(fuzz_statement + oneOf("= != < > >= <= =~ !~ ~") + (bbb_value ^ error_value ^ fuzz_statement ^ basic_primitives)).setParseAction(self.__compute_expr) definition = fuzz_statement ^ symbol_expr definition_not = not_operator + definition definition_expr = definition_not + ZeroOrMore( operator + definition_not) nested_definition = Group(Suppress("(") + definition_expr + Suppress(")")) nested_definition_not = not_operator + nested_definition self.finalformula = (nested_definition_not ^ definition_expr) + ZeroOrMore( operator + (nested_definition_not ^ definition_expr)) definition_not.setParseAction(self.__compute_not_operator) nested_definition_not.setParseAction(self.__compute_not_operator) nested_definition.setParseAction(self.__compute_formula) self.finalformula.setParseAction(self.__myreduce) if ffilter is not None and filter_string is not None: raise FuzzExceptInternalError(FuzzException.FATAL, "A filter must be initilized with a filter string or an object, not both") self.res = None if ffilter: self.hideparams = ffilter else: self.hideparams = dict( regex_show = None, codes_show = None, codes = [], words = [], lines = [], chars = [], regex = None, filter_string = "" ) if filter_string: self.hideparams['filter_string'] = filter_string self.baseline = None self.stack = {} self._cache = collections.defaultdict(set)