def getchunk(): """ Using pyparsing, create chunk reader for chunk strings. """ slot = pp.Word("".join([pp.alphas, "_"]), "".join([pp.alphanums, "_"])) special_value = pp.Group(pp.oneOf([ACTRVARIABLE, "".join([ACTRNEG, ACTRVARIABLE]), ACTRNEG, VISIONGREATER, VISIONSMALLER, "".join([VISIONGREATER, ACTRVARIABLE]), "".join([VISIONSMALLER, ACTRVARIABLE])])\ + pp.Word("".join([pp.alphanums, "_", '"', "'"]))) strvalue = pp.QuotedString('"', unquoteResults=False) strvalue2 = pp.QuotedString("'", unquoteResults=False) varvalue = pp.Word("".join([pp.alphanums, "_"])) value = varvalue | special_value | strvalue | strvalue2 chunk_reader = pp.OneOrMore(pp.Group(slot + value)) return chunk_reader
def build_attribute_parser(): quoted = pp.QuotedString('"') ^ pp.QuotedString("'") colon = pp.Literal(':').suppress() attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]')).setResultsName('name') data_type = pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted)).setResultsName( 'type') default = pp.Literal('=').suppress() + pp.SkipTo( colon, ignore=quoted).setResultsName('default') comment = pp.Literal('#').suppress() + pp.restOfLine.setResultsName( 'comment') return attribute_name + pp.Optional(default) + colon + data_type + comment
def craft_parse(text): """ """ LineComment = pyp.Combine(pyp.Literal('::') + pyp.restOfLine).suppress() BlockComment = pyp.Combine( pyp.Literal(':>') + pyp.SkipTo(pyp.Literal('<:')) + pyp.Literal('<:')).suppress() Comment = BlockComment | LineComment BlockComment = pyp.Combine( pyp.Literal(':<') + pyp.Combine( pyp.NotAny(pyp.Literal(':>')) + pyp.Word(pyp.printables + ' ')) + pyp.Literal('>:')) Identifier = pyp.Word(pyp.alphanums + '!#$%&()*+,./;<=>?@\\^-_`{|}~') Value = (Comment | pyp.QuotedString('"') | pyp.QuotedString("'") | Identifier.addParseAction(_type_cast_value)) LBRACKET, RBRACKET, COLON = map(pyp.Suppress, '[]:') Function = pyp.Forward() List = pyp.Forward() Function << pyp.Dict( pyp.Group(Identifier + pyp.Literal(':') + pyp.Group(LBRACKET + pyp.ZeroOrMore(Comment | Function | List | Value) + RBRACKET))) List << pyp.Group(LBRACKET + pyp.ZeroOrMore(Comment | Function | List | Value) + RBRACKET) Program = pyp.OneOrMore(Comment | Function) # Validate for syntax error messages: validator = SourceValidator() Value.setParseAction(validator.validate) List.setParseAction(validator.validate) Identifier.addParseAction(validator.validate) #Comment.setParseAction(validator.validate) Function.setParseAction(validator.validate) Program.setParseAction(validator.validate) syntax_error = None try: return __walk(Program.parseString(text)[0]) except Exception as e: syntax_error = validator.panic() # Now raise the exception with a clean stack trace raise syntax_error
def _make(): # Order is important - multi-char expressions need to come before narrow # ones. parts = [] for cls in filter_unary: f = pp.Literal(f"~{cls.code}") + pp.WordEnd() f.setParseAction(cls.make) parts.append(f) # This is a bit of a hack to simulate Word(pyparsing_unicode.printables), # which has a horrible performance with len(pyparsing.pyparsing_unicode.printables) == 1114060 unicode_words = pp.CharsNotIn("()~'\"" + pp.ParserElement.DEFAULT_WHITE_CHARS) unicode_words.skipWhitespace = True regex = ( unicode_words | pp.QuotedString('"', escChar='\\') | pp.QuotedString("'", escChar='\\') ) for cls in filter_rex: f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + regex.copy() f.setParseAction(cls.make) parts.append(f) for cls in filter_int: f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + pp.Word(pp.nums) f.setParseAction(cls.make) parts.append(f) # A naked rex is a URL rex: f = regex.copy() f.setParseAction(FUrl.make) parts.append(f) atom = pp.MatchFirst(parts) expr = pp.infixNotation( atom, [(pp.Literal("!").suppress(), 1, pp.opAssoc.RIGHT, lambda x: FNot(*x)), (pp.Literal("&").suppress(), 2, pp.opAssoc.LEFT, lambda x: FAnd(*x)), (pp.Literal("|").suppress(), 2, pp.opAssoc.LEFT, lambda x: FOr(*x)), ]) expr = pp.OneOrMore(expr) return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
def build_attribute_parser(): quoted = pp.QuotedString('"') ^ pp.QuotedString("'") colon = pp.Literal(":").suppress() attribute_name = pp.Word(pp.srange("[a-z]"), pp.srange("[a-z0-9_]")).setResultsName("name") data_type = ( pp.Combine(pp.Word(pp.alphas) + pp.SkipTo("#", ignore=quoted)) ^ pp.QuotedString("<", endQuoteChar=">", unquoteResults=False)).setResultsName("type") default = pp.Literal("=").suppress() + pp.SkipTo( colon, ignore=quoted).setResultsName("default") comment = pp.Literal("#").suppress() + pp.restOfLine.setResultsName( "comment") return attribute_name + pp.Optional(default) + colon + data_type + comment
def get_guild_role_parser(self, guild): """ Create a role parser for the specified guild. :param guild: :return: """ intersect = pp.CaselessKeyword(self.INTERSECT_TOKEN) union = pp.CaselessKeyword(self.UNION_TOKEN) complement = pp.CaselessKeyword(self.COMPLEMENT_TOKEN) left_paren = pp.Literal(self.LEFT_PAREN_TOKEN) right_paren = pp.Literal(self.RIGHT_PAREN_TOKEN) role = pp.Word(pp.alphanums) | pp.QuotedString("'", escChar="\\") expression = pp.Forward() term = pp.Forward() factor = pp.Forward() factor <<= left_paren + expression + pp.FollowedBy( right_paren) + right_paren | complement + factor | role term <<= factor + intersect + term | factor expression <<= term + union + expression | term factor.setParseAction(self.get_guild_factor_action(guild)) term.setParseAction(self.get_guild_term_action()) expression.setParseAction(self.expression_action) role_statement = pp.StringStart() + expression + pp.StringEnd() return role_statement
def get_parser(): "Construct and return the parser." left = pp.Suppress("(") right = pp.Suppress(")") value = pp.QuotedString(quote_char='"', esc_char="\\") | pp.CharsNotIn(")") identifier = pp.Word(pp.alphas, pp.alphanums).set_parse_action(_Identifier) label = (pp.Keyword("label") + left + value + right).set_parse_action(_Label) year = (pp.Keyword("year") + left + value + right).set_parse_action(_Year) author = (pp.Keyword("author") + left + value + right).set_parse_action(_Author) orcid = (pp.Keyword("orcid") + left + value + right).set_parse_action(_Orcid) issn = (pp.Keyword("issn") + left + value + right).set_parse_action(_Issn) published = (pp.Keyword("published") + left + value + right).set_parse_action(_Published) first = (pp.Keyword("first") + left + value + right).set_parse_action(_First) online = (pp.Keyword("online") + left + value + right).set_parse_action(_Online) modified = (pp.Keyword("modified") + left + value + right).set_parse_action(_Modified) no_pmid = (pp.Keyword("no_pmid") + left + right).set_parse_action(_NoPmid) no_doi = (pp.Keyword("no_doi") + left + right).set_parse_action(_NoDoi) no_label = (pp.Keyword("no_label") + left + right).set_parse_action(_NoLabel) function = (label | year | author | orcid | issn | published | first | online | modified | no_pmid | no_doi | no_label) if settings["TEMPORAL_LABELS"]: current = (pp.Keyword("active") + left + right).set_parse_action(_Active) active = (pp.Keyword("active") + left + value + right).set_parse_action(_Active) function = function | current | active union = pp.Literal("+").set_parse_action(_Union) symdifference = pp.Literal("^").set_parse_action(_Symdifference) intersection = pp.Literal("#").set_parse_action(_Intersection) difference = pp.Literal("-").set_parse_action(_Difference) operator = union | symdifference | difference | intersection expression = pp.Forward() atom = function | identifier | pp.Group(left + expression + right) expression <<= atom + (operator + atom)[...] expression.set_parse_action(_Expression) expression.ignore("!" + pp.rest_of_line) return expression
def _parse(self, expr): self.exprStack = [] parser = pp.Forward() ffn = (fident + pp.Literal('(').suppress() + pp.Group(pp.Optional(pp.delimitedList(parser))).setParseAction(self._pushLen) + pp.Literal(')').suppress()) ffield = (pp.Optional(pp.Word(pp.nums) + pp.Literal(":").suppress(), default=self.article.pk) + fident ).setParseAction(lambda strg, loc, st: dict( article_pk=st[0], name=st[1], )) atom = ((ffn.setResultsName('fn') | ffield.setResultsName('field') | pp.QuotedString('"').setResultsName('str') | fnumber.setResultsName('float') ).setParseAction(self._push) | pp.Group(pp.Literal('(').suppress() + parser + pp.Literal(')').suppress())) term = atom + pp.ZeroOrMore((pp.Combine(pp.Literal("*") | pp.Literal("/")).setResultsName('op') + atom).setParseAction(self._push)) parser << term + pp.ZeroOrMore((pp.Combine(pp.Literal("+") | pp.Literal("-")).setResultsName('op') + term).setParseAction(self._push)) parser.parseString(expr, True)
def parse(s): """ Parses a correctly-formatted string into a sympy expression, replacing column names in quotes w/ sympy Symbols. Parameters ---------- s : string detailing query. Column names should be in quotes (use \\" to escape quotes). Otherwise, s should follow sympy conventions. Returns ------- (expr, symbols) expr: sympy.Expr symbols: dict mapping sympy.Symbol -> string (column name) Examples ------------ >>> parse('log(cosh("Column1")) + 3') """ symbols = {} def register_var(t): name = t[0] if name in symbols: return symbols[name] else: symbols[name] = "x{}".format(len(symbols)) return symbols[name] pp_var << pp.QuotedString('"', "\\").setParseAction(register_var) sympy_expr = "".join(pp_expr.parseString(s)) return parse_expr(sympy_expr), symbols
def parse_expr(s): LPAR = pyp.Literal('(').suppress() RPAR = pyp.Literal(')').suppress() quote = pyp.Literal('"').suppress() sp = pyp.OneOrMore(pyp.White()).suppress() sps = pyp.ZeroOrMore(pyp.White()).suppress() nums = pyp.Word(pyp.srange("[0-9]")) num = (nums + pyp.Optional(pyp.Literal('.') + nums))\ .setParseAction(lambda toks: Num(''.join(toks))) var = (pyp.Word(pyp.alphas.lower(), pyp.srange("[a-zA-Z0-9]")) + pyp.Optional(pyp.Literal("'"))).\ setParseAction(lambda toks: Var(toks[0], toks[1] if len(toks) > 1 else None)) chars = (pyp.QuotedString('"')).setParseAction(lambda toks: Charstr(toks[0])) boolean = (pyp.oneOf("True False true false")).setParseAction(lambda toks: Bool(toks[0])) term = pyp.Forward() pterm = (LPAR + sps + term + sps + RPAR).setParseAction(lambda toks: toks[0]) term << pyp.infixNotation(num | var | pterm | chars | boolean, [ (pyp.oneOf("+ -"), 2, pyp.opAssoc.LEFT, lambda ts: BinOp(ts[0][0], ts[0][1], ts[0][2])), ]) formula = pyp.Forward() cmpop = pyp.oneOf("== < > <= >= !=") atom = (sps + term + sps + cmpop + sps + term + sps).\ setParseAction(lambda toks: Cmp(toks[1], toks[0], toks[2])) patom = (LPAR + sps + atom + sps + RPAR).setParseAction(lambda toks: toks[0]) formula << pyp.infixNotation(patom, [ (pyp.oneOf("&& ||"), 2, pyp.opAssoc.LEFT, lambda ts: BinCon(ts[0][0], ts[0][1], ts[0][2])), ]) res = formula.parseString(s) r = res[0] if len(res) > 0 else None return r
def generate_visualmin(): visualmin = p.QuotedString(quoteChar="-V/", unquoteResults=True, endQuoteChar="/") visualmin.setParseAction(lambda x: ["-V", x[0]]) return visualmin
def __parse(self): lbrace, rbrace, semi, quote = map(pyparsing.Suppress, '{};"') ip_address = pyparsing.Combine(pyparsing.Word(pyparsing.nums) + ('.' + pyparsing.Word(pyparsing.nums)) * 3) hex_int = pyparsing.Word(pyparsing.hexnums, exact=2) mac_address = pyparsing.Combine(hex_int + (':' + hex_int) * 5) hdw_type = pyparsing.Word(pyparsing.alphanums) yyyy_mm_dd = pyparsing.Combine((pyparsing.Word(pyparsing.nums, exact=4) | pyparsing.Word(pyparsing.nums, exact=2)) + ('/' + pyparsing.Word(pyparsing.nums, exact=2)) * 2) hh_mm_ss = pyparsing.Combine(pyparsing.Word(pyparsing.nums, exact=2) + (':' + pyparsing.Word(pyparsing.nums, exact=2)) * 2) date_ref = pyparsing.oneOf(list("0123456"))("weekday") + yyyy_mm_dd("date") + hh_mm_ss("time") def to_datetime(tokens): tokens["datetime"] = datetime.strptime("%(date)s %(time)s" % tokens, "%Y/%m/%d %H:%M:%S") date_ref.setParseAction(to_datetime) starts_stmt = "starts" + date_ref + semi ends_stmt = "ends" + (date_ref | "never") + semi tstp_stmt = "tstp" + date_ref + semi tsfp_stmt = "tsfp" + date_ref + semi hdw_stmt = "hardware" + hdw_type("type") + mac_address("mac") + semi uid_stmt = "uid" + pyparsing.QuotedString('"')("uid") + semi binding_stmt = "binding" + pyparsing.Word(pyparsing.alphanums) + pyparsing.Word(pyparsing.alphanums) + semi lease_statement = starts_stmt | ends_stmt | tstp_stmt | tsfp_stmt | hdw_stmt | uid_stmt | binding_stmt lease_def = "lease" + ip_address("ipaddress") + lbrace + \ pyparsing.Dict(pyparsing.ZeroOrMore(pyparsing.Group(lease_statement))) + rbrace with open(self.lease_file, 'r') as file: parsed = lease_def.scanString(file.read()) return parsed
def cross_link_db_filter_parser(inputstring): ''' example '"{ID_Score}" > 28 AND "{Sample}" == "%10_1%" OR ":Sample}" == "%10_2%" OR ":Sample}" == "%10_3%" OR ":Sample}" == "%8_1%" OR ":Sample}" == "%8_2%"' ''' import pyparsing as pp operator = pp.Regex(">=|<=|!=|>|<|==|in").setName("operator") value = pp.QuotedString( '"') | pp.Regex( r"[+-]?\d+(:?\.\d*)?(:?[eE][+-]?\d+)?") identifier = pp.Word(pp.alphas, pp.alphanums + "_") comparison_term = identifier | value condition = pp.Group(comparison_term + operator + comparison_term) expr = pp.operatorPrecedence(condition, [ ("OR", 2, pp.opAssoc.LEFT, ), ("AND", 2, pp.opAssoc.LEFT, ), ]) parsedstring = str(expr.parseString(inputstring)) \ .replace("[", "(") \ .replace("]", ")") \ .replace(",", " ") \ .replace("'", " ") \ .replace("%", "'") \ .replace("{", "float(entry['") \ .replace("}", "'])") \ .replace(":", "str(entry['") \ .replace("}", "'])") \ .replace("AND", "and") \ .replace("OR", "or") return parsedstring
def generate_range(): searchf = p.QuotedString(quoteChar="/", unquoteResults=False) searchb = p.QuotedString(quoteChar="?", unquoteResults=False) term = p.Word(p.nums) ^ p.oneOf("$ .") ^ searchf ^ searchb offset = p.oneOf("+ -") + p.Word(p.nums) offset.setParseAction(lambda x: ''.join(x)) fullterm = term("value") + p.Optional(offset("offset"), default='0') sep = p.Literal(",").suppress() wholebuffer = p.Literal("%") fullrange = fullterm("a") + p.Optional(sep + fullterm)("b") return fullrange ^ wholebuffer("all")
def DOCSTR_BLOCK(expr, resultsName=None): # pylint: disable=invalid-name """Block with an optional docstring followed by one of more `expr`.""" # Copied from pyparsing.indentedBlock def checkSubIndent(s, l, t): # pylint: disable=invalid-name curCol = pp.col(l, s) # pylint: disable=invalid-name if curCol > _indentation_stack[-1]: _indentation_stack.append(curCol) else: raise pp.ParseException(s, l, 'not a subentry') def checkUnindent(s, l, t): # pylint: disable=invalid-name del t if l >= len(s): return curCol = pp.col(l, s) # pylint: disable=invalid-name if not (_indentation_stack and curCol < _indentation_stack[-1] and curCol <= _indentation_stack[-2]): raise pp.ParseException(s, l, 'not an unindent') _indentation_stack.pop() INDENT = ( # pylint: disable=invalid-name pp.Empty() + pp.Empty().setParseAction(checkSubIndent)).setName('INDENT') UNDENT = pp.Empty().setParseAction(checkUnindent).setName('UNINDENT') # pylint: disable=invalid-name docstring = Group( Tag('docstring') + pp.QuotedString('"""', multiline=True)) code = Optional(pp.indentedBlock(expr, _indentation_stack, False), []) if resultsName: code = code.setResultsName(resultsName) body = INDENT + Optional(docstring)('docstring') + code + UNDENT return S(':') + NEWLINE - body
def inner_flatten(text): while(True): #Got to allow for for recursive includes changed=False def subsitute_include(string,loc, tokens): path=tokens[0][0][1:-1] if path in past_includes: warn("Denied attempt to re-include "+path ) return "" #Avoid including something already included, (no infinite loops) else: past_includes.add(path) changed=True; print("* expanding", path) directory, file = os.path.split(path) with Context(directory): include_text = load_include(file) return inner_flatten(include_text) pp_include = (pp.LineStart() + pp.Literal("include").suppress() + pp.nestedExpr( content=pp.QuotedString('"') ).addParseAction(subsitute_include) ) text = pp_include.transformString(text) if not(changed): return text
class Tokens(object): # shared tokens delim_chars = '[]{},' pause = pp.FollowedBy(pp.Word(delim_chars) | pp.StringEnd()) number = (pp.pyparsing_common.number + pause) quoted_string = pp.QuotedString('"', escChar='\\') true = (pp.Regex(r'(True|true|yes|on)') + pause). \ setParseAction(lambda _: True) false = (pp.Regex(r'(False|false|no|off)') + pause). \ setParseAction(lambda _: False) unquoted_string = pp.CharsNotIn(delim_chars). \ setParseAction(lambda toks: toks[0].strip()) empty_value = pp.Empty(). \ setParseAction(lambda _: '') # tokens for configs identifier = pp.pyparsing_common.identifier.copy() comma = pp.Literal(',').suppress() assign = pp.Literal('=').suppress() config_value = ( number | true | false | quoted_string | unquoted_string | empty_value) key_value_pair = (identifier + assign + config_value). \ setParseAction(lambda toks: (toks[0], toks[1])) key_value_pair_list = pp.Optional( key_value_pair + pp.ZeroOrMore(comma + key_value_pair)) # tokens for tags tag = quoted_string | unquoted_string | pp.Empty().suppress() tag_list = pp.Optional(tag + pp.ZeroOrMore(comma + tag))
def from_str(s, clazz, sympy_coefficients=False): element_parser = clazz.parser() if sympy_coefficients: from sympy.parsing.sympy_parser import parse_expr coeff_s = pp.QuotedString("[", endQuoteChar="]") coeff_s.setParseAction(lambda t: [parse_expr(t[0])]) coeff = pp.Optional(coeff_s, 1) else: coeff_i = pp.Suppress("[") + pp.Word(pp.nums) + pp.Suppress("]") coeff_i.setParseAction(lambda t: [int(t[0])]) coeff_f = pp.Suppress("[") + pp.Combine( pp.Optional(pp.Word(pp.nums)) + "." + pp.Optional(pp.Word(pp.nums))) + pp.Suppress("]") coeff_f.setParseAction(lambda t: [float(t[0])]) coeff = pp.Optional(coeff_i | coeff_f, 1) if six.PY2: minus = pp.Literal("-") else: #In python 3, where str is unicode, it is easy to allow the minus sign character. #This means you can copy from a formula in a pdf minus = pp.Literal("-") | pp.Literal(chr(0x2212)) minus.setParseAction(lambda t: ["-"]) firstTerm = pp.Optional(minus, "+") + coeff + pp.Optional( element_parser, "") otherTerm = (pp.Literal("+") | minus) + coeff + pp.Optional( element_parser, "") expn = pp.Group(firstTerm) + pp.ZeroOrMore(pp.Group(otherTerm)) exp = expn.parseString(s, True) x = [(b if a == "+" else -b) * clazz.from_str(c) for a, b, c in exp] out = functools.reduce(operator.add, x) return out
def parse(cls, search=False): """Parse the main query text. This method will also set the class attribute `parsed_search` to the parsed query, and it will return it too. :param cls: The class object, since it is a static method :type cls: object :param search: Search text string if a custom search string is to be used. False if the `cls.search` class attribute is to be used. :type search: str :returns: Parsed query :rtype: list >>> print(DocMatcher.parse('hello author = einstein')) [['hello'], ['author', '=', 'einstein']] >>> print(DocMatcher.parse('')) [] >>> print(\ DocMatcher.parse(\ '"hello world whatever =" tags = \\\'hello ====\\\'')) [['hello world whatever ='], ['tags', '=', 'hello ====']] >>> print(DocMatcher.parse('hello')) [['hello']] """ import pyparsing cls.logger.debug('Parsing search') search = search or cls.search papis_alphas = pyparsing.printables.replace('=', '') papis_key = pyparsing.Word(pyparsing.alphanums + '-') papis_value = pyparsing.QuotedString( quoteChar='"', escChar='\\', escQuote='\\' ) ^ pyparsing.QuotedString( quoteChar="'", escChar='\\', escQuote='\\' ) ^ papis_key equal = pyparsing.ZeroOrMore(" ") + \ pyparsing.Literal('=') + \ pyparsing.ZeroOrMore(" ") papis_query = pyparsing.ZeroOrMore( pyparsing.Group( pyparsing.ZeroOrMore( papis_key + equal ) + papis_value ) ) parsed = papis_query.parseString(search) cls.logger.debug('Parsed search = %s' % parsed) cls.parsed_search = parsed return cls.parsed_search
def __header_pattern(self): date_pattern = pyp.Combine( pyp.Word(pyp.alphas) + pyp.White() + pyp.Word(pyp.nums) + pyp.White() + pyp.Word(pyp.nums + ':')) hostname = pyp.Word(pyp.alphanums + '-') pid = pyp.Suppress('postgres') + pyp.QuotedString( '[', endQuoteChar=']') + pyp.Suppress(':' + pyp.White()) return date_pattern + hostname + pid
def get_parser(implicit_bin_op='AND'): COLON, LBRACK, RBRACK, LBRACE, RBRACE, TILDE, CARAT = map(pp.Literal, ':[]{}~^') LPAR, RPAR = map(pp.Suppress, '()') AND_, OR_, NOT_, TO_ = map(pp.CaselessKeyword, 'AND OR NOT TO'.split()) LTE, LT, GTE, GT, EQ = map(pp.Literal, ['<=', '<', '>=', '>', '=']) keyword = AND_ | OR_ | NOT_ | TO_ expression = pp.Forward() valid_keyword = pp.Regex(r'[a-zA-Z_][a-zA-Z0-9_.\[\]]*') valid_text = pp.Regex(r'([^\s\)]+)').setParseAction(lambda t: ValidText(t[0])) quoted_string = pp.QuotedString('"').setParseAction(lambda t: QuotedString(t[0])) rquoted_string = (pp.QuotedString('/') + pp.Optional(pp.Regex(r'[i]'))) \ .setParseAction(lambda t: RQuotedString(t[0], t[1]) if len(t) == 2 else RQuotedString(t[0], '')) field_text_value = (quoted_string | rquoted_string | valid_text )('text_field_value').setParseAction(lambda t: TextMatcher(t[0])) field_operate_value = ((LTE | LT | GTE | GT | EQ) + (valid_text | quoted_string))('operate_field_value').setParseAction(lambda t: Operator(t[0], t[1])) range_text_value = (pp.Regex(r'([^\s\]\}]+)')) incl_range_search = pp.Group(LBRACK + range_text_value + TO_ + range_text_value + RBRACK) incl_range_search.setParseAction(lambda t: RangeMatcher(True, t[0][1], t[0][3])) excl_range_search = pp.Group(LBRACE + range_text_value + TO_ + range_text_value + RBRACE) excl_range_search.setParseAction(lambda t: RangeMatcher(False, t[0][1], t[0][3])) field_range_value = (incl_range_search | excl_range_search)('range_field_value') multiple_field_text_value = (quoted_string | rquoted_string | valid_text) field_multiple_value = (LPAR + pp.OneOrMore(multiple_field_text_value) + RPAR)('multiple_field_value').setParseAction(lambda t: MultipleTextMatcher(t)) field_name = valid_keyword field_value = (field_multiple_value | field_operate_value | field_range_value | field_text_value) field_term = pp.Group(field_name('field_name') + COLON + field_value).setParseAction(lambda t: build_term_matcher(t[0][0], t[0][2])) term = pp.Forward() term << (field_term | pp.Group(LPAR + expression + RPAR).setParseAction(lambda t: t[0])) not_expression = ((NOT_ | '!').setParseAction(lambda: "NOT"), 1, pp.opAssoc.RIGHT, lambda t: NotMatcher(t[0][1])) if implicit_bin_op == 'AND': and_expression = (pp.Optional(AND_ | '&&').setParseAction(lambda: "AND"), 2, pp.opAssoc.LEFT, lambda t: build_binary_matcher(t[0], AndMatcher)) or_expression = ((OR_ | '||').setParseAction(lambda: "OR"), 2, pp.opAssoc.LEFT, lambda t: build_binary_matcher(t[0], OrMatcher)) else: and_expression = ((AND_ | '&&').setParseAction(lambda: "AND"), 2, pp.opAssoc.LEFT, lambda t: build_binary_matcher(t[0], AndMatcher)) or_expression = (pp.Optional(OR_ | '||').setParseAction(lambda: "OR"), 2, pp.opAssoc.LEFT, lambda t: build_binary_matcher(t[0], OrMatcher)) expression << pp.infixNotation(term, [ not_expression, and_expression, or_expression, ]) return expression
def regex_literal(): """"Defines an expression for regular expression literals. :AST Node: `RegexLiteral` :Example: ``/a+b+/`` """ regex = pyparsing.QuotedString("/") return regex.setParseAction(single_value_holder(ast.RegexLiteral))
def _make(): # Order is important - multi-char expressions need to come before narrow # ones. parts = [] for klass in filt_unary: f = pp.Literal("~%s" % klass.code) + pp.WordEnd() f.setParseAction(klass.make) parts.append(f) simplerex = "".join(c for c in pp.printables if c not in "()~'\"") rex = pp.Word(simplerex) |\ pp.QuotedString("\"", escChar='\\') |\ pp.QuotedString("'", escChar='\\') for klass in filt_rex: f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + rex.copy() f.setParseAction(klass.make) parts.append(f) for klass in filt_int: f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + pp.Word(pp.nums) f.setParseAction(klass.make) parts.append(f) # A naked rex is a URL rex: f = rex.copy() f.setParseAction(FUrl.make) parts.append(f) atom = pp.MatchFirst(parts) expr = pp.operatorPrecedence(atom, [(pp.Literal("!").suppress(), 1, pp.opAssoc.RIGHT, lambda x: FNot(*x)), (pp.Literal("&").suppress(), 2, pp.opAssoc.LEFT, lambda x: FAnd(*x)), (pp.Literal("|").suppress(), 2, pp.opAssoc.LEFT, lambda x: FOr(*x)), ]) expr = pp.OneOrMore(expr) return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
def mix_fstring_expressions(t): if not t.f_string_prefix: return fstring_arg = pp.QuotedString("{", end_quote_char="}") fstring_arg.add_parse_action( lambda tt: "{" + transformer.transform_string(tt[0]) + "}") ret = t.f_string_prefix + fstring_arg.transform_string( t.quoted_string_body) return ret
def load_steam_folders(self): if self.loaded_steam: return hkey = reg.OpenKey(reg.HKEY_CURRENT_USER, "Software\\Valve\\Steam") steam_path = reg.QueryValueEx(hkey, "SteamPath")[0] dl_folder = steam_path + "/steamapps/downloading" if os.path.isdir(dl_folder): self.directories.add(dl_folder) # Read the steam vdf file that contains path strings to all # game install directories. try: file = open(steam_path + "/steamapps/LibraryFolders.vdf").read() except OSError: print("Unable to open {}.".format( steam_path + "/steamapps/LibraryFolders.vdf")) # parse Valve's weird cfg format (its like a shitty version of JSON) # forward declare the value of a key value = pp.Forward() # expression for our dict structure that looks like: ["key1", value] key_value = pp.Group(pp.QuotedString('"') + value) # create a parse structure for value so value looks like: c expression = pp.Suppress( '{') + pp.Dict(pp.ZeroOrMore(key_value)) + pp.Suppress('}') # set our value to be a quoted string follow by the structure we defined, # looks like this in Python: # ["outer_key", { ["inner_key1", value], ["inner_key2", value] } ] # we can acess the above as either a dict or array. value <<= pp.QuotedString('"') | expression parser = pp.Dict(key_value) content = parser.parseString(file) # get the last pair's key, this should be the last folder numbered folder, # so we can use it as our max nr of folders for looping. max_folders = int(content["LibraryFolders"][-1][0]) # loop from 1 to (incl) max folders and use it as a dictionary key to get # the value of that key which should be a steam library folder path. for i in range(1, max_folders + 1): libpath = content["LibraryFolders"][str(i)] dlpath = libpath + "\\steamapps\\downloading" if os.path.isdir(dlpath): self.directories.add(dlpath) self.loaded_steam = True
def __parse_parameters(self, data): param_pattern = pyp.delimitedList(pyp.OneOrMore( pyp.Word('$' + pyp.nums) + pyp.Suppress(pyp.White() + '=' + pyp.White()) + pyp.QuotedString('\'')), delim=",") parsed_parameters = param_pattern.parseString(data) iterator = iter(parsed_parameters) parameters_dict = dict(izip(iterator, iterator)) return parameters_dict
def parse_query(query_string): import pyparsing logger = logging.getLogger('query_parser') logger.debug('Parsing search') papis_key = pyparsing.Word(pyparsing.alphanums + '-._/') papis_value = pyparsing.QuotedString( quoteChar='"', escChar='\\', escQuote='\\') ^ pyparsing.QuotedString( quoteChar="'", escChar='\\', escQuote='\\') ^ papis_key equal = (pyparsing.ZeroOrMore(" ") + pyparsing.Literal(':') + pyparsing.ZeroOrMore(" ")) papis_query = pyparsing.ZeroOrMore( pyparsing.Group(pyparsing.ZeroOrMore(papis_key + equal) + papis_value)) parsed = papis_query.parseString(query_string) logger.debug('Parsed query = %s' % parsed) return parsed
def __init__(self): # create parsing grammer sQStringLiteral = pyparsing.QuotedString("'") sQStringLiteral.setParseAction( lambda s, loc, toks: StringLiteral(s, loc, toks, False)) dQStringLiteral = pyparsing.QuotedString('"', '\\') dQStringLiteral.setParseAction( lambda s, loc, toks: StringLiteral(s, loc, toks, True)) stringLiteral = sQStringLiteral | dQStringLiteral functionCall = pyparsing.Forward() functionArg = stringLiteral | functionCall functionCall << pyparsing.Word(pyparsing.alphas, pyparsing.alphanums+'-') + \ pyparsing.Suppress('(') + \ pyparsing.Optional(functionArg + pyparsing.ZeroOrMore(pyparsing.Suppress(',') + functionArg)) + \ pyparsing.Suppress(')') functionCall.setParseAction( lambda s, loc, toks: FunctionCall(s, loc, toks)) predExpr = pyparsing.infixNotation(stringLiteral ^ functionCall, [ ('!', 1, pyparsing.opAssoc.RIGHT, lambda s, loc, toks: NotOperator(s, loc, toks)), ('<', 2, pyparsing.opAssoc.LEFT, infixBinaryOp(BinaryStrOperator)), ('<=', 2, pyparsing.opAssoc.LEFT, infixBinaryOp(BinaryStrOperator)), ('>', 2, pyparsing.opAssoc.LEFT, infixBinaryOp(BinaryStrOperator)), ('>=', 2, pyparsing.opAssoc.LEFT, infixBinaryOp(BinaryStrOperator)), ('==', 2, pyparsing.opAssoc.LEFT, infixBinaryOp(BinaryStrOperator)), ('!=', 2, pyparsing.opAssoc.LEFT, infixBinaryOp(BinaryStrOperator)), ('&&', 2, pyparsing.opAssoc.LEFT, infixBinaryOp(BinaryBoolOperator)), ('||', 2, pyparsing.opAssoc.LEFT, infixBinaryOp(BinaryBoolOperator)) ]) self.__ifgrammer = predExpr
def __init__(self): maybe_fuzzy = pp.Optional(pp.Literal('~')).setResultsName('fuzzy') q_str = pp.MatchFirst([pp.QuotedString('"'), pp.QuotedString("'")]) nq_str = pp.Word(pp.alphanums + '_').setResultsName('no_quotes') term = maybe_fuzzy + pp.MatchFirst([q_str, nq_str ]).setResultsName('term') v_term = pp.MatchFirst([q_str, nq_str]).setResultsName('value_term') number = pp.Word(pp.nums + '.').setResultsName('number') string_eq = pp.MatchFirst( [pp.Literal('!~').setResultsName('fuzzy_neq'), pp.Literal('~')]) simple_eq = pp.MatchFirst([pp.Literal('='), pp.Literal('!=')]).setResultsName('eq_rel') order = pp.MatchFirst([ pp.Literal('<='), pp.Literal('>='), pp.Literal('<'), pp.Literal('>') ]) num_relation = pp.MatchFirst([simple_eq, order ]).setResultsName('num_rel') + number str_relation = pp.MatchFirst([simple_eq, string_eq]) + v_term relation = pp.MatchFirst([num_relation, str_relation]).setResultsName('value_test') prop_test = pp.Group(term + pp.Optional(relation)) op_or = pp.Keyword('OR', caseless=True) op_and = pp.Keyword('AND', caseless=True) op_both = pp.MatchFirst([op_or, op_and]) self.parser = pp.Or([ prop_test, pp.delimitedList(prop_test, delim=op_or).setResultsName('op_or'), pp.delimitedList(prop_test, delim=op_and).setResultsName('op_and'), pp.delimitedList(prop_test, delim=op_both).setResultsName('op_both') ])
def _generate_grammar(*, debug_parser: bool = False): pp.ParserElement.setDefaultWhitespaceChars(" \t") EOL = pp.Optional(pp.pythonStyleComment()) + pp.LineEnd() LC = pp.Suppress(pp.OneOrMore(EOL) + pp.White(ws=" \t", min=4)) Identifier = pp.Word(initChars=pp.alphas, bodyChars=pp.alphanums + "_-") MultilineArgument = pp.QuotedString( quoteChar="<<<<", endQuoteChar=">>>>", multiline=True ) SingleQuotedArgument = pp.QuotedString(quoteChar="'", escChar="\\") DoubleQuotedArgument = pp.QuotedString(quoteChar='"', escChar="\\") QuotedArgument = (SingleQuotedArgument | DoubleQuotedArgument | MultilineArgument)( "quoted" ) SimpleArgument = pp.Word(pp.alphanums + "_-+*!$%&/()[]{}.,;:")("simple") Argument = (QuotedArgument | SimpleArgument) + pp.Optional(LC) KwArgument = pp.Combine(Identifier("key") + "=" + Argument) ArgumentList = pp.Group(pp.ZeroOrMore(pp.Group(KwArgument | Argument))) Command = ( pp.locatedExpr(Identifier)("command") + pp.Optional(LC) + ArgumentList("args") ) Grammar = pp.ZeroOrMore(pp.Group(pp.Optional(Command) + pp.Suppress(EOL))) if debug_parser: for expr_name in ( "Grammar Command ArgumentList KwArgument Argument " "SimpleArgument QuotedArgument DoubleQuotedArgument " "SingleQuotedArgument MultilineArgument " "Identifier LC EOL".split() ): expr = locals()[expr_name] expr.setName(expr_name) expr.setDebug() Grammar.parseWithTabs() # Keep tabs unexpanded! return Grammar