def __init__(self, path, text, state=None): self.path = path self.base_path = os.path.dirname(path) self.text = text self.state = state opcode_name = Word(alphanums + '_') value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))', re.MULTILINE) opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value opcode.setParseAction(self.handle_opcode) section_name = Literal('<').suppress() + Word(alphas) + Literal( '>').suppress() section = section_name section.setParseAction(self.handle_section) include = Literal('#include').suppress() + locatedExpr( QuotedString('"')) include.setParseAction(self.handle_include) statement = (section ^ opcode ^ include) self.sfz_file = ZeroOrMore(statement) + stringEnd comment = Literal('//') + restOfLine self.sfz_file.ignore(comment)
def __init__(self, path, text, state=None): self.path = path self.base_path = os.path.dirname(path) self.text = text self.state = state opcode_name = Word(alphanums + '_') value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))', re.MULTILINE) opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value opcode.setParseAction(self.handle_opcode) section_name = Literal('<').suppress() + Word(alphas) + Literal('>').suppress() section = section_name section.setParseAction(self.handle_section) include = Literal('#include').suppress() + locatedExpr(QuotedString('"')) include.setParseAction(self.handle_include) statement = (section ^ opcode ^ include) self.sfz_file = ZeroOrMore(statement) + stringEnd comment = Literal('//') + restOfLine self.sfz_file.ignore(comment)
def message_type_definition() -> Token: first_aspect = Keyword("First") - Keyword("=>") - mathematical_expression() first_aspect.setParseAction(parse_aspect) length_aspect = Keyword("Length") - Keyword( "=>") - mathematical_expression() length_aspect.setParseAction(parse_aspect) component_aspects = Keyword("with") - delimitedList(first_aspect | length_aspect) component_aspects.setParseAction(parse_aspects) then = locatedExpr( Keyword("then") - (Keyword("null") | unqualified_identifier()) - Group(Optional(component_aspects)) - Group(Optional(value_constraint()))) then.setParseAction(parse_then) then_list = then + ZeroOrMore(comma() - then) then_list.setParseAction(lambda t: [t.asList()]) component_item = (~Keyword("end") + ~CaselessKeyword("Message") - unqualified_identifier() + Literal(":") - qualified_identifier() - Optional(then_list) - semicolon()) component_item.setParseAction(lambda t: Component(t[0], t[2], t[3]) if len(t) >= 4 else Component(t[0], t[2])) component_item.setName("Component") null_component_item = Keyword("null") - then - semicolon() null_component_item.setParseAction(lambda t: Component(None, None, [t[1]])) null_component_item.setName("NullComponent") component_list = Group( Optional(null_component_item) - component_item - ZeroOrMore(component_item)) component_list.setParseAction(lambda t: t.asList()) return (Keyword("message") - component_list - Keyword("end message") | Keyword("null message")).setName("Message")
def _generate_grammar(*, debug_parser: bool = False): pp.ParserElement.setDefaultWhitespaceChars(" \t") EOL = pp.Optional(pp.pythonStyleComment()) + pp.LineEnd() LC = pp.Suppress(pp.OneOrMore(EOL) + pp.White(ws=" \t", min=4)) Identifier = pp.Word(initChars=pp.alphas, bodyChars=pp.alphanums + "_-") MultilineArgument = pp.QuotedString( quoteChar="<<<<", endQuoteChar=">>>>", multiline=True ) SingleQuotedArgument = pp.QuotedString(quoteChar="'", escChar="\\") DoubleQuotedArgument = pp.QuotedString(quoteChar='"', escChar="\\") QuotedArgument = (SingleQuotedArgument | DoubleQuotedArgument | MultilineArgument)( "quoted" ) SimpleArgument = pp.Word(pp.alphanums + "_-+*!$%&/()[]{}.,;:")("simple") Argument = (QuotedArgument | SimpleArgument) + pp.Optional(LC) KwArgument = pp.Combine(Identifier("key") + "=" + Argument) ArgumentList = pp.Group(pp.ZeroOrMore(pp.Group(KwArgument | Argument))) Command = ( pp.locatedExpr(Identifier)("command") + pp.Optional(LC) + ArgumentList("args") ) Grammar = pp.ZeroOrMore(pp.Group(pp.Optional(Command) + pp.Suppress(EOL))) if debug_parser: for expr_name in ( "Grammar Command ArgumentList KwArgument Argument " "SimpleArgument QuotedArgument DoubleQuotedArgument " "SingleQuotedArgument MultilineArgument " "Identifier LC EOL".split() ): expr = locals()[expr_name] expr.setName(expr_name) expr.setDebug() Grammar.parseWithTabs() # Keep tabs unexpanded! return Grammar
def numeric_literal() -> Token: numeral = Combine( Word(nums) + ZeroOrMore(Optional(Word("_")) + Word(nums))) numeral.setParseAction(lambda t: t[0].replace("_", "")) decimal_literal = Group(numeral) decimal_literal.setParseAction(lambda t: (int(t[0][0]), 0)) extended_digit = Word(nums + "ABCDEF") based_numeral = Combine(extended_digit + ZeroOrMore(Optional("_") + extended_digit)) based_numeral.setParseAction(lambda t: t[0].replace("_", "")) based_literal = numeral + Literal("#") - based_numeral - Literal("#") based_literal.setParseAction(lambda t: (int(t[2], int(t[0])), int(t[0]))) num_literal = based_literal | decimal_literal num_literal.setName("Number") return locatedExpr(num_literal).setParseAction(lambda s, l, t: Number( t[0][1][0], t[0][1][1], parser_location(t[0][0], t[0][2], s)))
def _generate_grammar(*, debug_parser: bool = False): pp.ParserElement.setDefaultWhitespaceChars(' \t') EOL = pp.Optional(pp.pythonStyleComment()) + pp.LineEnd() LC = pp.Suppress(pp.OneOrMore(EOL) + pp.White(ws=' \t', min=4)) Identifier = pp.Word(initChars=pp.alphas, bodyChars=pp.alphanums + '_-') MultilineArgument = pp.QuotedString(quoteChar = '<<<<', endQuoteChar = '>>>>', multiline=True) SingleQuotedArgument = pp.QuotedString(quoteChar = '\'', escChar = '\\') DoubleQuotedArgument = pp.QuotedString(quoteChar = '"', escChar = '\\') QuotedArgument = (SingleQuotedArgument | DoubleQuotedArgument | MultilineArgument)('quoted') SimpleArgument = pp.Word(pp.alphanums + '_-+*!$%&/()[]{}.,;:')('simple') Argument = (QuotedArgument | SimpleArgument) + pp.Optional(LC) KwArgument = pp.Combine(Identifier('key') + '=' + Argument) ArgumentList = pp.Group(pp.ZeroOrMore(pp.Group(KwArgument | Argument))) Command = pp.locatedExpr(Identifier)('command') + pp.Optional(LC) \ + ArgumentList('args') Grammar = pp.ZeroOrMore(pp.Group(pp.Optional(Command) + pp.Suppress(EOL))) if debug_parser: for ename in 'Grammar Command ArgumentList KwArgument Argument ' \ 'SimpleArgument QuotedArgument DoubleQuotedArgument ' \ 'SingleQuotedArgument MultilineArgument ' \ 'Identifier LC EOL'.split(): expr = locals()[ename] expr.setName(ename) expr.setDebug() Grammar.parseWithTabs() # Keep tabs unexpanded! return Grammar
def _generate_grammar(self): # Define grammar: pp.ParserElement.setDefaultWhitespaceChars(" \t") def add_element(name: str, value: pp.ParserElement): nonlocal self if self.debug: value.setName(name) value.setDebug() return value EOL = add_element("EOL", pp.Suppress(pp.LineEnd())) Else = add_element("Else", pp.Keyword("else")) Identifier = add_element( "Identifier", pp.Word(f"{pp.alphas}_", bodyChars=pp.alphanums + "_-./")) BracedValue = add_element( "BracedValue", pp.nestedExpr(ignoreExpr=pp.quotedString | pp.QuotedString(quoteChar="$(", endQuoteChar=")", escQuote="\\", unquoteResults=False)). setParseAction(lambda s, l, t: ["(", *t[0], ")"]), ) Substitution = add_element( "Substitution", pp.Combine( pp.Literal("$") + (((pp.Literal("$") + Identifier + pp.Optional(pp.nestedExpr())) | (pp.Literal("(") + Identifier + pp.Literal(")")) | (pp.Literal("{") + Identifier + pp.Literal("}")) | (pp.Literal("$") + pp.Literal("{") + Identifier + pp.Optional(pp.nestedExpr()) + pp.Literal("}")) | (pp.Literal("$") + pp.Literal("[") + Identifier + pp.Literal("]"))))), ) LiteralValuePart = add_element( "LiteralValuePart", pp.Word(pp.printables, excludeChars="$#{}()")) SubstitutionValue = add_element( "SubstitutionValue", pp.Combine( pp.OneOrMore(Substitution | LiteralValuePart | pp.Literal("$"))), ) FunctionValue = add_element( "FunctionValue", pp.Group( pp.Suppress(pp.Literal("$") + pp.Literal("$")) + Identifier + pp.nestedExpr( ) # .setParseAction(lambda s, l, t: ['(', *t[0], ')']) ).setParseAction(lambda s, l, t: handle_function_value(*t)), ) Value = add_element( "Value", pp.NotAny(Else | pp.Literal("}") | EOL) + (pp.QuotedString(quoteChar='"', escChar="\\") | FunctionValue | SubstitutionValue | BracedValue), ) Values = add_element("Values", pp.ZeroOrMore(Value)("value")) Op = add_element( "OP", pp.Literal("=") | pp.Literal("-=") | pp.Literal("+=") | pp.Literal("*=") | pp.Literal("~="), ) Key = add_element("Key", Identifier) Operation = add_element( "Operation", Key("key") + pp.locatedExpr(Op)("operation") + Values("value")) CallArgs = add_element("CallArgs", pp.nestedExpr()) def parse_call_args(results): out = "" for item in chain(*results): if isinstance(item, str): out += item else: out += "(" + parse_call_args(item) + ")" return out CallArgs.setParseAction(parse_call_args) Load = add_element("Load", pp.Keyword("load") + CallArgs("loaded")) Include = add_element( "Include", pp.Keyword("include") + pp.locatedExpr(CallArgs)("included")) Option = add_element("Option", pp.Keyword("option") + CallArgs("option")) RequiresCondition = add_element("RequiresCondition", pp.originalTextFor(pp.nestedExpr())) def parse_requires_condition(s, l_unused, t): # The following expression unwraps the condition via the additional info # set by originalTextFor. condition_without_parentheses = s[t._original_start + 1:t._original_end - 1] # And this replaces the colons with '&&' similar how it's done for 'Condition'. condition_without_parentheses = ( condition_without_parentheses.strip().replace( ":", " && ").strip(" && ")) return condition_without_parentheses RequiresCondition.setParseAction(parse_requires_condition) Requires = add_element( "Requires", pp.Keyword("requires") + RequiresCondition("project_required_condition")) FunctionArgumentsAsString = add_element( "FunctionArgumentsAsString", pp.originalTextFor(pp.nestedExpr())) QtNoMakeTools = add_element( "QtNoMakeTools", pp.Keyword("qtNomakeTools") + FunctionArgumentsAsString("qt_no_make_tools_arguments"), ) # ignore the whole thing... DefineTestDefinition = add_element( "DefineTestDefinition", pp.Suppress( pp.Keyword("defineTest") + CallArgs + pp.nestedExpr(opener="{", closer="}", ignoreExpr=pp.LineEnd()) ), ) # ignore the whole thing... ForLoop = add_element( "ForLoop", pp.Suppress( pp.Keyword("for") + CallArgs + pp.nestedExpr(opener="{", closer="}", ignoreExpr=pp.LineEnd()) ), ) # ignore the whole thing... ForLoopSingleLine = add_element( "ForLoopSingleLine", pp.Suppress( pp.Keyword("for") + CallArgs + pp.Literal(":") + pp.SkipTo(EOL)), ) # ignore the whole thing... FunctionCall = add_element("FunctionCall", pp.Suppress(Identifier + pp.nestedExpr())) Scope = add_element("Scope", pp.Forward()) Statement = add_element( "Statement", pp.Group(Load | Include | Option | Requires | QtNoMakeTools | ForLoop | ForLoopSingleLine | DefineTestDefinition | FunctionCall | Operation), ) StatementLine = add_element("StatementLine", Statement + (EOL | pp.FollowedBy("}"))) StatementGroup = add_element( "StatementGroup", pp.ZeroOrMore(StatementLine | Scope | pp.Suppress(EOL))) Block = add_element( "Block", pp.Suppress("{") + pp.Optional(EOL) + StatementGroup + pp.Optional(EOL) + pp.Suppress("}") + pp.Optional(EOL), ) ConditionEnd = add_element( "ConditionEnd", pp.FollowedBy( (pp.Optional(pp.White()) + (pp.Literal(":") | pp.Literal("{") | pp.Literal("|")))), ) ConditionPart1 = add_element( "ConditionPart1", (pp.Optional("!") + Identifier + pp.Optional(BracedValue))) ConditionPart2 = add_element("ConditionPart2", pp.CharsNotIn("#{}|:=\\\n")) ConditionPart = add_element( "ConditionPart", (ConditionPart1 ^ ConditionPart2) + ConditionEnd) ConditionOp = add_element("ConditionOp", pp.Literal("|") ^ pp.Literal(":")) ConditionWhiteSpace = add_element( "ConditionWhiteSpace", pp.Suppress(pp.Optional(pp.White(" ")))) ConditionRepeated = add_element( "ConditionRepeated", pp.ZeroOrMore(ConditionOp + ConditionWhiteSpace + ConditionPart)) Condition = add_element("Condition", pp.Combine(ConditionPart + ConditionRepeated)) Condition.setParseAction( lambda x: " ".join(x).strip().replace(":", " && ").strip(" && ")) # Weird thing like write_file(a)|error() where error() is the alternative condition # which happens to be a function call. In this case there is no scope, but our code expects # a scope with a list of statements, so create a fake empty statement. ConditionEndingInFunctionCall = add_element( "ConditionEndingInFunctionCall", pp.Suppress(ConditionOp) + FunctionCall + pp.Empty().setParseAction(lambda x: [[]]).setResultsName( "statements"), ) SingleLineScope = add_element( "SingleLineScope", pp.Suppress(pp.Literal(":")) + pp.Group(Block | (Statement + EOL))("statements"), ) MultiLineScope = add_element("MultiLineScope", Block("statements")) SingleLineElse = add_element( "SingleLineElse", pp.Suppress(pp.Literal(":")) + (Scope | Block | (Statement + pp.Optional(EOL))), ) MultiLineElse = add_element("MultiLineElse", Block) ElseBranch = add_element( "ElseBranch", pp.Suppress(Else) + (SingleLineElse | MultiLineElse)) # Scope is already add_element'ed in the forward declaration above. Scope <<= pp.Group( Condition("condition") + (SingleLineScope | MultiLineScope | ConditionEndingInFunctionCall) + pp.Optional(ElseBranch)("else_statements")) Grammar = StatementGroup("statements") Grammar.ignore(pp.pythonStyleComment()) return Grammar
from functools import wraps from pyparsing import Word, Literal, Optional, CharsNotIn, OneOrMore, Combine from pyparsing import alphas, nums, quotedString from pyparsing import nestedExpr, locatedExpr, ParserElement, ParseResults from sqlops import is_valid_slug, slug_to_link from itertools import chain import sys import traceback # make \n significant ParserElement.setDefaultWhitespaceChars(' \t') ALLOWED_CHARS = frozenset(alphas + nums) MAX_CMD_LENGTH = 100 list_opener = locatedExpr(Word('{') | Word('|{')) list_closer = locatedExpr(Word('}|') | Word('}')) CMD = locatedExpr(Word(alphas + nums + '-_', max=100)) list_header = Literal('@') + Optional(CMD) + list_opener text_body = CharsNotIn('@\n|{}') | '|' line_break = Combine(OneOrMore(Literal('\n'))) list_stuff = list_header | list_opener | list_closer scribble = OneOrMore(text_body | line_break | list_stuff) lstrip_regex = re.compile('^[ \t\r]+') class Cons(object): __slots__ = ('data', 'params')
def parse(text): r = locatedExpr(regex_to_find)('regex_to_find') r.parseWithTabs( ) # a work around for tabs conversion, see https://github.com/pyparsing/pyparsing/issues/293 return [match for match in r.scanString(text)]
#!/usr/bin/python from pyparsing import Regex, Word, alphanums, LineStart, LineEnd, ParserElement, locatedExpr # type: ignore from sys import argv ParserElement.setDefaultWhitespaceChars( '') # must be before any pyparsing usage Ident = Word(alphanums) # Quote = Or(['"', "'"]) regex_to_find \ = locatedExpr(Regex(r'if\s*\(.+?= foo\(\w+\)\).*\n\s*{\n'))('pattern1') \ + locatedExpr(Regex('\s*sts\s*=\s*ERR_SOME.*'))('line_to_del') def parse(text): r = locatedExpr(regex_to_find)('regex_to_find') r.parseWithTabs( ) # a work around for tabs conversion, see https://github.com/pyparsing/pyparsing/issues/293 return [match for match in r.scanString(text)] def read_text(filename) -> str: with open(filename, 'r', encoding="utf8", errors='ignore') as f: return f.read() def overwrite_file(filename, new_content): with open(filename, 'w', encoding="utf8", errors='ignore') as f: f.seek(0) f.write(new_content)
def __init__(self, preferences_dir=None): ''' Creates a new ConfigShell. @param preferences_dir: Directory to load/save preferences from/to @type preferences_dir: str ''' self._current_node = None self._root_node = None self._exit = False # Grammar of the command line command = locatedExpr(Word(alphanums + '_'))('command') var = Word(alphanums + '?;&*$!#,=_\+/.<>()~@:-%[]') value = var keyword = Word(alphanums + '_\-') kparam = locatedExpr(keyword + Suppress('=') + Optional(value, default=''))('kparams*') pparam = locatedExpr(var)('pparams*') parameter = kparam | pparam parameters = OneOrMore(parameter) bookmark = Regex('@([A-Za-z0-9:_.]|-)+') pathstd = Regex('([A-Za-z0-9:_.\[\]]|-)*' + '/' + '([A-Za-z0-9:_.\[\]/]|-)*') \ | '..' | '.' path = locatedExpr(bookmark | pathstd | '*')('path') parser = Optional(path) + Optional(command) + Optional(parameters) self._parser = parser if tty: readline.set_completer_delims('\t\n ~!#$^&(){}\|;\'",?') readline.set_completion_display_matches_hook( self._display_completions) self.log = log.Log() if preferences_dir is not None: preferences_dir = os.path.expanduser(preferences_dir) if not os.path.exists(preferences_dir): os.makedirs(preferences_dir) self._prefs_file = preferences_dir + '/prefs.bin' self.prefs = prefs.Prefs(self._prefs_file) self._cmd_history = preferences_dir + '/history.txt' self._save_history = True if not os.path.isfile(self._cmd_history): try: open(self._cmd_history, 'w').close() except: self.log.warning("Cannot create history file %s, " % self._cmd_history + "command history will not be saved.") self._save_history = False if os.path.isfile(self._cmd_history) and tty: try: readline.read_history_file(self._cmd_history) except IOError: self.log.warning("Cannot read command history file %s." % self._cmd_history) if self.prefs['logfile'] is None: self.prefs['logfile'] = preferences_dir + '/' + 'log.txt' self.prefs.autosave = True else: self.prefs = prefs.Prefs() self._save_history = False try: self.prefs.load() except IOError: self.log.warning("Could not load preferences file %s." % self._prefs_file) for pref, value in six.iteritems(self.default_prefs): if pref not in self.prefs: self.prefs[pref] = value self.con = console.Console()
def unqualified_identifier() -> Token: return (locatedExpr( WordStart(alphas) + Word(alphanums + "_") + WordEnd(alphanums + "_")).setParseAction(verify_identifier).setName("Identifier"))