def __init__(self, path, text, state=None):
self.path = path
self.base_path = os.path.dirname(path)
self.text = text
self.state = state
opcode_name = Word(alphanums + '_')
value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))',
re.MULTILINE)
opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value
opcode.setParseAction(self.handle_opcode)
section_name = Literal('<').suppress() + Word(alphas) + Literal(
'>').suppress()
section = section_name
section.setParseAction(self.handle_section)
include = Literal('#include').suppress() + locatedExpr(
QuotedString('"'))
include.setParseAction(self.handle_include)
statement = (section ^ opcode ^ include)
self.sfz_file = ZeroOrMore(statement) + stringEnd
comment = Literal('//') + restOfLine
self.sfz_file.ignore(comment)
def __init__(self, path, text, state=None):
self.path = path
self.base_path = os.path.dirname(path)
self.text = text
self.state = state
opcode_name = Word(alphanums + '_')
value = Regex(r'.*?(?=\s*(([a-zA-Z0-9_]+=)|//|<[a-z]|$))', re.MULTILINE)
opcode = locatedExpr(opcode_name) + Literal('=').suppress() + value
opcode.setParseAction(self.handle_opcode)
section_name = Literal('<').suppress() + Word(alphas) + Literal('>').suppress()
section = section_name
section.setParseAction(self.handle_section)
include = Literal('#include').suppress() + locatedExpr(QuotedString('"'))
include.setParseAction(self.handle_include)
statement = (section
^ opcode
^ include)
self.sfz_file = ZeroOrMore(statement) + stringEnd
comment = Literal('//') + restOfLine
self.sfz_file.ignore(comment)
def message_type_definition() -> Token:
first_aspect = Keyword("First") - Keyword("=>") - mathematical_expression()
first_aspect.setParseAction(parse_aspect)
length_aspect = Keyword("Length") - Keyword(
"=>") - mathematical_expression()
length_aspect.setParseAction(parse_aspect)
component_aspects = Keyword("with") - delimitedList(first_aspect
| length_aspect)
component_aspects.setParseAction(parse_aspects)
then = locatedExpr(
Keyword("then") - (Keyword("null") | unqualified_identifier()) -
Group(Optional(component_aspects)) -
Group(Optional(value_constraint())))
then.setParseAction(parse_then)
then_list = then + ZeroOrMore(comma() - then)
then_list.setParseAction(lambda t: [t.asList()])
component_item = (~Keyword("end") + ~CaselessKeyword("Message") -
unqualified_identifier() + Literal(":") -
qualified_identifier() - Optional(then_list) -
semicolon())
component_item.setParseAction(lambda t: Component(t[0], t[2], t[3])
if len(t) >= 4 else Component(t[0], t[2]))
component_item.setName("Component")
null_component_item = Keyword("null") - then - semicolon()
null_component_item.setParseAction(lambda t: Component(None, None, [t[1]]))
null_component_item.setName("NullComponent")
component_list = Group(
Optional(null_component_item) - component_item -
ZeroOrMore(component_item))
component_list.setParseAction(lambda t: t.asList())
return (Keyword("message") - component_list - Keyword("end message")
| Keyword("null message")).setName("Message")
def _generate_grammar(*, debug_parser: bool = False):
pp.ParserElement.setDefaultWhitespaceChars(" \t")
EOL = pp.Optional(pp.pythonStyleComment()) + pp.LineEnd()
LC = pp.Suppress(pp.OneOrMore(EOL) + pp.White(ws=" \t", min=4))
Identifier = pp.Word(initChars=pp.alphas, bodyChars=pp.alphanums + "_-")
MultilineArgument = pp.QuotedString(
quoteChar="<<<<", endQuoteChar=">>>>", multiline=True
)
SingleQuotedArgument = pp.QuotedString(quoteChar="'", escChar="\\")
DoubleQuotedArgument = pp.QuotedString(quoteChar='"', escChar="\\")
QuotedArgument = (SingleQuotedArgument | DoubleQuotedArgument | MultilineArgument)(
"quoted"
)
SimpleArgument = pp.Word(pp.alphanums + "_-+*!$%&/()[]{}.,;:")("simple")
Argument = (QuotedArgument | SimpleArgument) + pp.Optional(LC)
KwArgument = pp.Combine(Identifier("key") + "=" + Argument)
ArgumentList = pp.Group(pp.ZeroOrMore(pp.Group(KwArgument | Argument)))
Command = (
pp.locatedExpr(Identifier)("command") + pp.Optional(LC) + ArgumentList("args")
)
Grammar = pp.ZeroOrMore(pp.Group(pp.Optional(Command) + pp.Suppress(EOL)))
if debug_parser:
for expr_name in (
"Grammar Command ArgumentList KwArgument Argument "
"SimpleArgument QuotedArgument DoubleQuotedArgument "
"SingleQuotedArgument MultilineArgument "
"Identifier LC EOL".split()
):
expr = locals()[expr_name]
expr.setName(expr_name)
expr.setDebug()
Grammar.parseWithTabs() # Keep tabs unexpanded!
return Grammar
def numeric_literal() -> Token:
numeral = Combine(
Word(nums) + ZeroOrMore(Optional(Word("_")) + Word(nums)))
numeral.setParseAction(lambda t: t[0].replace("_", ""))
decimal_literal = Group(numeral)
decimal_literal.setParseAction(lambda t: (int(t[0][0]), 0))
extended_digit = Word(nums + "ABCDEF")
based_numeral = Combine(extended_digit +
ZeroOrMore(Optional("_") + extended_digit))
based_numeral.setParseAction(lambda t: t[0].replace("_", ""))
based_literal = numeral + Literal("#") - based_numeral - Literal("#")
based_literal.setParseAction(lambda t: (int(t[2], int(t[0])), int(t[0])))
num_literal = based_literal | decimal_literal
num_literal.setName("Number")
return locatedExpr(num_literal).setParseAction(lambda s, l, t: Number(
t[0][1][0], t[0][1][1], parser_location(t[0][0], t[0][2], s)))
def _generate_grammar(*, debug_parser: bool = False):
pp.ParserElement.setDefaultWhitespaceChars(' \t')
EOL = pp.Optional(pp.pythonStyleComment()) + pp.LineEnd()
LC = pp.Suppress(pp.OneOrMore(EOL) + pp.White(ws=' \t', min=4))
Identifier = pp.Word(initChars=pp.alphas, bodyChars=pp.alphanums + '_-')
MultilineArgument = pp.QuotedString(quoteChar = '<<<<',
endQuoteChar = '>>>>', multiline=True)
SingleQuotedArgument = pp.QuotedString(quoteChar = '\'', escChar = '\\')
DoubleQuotedArgument = pp.QuotedString(quoteChar = '"', escChar = '\\')
QuotedArgument = (SingleQuotedArgument | DoubleQuotedArgument |
MultilineArgument)('quoted')
SimpleArgument = pp.Word(pp.alphanums + '_-+*!$%&/()[]{}.,;:')('simple')
Argument = (QuotedArgument | SimpleArgument) + pp.Optional(LC)
KwArgument = pp.Combine(Identifier('key') + '=' + Argument)
ArgumentList = pp.Group(pp.ZeroOrMore(pp.Group(KwArgument | Argument)))
Command = pp.locatedExpr(Identifier)('command') + pp.Optional(LC) \
+ ArgumentList('args')
Grammar = pp.ZeroOrMore(pp.Group(pp.Optional(Command) + pp.Suppress(EOL)))
if debug_parser:
for ename in 'Grammar Command ArgumentList KwArgument Argument ' \
'SimpleArgument QuotedArgument DoubleQuotedArgument ' \
'SingleQuotedArgument MultilineArgument ' \
'Identifier LC EOL'.split():
expr = locals()[ename]
expr.setName(ename)
expr.setDebug()
Grammar.parseWithTabs() # Keep tabs unexpanded!
return Grammar
def _generate_grammar(self):
# Define grammar:
pp.ParserElement.setDefaultWhitespaceChars(" \t")
def add_element(name: str, value: pp.ParserElement):
nonlocal self
if self.debug:
value.setName(name)
value.setDebug()
return value
EOL = add_element("EOL", pp.Suppress(pp.LineEnd()))
Else = add_element("Else", pp.Keyword("else"))
Identifier = add_element(
"Identifier",
pp.Word(f"{pp.alphas}_", bodyChars=pp.alphanums + "_-./"))
BracedValue = add_element(
"BracedValue",
pp.nestedExpr(ignoreExpr=pp.quotedString
| pp.QuotedString(quoteChar="$(",
endQuoteChar=")",
escQuote="\\",
unquoteResults=False)).
setParseAction(lambda s, l, t: ["(", *t[0], ")"]),
)
Substitution = add_element(
"Substitution",
pp.Combine(
pp.Literal("$") +
(((pp.Literal("$") + Identifier + pp.Optional(pp.nestedExpr()))
| (pp.Literal("(") + Identifier + pp.Literal(")"))
| (pp.Literal("{") + Identifier + pp.Literal("}"))
| (pp.Literal("$") + pp.Literal("{") + Identifier +
pp.Optional(pp.nestedExpr()) + pp.Literal("}"))
| (pp.Literal("$") + pp.Literal("[") + Identifier +
pp.Literal("]"))))),
)
LiteralValuePart = add_element(
"LiteralValuePart", pp.Word(pp.printables, excludeChars="$#{}()"))
SubstitutionValue = add_element(
"SubstitutionValue",
pp.Combine(
pp.OneOrMore(Substitution | LiteralValuePart
| pp.Literal("$"))),
)
FunctionValue = add_element(
"FunctionValue",
pp.Group(
pp.Suppress(pp.Literal("$") + pp.Literal("$")) + Identifier +
pp.nestedExpr(
) # .setParseAction(lambda s, l, t: ['(', *t[0], ')'])
).setParseAction(lambda s, l, t: handle_function_value(*t)),
)
Value = add_element(
"Value",
pp.NotAny(Else | pp.Literal("}") | EOL) +
(pp.QuotedString(quoteChar='"', escChar="\\")
| FunctionValue
| SubstitutionValue
| BracedValue),
)
Values = add_element("Values", pp.ZeroOrMore(Value)("value"))
Op = add_element(
"OP",
pp.Literal("=")
| pp.Literal("-=")
| pp.Literal("+=")
| pp.Literal("*=")
| pp.Literal("~="),
)
Key = add_element("Key", Identifier)
Operation = add_element(
"Operation",
Key("key") + pp.locatedExpr(Op)("operation") + Values("value"))
CallArgs = add_element("CallArgs", pp.nestedExpr())
def parse_call_args(results):
out = ""
for item in chain(*results):
if isinstance(item, str):
out += item
else:
out += "(" + parse_call_args(item) + ")"
return out
CallArgs.setParseAction(parse_call_args)
Load = add_element("Load", pp.Keyword("load") + CallArgs("loaded"))
Include = add_element(
"Include",
pp.Keyword("include") + pp.locatedExpr(CallArgs)("included"))
Option = add_element("Option",
pp.Keyword("option") + CallArgs("option"))
RequiresCondition = add_element("RequiresCondition",
pp.originalTextFor(pp.nestedExpr()))
def parse_requires_condition(s, l_unused, t):
# The following expression unwraps the condition via the additional info
# set by originalTextFor.
condition_without_parentheses = s[t._original_start +
1:t._original_end - 1]
# And this replaces the colons with '&&' similar how it's done for 'Condition'.
condition_without_parentheses = (
condition_without_parentheses.strip().replace(
":", " && ").strip(" && "))
return condition_without_parentheses
RequiresCondition.setParseAction(parse_requires_condition)
Requires = add_element(
"Requires",
pp.Keyword("requires") +
RequiresCondition("project_required_condition"))
FunctionArgumentsAsString = add_element(
"FunctionArgumentsAsString", pp.originalTextFor(pp.nestedExpr()))
QtNoMakeTools = add_element(
"QtNoMakeTools",
pp.Keyword("qtNomakeTools") +
FunctionArgumentsAsString("qt_no_make_tools_arguments"),
)
# ignore the whole thing...
DefineTestDefinition = add_element(
"DefineTestDefinition",
pp.Suppress(
pp.Keyword("defineTest") + CallArgs +
pp.nestedExpr(opener="{", closer="}", ignoreExpr=pp.LineEnd())
),
)
# ignore the whole thing...
ForLoop = add_element(
"ForLoop",
pp.Suppress(
pp.Keyword("for") + CallArgs +
pp.nestedExpr(opener="{", closer="}", ignoreExpr=pp.LineEnd())
),
)
# ignore the whole thing...
ForLoopSingleLine = add_element(
"ForLoopSingleLine",
pp.Suppress(
pp.Keyword("for") + CallArgs + pp.Literal(":") +
pp.SkipTo(EOL)),
)
# ignore the whole thing...
FunctionCall = add_element("FunctionCall",
pp.Suppress(Identifier + pp.nestedExpr()))
Scope = add_element("Scope", pp.Forward())
Statement = add_element(
"Statement",
pp.Group(Load
| Include
| Option
| Requires
| QtNoMakeTools
| ForLoop
| ForLoopSingleLine
| DefineTestDefinition
| FunctionCall
| Operation),
)
StatementLine = add_element("StatementLine",
Statement + (EOL | pp.FollowedBy("}")))
StatementGroup = add_element(
"StatementGroup",
pp.ZeroOrMore(StatementLine | Scope | pp.Suppress(EOL)))
Block = add_element(
"Block",
pp.Suppress("{") + pp.Optional(EOL) + StatementGroup +
pp.Optional(EOL) + pp.Suppress("}") + pp.Optional(EOL),
)
ConditionEnd = add_element(
"ConditionEnd",
pp.FollowedBy(
(pp.Optional(pp.White()) +
(pp.Literal(":") | pp.Literal("{") | pp.Literal("|")))),
)
ConditionPart1 = add_element(
"ConditionPart1",
(pp.Optional("!") + Identifier + pp.Optional(BracedValue)))
ConditionPart2 = add_element("ConditionPart2",
pp.CharsNotIn("#{}|:=\\\n"))
ConditionPart = add_element(
"ConditionPart", (ConditionPart1 ^ ConditionPart2) + ConditionEnd)
ConditionOp = add_element("ConditionOp",
pp.Literal("|") ^ pp.Literal(":"))
ConditionWhiteSpace = add_element(
"ConditionWhiteSpace", pp.Suppress(pp.Optional(pp.White(" "))))
ConditionRepeated = add_element(
"ConditionRepeated",
pp.ZeroOrMore(ConditionOp + ConditionWhiteSpace + ConditionPart))
Condition = add_element("Condition",
pp.Combine(ConditionPart + ConditionRepeated))
Condition.setParseAction(
lambda x: " ".join(x).strip().replace(":", " && ").strip(" && "))
# Weird thing like write_file(a)|error() where error() is the alternative condition
# which happens to be a function call. In this case there is no scope, but our code expects
# a scope with a list of statements, so create a fake empty statement.
ConditionEndingInFunctionCall = add_element(
"ConditionEndingInFunctionCall",
pp.Suppress(ConditionOp) + FunctionCall +
pp.Empty().setParseAction(lambda x: [[]]).setResultsName(
"statements"),
)
SingleLineScope = add_element(
"SingleLineScope",
pp.Suppress(pp.Literal(":")) +
pp.Group(Block | (Statement + EOL))("statements"),
)
MultiLineScope = add_element("MultiLineScope", Block("statements"))
SingleLineElse = add_element(
"SingleLineElse",
pp.Suppress(pp.Literal(":")) + (Scope | Block |
(Statement + pp.Optional(EOL))),
)
MultiLineElse = add_element("MultiLineElse", Block)
ElseBranch = add_element(
"ElseBranch",
pp.Suppress(Else) + (SingleLineElse | MultiLineElse))
# Scope is already add_element'ed in the forward declaration above.
Scope <<= pp.Group(
Condition("condition") + (SingleLineScope | MultiLineScope
| ConditionEndingInFunctionCall) +
pp.Optional(ElseBranch)("else_statements"))
Grammar = StatementGroup("statements")
Grammar.ignore(pp.pythonStyleComment())
return Grammar
from functools import wraps
from pyparsing import Word, Literal, Optional, CharsNotIn, OneOrMore, Combine
from pyparsing import alphas, nums, quotedString
from pyparsing import nestedExpr, locatedExpr, ParserElement, ParseResults
from sqlops import is_valid_slug, slug_to_link
from itertools import chain
import sys
import traceback
# make \n significant
ParserElement.setDefaultWhitespaceChars(' \t')
ALLOWED_CHARS = frozenset(alphas + nums)
MAX_CMD_LENGTH = 100
list_opener = locatedExpr(Word('{') | Word('|{'))
list_closer = locatedExpr(Word('}|') | Word('}'))
CMD = locatedExpr(Word(alphas + nums + '-_', max=100))
list_header = Literal('@') + Optional(CMD) + list_opener
text_body = CharsNotIn('@\n|{}') | '|'
line_break = Combine(OneOrMore(Literal('\n')))
list_stuff = list_header | list_opener | list_closer
scribble = OneOrMore(text_body | line_break | list_stuff)
lstrip_regex = re.compile('^[ \t\r]+')
class Cons(object):
__slots__ = ('data', 'params')
def parse(text):
r = locatedExpr(regex_to_find)('regex_to_find')
r.parseWithTabs(
) # a work around for tabs conversion, see https://github.com/pyparsing/pyparsing/issues/293
return [match for match in r.scanString(text)]
#!/usr/bin/python
from pyparsing import Regex, Word, alphanums, LineStart, LineEnd, ParserElement, locatedExpr # type: ignore
from sys import argv
ParserElement.setDefaultWhitespaceChars(
'') # must be before any pyparsing usage
Ident = Word(alphanums)
# Quote = Or(['"', "'"])
regex_to_find \
= locatedExpr(Regex(r'if\s*\(.+?= foo\(\w+\)\).*\n\s*{\n'))('pattern1') \
+ locatedExpr(Regex('\s*sts\s*=\s*ERR_SOME.*'))('line_to_del')
def parse(text):
r = locatedExpr(regex_to_find)('regex_to_find')
r.parseWithTabs(
) # a work around for tabs conversion, see https://github.com/pyparsing/pyparsing/issues/293
return [match for match in r.scanString(text)]
def read_text(filename) -> str:
with open(filename, 'r', encoding="utf8", errors='ignore') as f:
return f.read()
def overwrite_file(filename, new_content):
with open(filename, 'w', encoding="utf8", errors='ignore') as f:
f.seek(0)
f.write(new_content)
def __init__(self, preferences_dir=None):
'''
Creates a new ConfigShell.
@param preferences_dir: Directory to load/save preferences from/to
@type preferences_dir: str
'''
self._current_node = None
self._root_node = None
self._exit = False
# Grammar of the command line
command = locatedExpr(Word(alphanums + '_'))('command')
var = Word(alphanums + '?;&*$!#,=_\+/.<>()~@:-%[]')
value = var
keyword = Word(alphanums + '_\-')
kparam = locatedExpr(keyword + Suppress('=') +
Optional(value, default=''))('kparams*')
pparam = locatedExpr(var)('pparams*')
parameter = kparam | pparam
parameters = OneOrMore(parameter)
bookmark = Regex('@([A-Za-z0-9:_.]|-)+')
pathstd = Regex('([A-Za-z0-9:_.\[\]]|-)*' + '/' + '([A-Za-z0-9:_.\[\]/]|-)*') \
| '..' | '.'
path = locatedExpr(bookmark | pathstd | '*')('path')
parser = Optional(path) + Optional(command) + Optional(parameters)
self._parser = parser
if tty:
readline.set_completer_delims('\t\n ~!#$^&(){}\|;\'",?')
readline.set_completion_display_matches_hook(
self._display_completions)
self.log = log.Log()
if preferences_dir is not None:
preferences_dir = os.path.expanduser(preferences_dir)
if not os.path.exists(preferences_dir):
os.makedirs(preferences_dir)
self._prefs_file = preferences_dir + '/prefs.bin'
self.prefs = prefs.Prefs(self._prefs_file)
self._cmd_history = preferences_dir + '/history.txt'
self._save_history = True
if not os.path.isfile(self._cmd_history):
try:
open(self._cmd_history, 'w').close()
except:
self.log.warning("Cannot create history file %s, " %
self._cmd_history +
"command history will not be saved.")
self._save_history = False
if os.path.isfile(self._cmd_history) and tty:
try:
readline.read_history_file(self._cmd_history)
except IOError:
self.log.warning("Cannot read command history file %s." %
self._cmd_history)
if self.prefs['logfile'] is None:
self.prefs['logfile'] = preferences_dir + '/' + 'log.txt'
self.prefs.autosave = True
else:
self.prefs = prefs.Prefs()
self._save_history = False
try:
self.prefs.load()
except IOError:
self.log.warning("Could not load preferences file %s." %
self._prefs_file)
for pref, value in six.iteritems(self.default_prefs):
if pref not in self.prefs:
self.prefs[pref] = value
self.con = console.Console()
def unqualified_identifier() -> Token:
return (locatedExpr(
WordStart(alphas) + Word(alphanums + "_") +
WordEnd(alphanums +
"_")).setParseAction(verify_identifier).setName("Identifier"))
