def __init__(self, ctx):
self.ctx = ctx
scripts = set("postrotate prerotate firstaction lastaction preremove".split())
Stanza = Forward()
Spaces = Many(WSChar)
Bare = String(set(string.printable) - (set(string.whitespace) | set("#{}'\"")))
Num = Number & (WSChar | LineEnd)
Comment = OneLineComment("#").map(lambda x: None)
ScriptStart = WS >> PosMarker(Choice([Literal(s) for s in scripts])) << WS
ScriptEnd = Literal("endscript")
Line = (WS >> AnyChar.until(EOL) << WS).map(lambda x: "".join(x))
Lines = Line.until(ScriptEnd | EOF).map(lambda x: "\n".join(x))
Script = ScriptStart + Lines << Opt(ScriptEnd)
Script = Script.map(lambda x: [x[0], [x[1]], None])
BeginBlock = WS >> LeftCurly << WS
EndBlock = WS >> RightCurly
First = PosMarker((Bare | QuotedString)) << Spaces
Attr = Spaces >> (Num | Bare | QuotedString) << Spaces
Rest = Many(Attr)
Block = BeginBlock >> Many(Stanza).map(skip_none).map(self.to_entries) << EndBlock
Stmt = WS >> (Script | (First + Rest + Opt(Block))) << WS
Stanza <= WS >> (Stmt | Comment) << WS
Doc = Many(Stanza).map(skip_none).map(self.to_entries)
self.Top = Doc << EOF
def test_many1():
a = Char("a")
b = Char("b")
x = Char("x")
xs = Many(x, lower=1)
with pytest.raises(Exception):
xs("")
with pytest.raises(Exception):
xs("a")
assert xs("x") == ["x"]
assert xs("xxxxx") == ["x", "x", "x", "x", "x"]
assert xs("xxxxb") == ["x", "x", "x", "x"]
ab = Many(a + b, lower=1)
with pytest.raises(Exception):
ab("")
with pytest.raises(Exception):
ab("ba")
assert ab("ab") == [["a", "b"]]
assert ab("ababab") == [["a", "b"], ["a", "b"], ["a", "b"]]
ab = Many(a | b, lower=1)
assert ab("aababb") == ["a", "a", "b", "a", "b", "b"]
def __init__(self, *args, **kwargs):
def to_entry(name, attrs, body):
if body == ";":
return Directive(name=name.value,
attrs=attrs,
lineno=name.lineno,
src=self)
return Section(name=name.value,
attrs=attrs,
children=body,
lineno=name.lineno,
src=self)
name_chars = string.ascii_letters + "_/"
Stmt = Forward()
Num = Number & (WSChar | LineEnd | SemiColon)
Comment = OneLineComment("#").map(lambda x: None)
BeginBlock = WS >> LeftCurly << WS
EndBlock = WS >> RightCurly << WS
Bare = String(
set(string.printable) - (set(string.whitespace) | set("#;{}'\"")))
Name = WS >> PosMarker(
String(name_chars) | EmptyQuotedString(name_chars)) << WS
Attr = WS >> (Num | Bare | QuotedString) << WS
Attrs = Many(Attr)
Block = BeginBlock >> Many(Stmt).map(skip_none) << EndBlock
Stanza = (Lift(to_entry) * Name * Attrs *
(Block | SemiColon)) | Comment
Stmt <= WS >> Stanza << WS
Doc = Many(Stmt).map(skip_none)
self.Top = Doc + EOF
super(_NginxConf, self).__init__(*args, **kwargs)
def parse_doc(f, ctx=None, line_end="\n"):
def to_entry(name, rest):
if isinstance(rest, list):
return Section(name=name.value,
children=rest,
lineno=name.lineno,
src=ctx)
return Directive(name=name.value,
attrs=[rest],
lineno=name.lineno,
src=ctx)
Sep = InSet(":=")
Stmt = Forward()
Num = Number & (WSChar | LineEnd)
NULL = Literal("none", value=None)
Comment = (WS >> OneLineComment("#").map(lambda x: None))
BeginBlock = (WS >> LeftCurly << WS)
EndBlock = (WS >> RightCurly << WS)
Bare = String(
set(string.printable) - (set(string.whitespace) | set("#{}'\"")))
Name = WS >> PosMarker(
String(string.ascii_letters + "_" + string.digits)) << WS
Value = WS >> (Num | NULL | QuotedString | Bare) << WS
Block = BeginBlock >> Many(Stmt).map(skip_none) << EndBlock
Stanza = (Lift(to_entry) * Name * (Block | (Sep >> Value))) | Comment
Stmt <= WS >> Stanza << WS
Doc = Many(Stmt).map(skip_none)
Top = Doc + EOF
return Entry(children=Top(f)[0], src=ctx)
def parse_doc(content, ctx):
def to_directive(x):
name, rest = x
rest = [rest] if rest is not None else []
return Directive(name=name.value.strip(),
attrs=rest,
lineno=name.lineno,
src=ctx)
def to_section(name, rest):
return Section(name=name.value.strip(),
children=rest,
lineno=name.lineno,
src=ctx)
def apply_defaults(cfg):
if "DEFAULT" not in cfg:
return cfg
defaults = cfg["DEFAULT"]
not_defaults = cfg[~eq("DEFAULT")]
for c in not_defaults:
for d in defaults.grandchildren:
if d.name not in c:
c.children.append(d)
cfg.children = list(not_defaults)
return cfg
header_chars = (set(string.printable) - set(string.whitespace) -
set("[]")) | set(" ")
sep_chars = set("=:")
key_chars = header_chars - sep_chars
value_chars = set(string.printable) - set("\n\r")
Yes = Literal("yes", True, ignore_case=True)
No = Literal("no", False, ignore_case=True)
Tru = Literal("true", True, ignore_case=True)
Fals = Literal("false", False, ignore_case=True)
Boolean = ((Yes | No | Tru | Fals) & (WSChar | LineEnd)) % "Boolean"
LeftEnd = (WS + LeftBracket + WS)
RightEnd = (WS + RightBracket + WS)
Header = (
LeftEnd >> PosMarker(String(header_chars)) << RightEnd) % "Header"
Key = WS >> PosMarker(String(key_chars)) << WS
Sep = InSet(sep_chars, "Sep")
Value = WS >> (Boolean | HangingString(value_chars))
KVPair = WithIndent(Key + Opt(Sep >> Value)) % "KVPair"
Comment = (WS >>
(OneLineComment("#") | OneLineComment(";")).map(lambda x: None))
Line = Comment | KVPair.map(to_directive)
Sect = Lift(to_section) * Header * Many(Line).map(skip_none)
Doc = Many(Comment | Sect).map(skip_none)
Top = Doc + EOF
res = Entry(children=Top(content)[0], src=ctx)
return apply_defaults(res)
def __init__(self, ctx):
self.ctx = ctx
Complex = Forward()
Comment = (WS >> OneLineComment("#")).map(lambda x: None)
Name = String(string.ascii_letters + "_/")
Num = Number & (WSChar | LineEnd)
StartName = WS >> PosMarker(StartTagName(Letters)) << WS
EndName = WS >> EndTagName(Letters, ignore_case=True) << WS
Cont = Char("\\") + EOL
AttrStart = Many(WSChar)
AttrEnd = (Many(WSChar) + Cont) | Many(WSChar)
OpAttr = (Literal("!=") | Literal("<=") | Literal(">=")
| InSet("<>")) & WSChar
BareAttr = String(
set(string.printable) - (set(string.whitespace) | set("<>'\"")))
Attr = AttrStart >> (Num | QuotedString | OpAttr | BareAttr) << AttrEnd
Attrs = Many(Attr)
StartTag = (WS + LT) >> (StartName + Attrs) << (GT + WS)
EndTag = (WS + LT + FS) >> EndName << (GT + WS)
Simple = WS >> (Lift(self.to_directive) * PosMarker(Name) *
Attrs) << WS
Stanza = Simple | Complex | Comment | Many(WSChar | EOL,
lower=1).map(lambda x: None)
Complex <= (Lift(self.to_section) * StartTag *
Many(Stanza).map(skip_none)) << EndTag
Doc = Many(Stanza).map(skip_none)
self.Top = Doc + EOF
def test_many():
a = Char("a")
b = Char("b")
x = Char("x")
xs = Many(x)
assert xs("") == []
assert xs("a") == []
assert xs("x") == ["x"]
assert xs("xxxxx") == ["x", "x", "x", "x", "x"]
assert xs("xxxxb") == ["x", "x", "x", "x"]
ab = Many(a + b)
assert ab("") == []
assert ab("ba") == []
assert ab("ab") == [["a", "b"]]
assert ab("ababab") == [["a", "b"], ["a", "b"], ["a", "b"]]
ab = Many(a | b)
assert ab("aababb") == ["a", "a", "b", "a", "b", "b"]
def __init__(self, sep_chars="=:", comment_chars="#;"):
eol_chars = set("\n\r")
sep_chars = set(sep_chars)
comment_chars = set(comment_chars)
key_chars = set(string.printable) - (sep_chars | eol_chars | comment_chars)
value_chars = set(string.printable) - (eol_chars | comment_chars)
OLC = reduce(operator.__or__, [OneLineComment(c) for c in comment_chars])
Comment = (WS >> OLC).map(lambda x: None)
Num = Number & (WSChar | LineEnd)
Key = WS >> PosMarker(String(key_chars).map(lambda x: x.strip())) << WS
Sep = InSet(sep_chars)
Value = WS >> (Num | String(value_chars).map(lambda x: x.strip()))
KVPair = (Key + Opt(Sep + Value, default=[None, None])).map(lambda a: (a[0], a[1][1]))
Line = Comment | KVPair | EOL.map(lambda x: None)
Doc = Many(Line).map(skip_none).map(to_entry)
self.Top = Doc + EOF
def parse_doc(f, ctx=None, line_end="\n"):
def to_entry(name, rest):
rest = [] if not rest else [rest]
return Directive(name=name.value, attrs=rest, lineno=name.lineno, src=ctx)
Sep = Char("=")
Stmt = Forward()
Num = Number & (WSChar | LineEnd)
Comment = (WS >> OneLineComment("#").map(lambda x: None))
Bare = String(set(string.printable) - (set("#\n\r")))
Name = WS >> PosMarker(String(string.ascii_letters + "_-" + string.digits)) << WS
Value = WS >> (Num | Bare) << WS
Stanza = (Lift(to_entry) * Name * (Opt(Sep >> Value))) | Comment
Stmt <= WS >> Stanza << WS
Doc = Many(Stmt).map(skip_none)
Top = Doc + EOF
return Entry(children=Top(f)[0], src=ctx)
def parse_doc(self, content):
try:
def to_directive(x):
name, rest = x
rest = [rest] if rest is not None else []
return Directive(name=name.value.strip(), attrs=rest, lineno=name.lineno, src=self)
def to_section(name, rest):
return Section(name=name.value.strip(), children=rest, lineno=name.lineno, src=self)
def apply_defaults(cfg):
if "DEFAULT" not in cfg:
return cfg
defaults = cfg["DEFAULT"]
not_defaults = cfg[~eq("DEFAULT")]
for c in not_defaults:
for d in defaults.grandchildren:
if d.name not in c:
c.children.append(d)
cfg.children = list(not_defaults)
return cfg
def make_bytes(number, char_multiple):
if char_multiple.lower() == 'k':
return number * 2**10
if char_multiple.lower() == 'm':
return number * 2**20
if char_multiple.lower() == 'g':
return number * 2**30
content = "\n".join(content)
header_chars = (set(string.printable) - set(string.whitespace) - set("[]")) | set(" ")
sep_chars = set("=")
key_chars = header_chars - sep_chars
value_chars = set(string.printable) - set("\n\r")
On = Literal("on", True, ignore_case=True)
Off = Literal("off", False, ignore_case=True)
Tru = Literal("true", True, ignore_case=True)
Fals = Literal("false", False, ignore_case=True)
Boolean = ((On | Off | Tru | Fals) & (WSChar | LineEnd)) % "Boolean"
Num = Number & (WSChar | LineEnd)
QuoStr = QuotedString & (WSChar | LineEnd)
# Handle php.ini shorthand notation for memory limits: 1G, 8M, 50K
# https://www.php.net/manual/en/faq.using.php#faq.using.shorthandbytes
MemNum = (Lift(make_bytes) * Number * (Char('K') | Char('M') | Char('G'))) & (WSChar | LineEnd)
LeftEnd = (WS + LeftBracket + WS)
RightEnd = (WS + RightBracket + WS)
Header = (LeftEnd >> PosMarker(String(header_chars)) << RightEnd) % "Header"
Key = WS >> PosMarker(String(key_chars)) << WS
Sep = InSet(sep_chars, "Sep")
Value = WS >> (Boolean | MemNum | Num | QuoStr | HangingString(value_chars))
KVPair = WithIndent(Key + Opt(Sep >> Value)) % "KVPair"
Comment = (WS >> (OneLineComment(";")).map(lambda x: None))
Line = Comment | KVPair.map(to_directive)
Sect = Lift(to_section) * Header * Many(Line).map(skip_none)
Doc = Many(Comment | Sect).map(skip_none)
Top = Doc << WS << EOF
res = Entry(children=Top(content), src=self)
return apply_defaults(res)
except SkipException:
raise
except:
raise ParseException(ParseException("Could not parse content: '{0}'".
format(content)))
set("[]")) | set(" ")
sep_chars = set(":=")
key_chars = header_chars - sep_chars - set(" ")
value_chars = set(string.printable) - set("\n\r")
LeftEnd = WS >> Char("[") << WS
RightEnd = WS >> Char("]") << WS
Header = LeftEnd >> String(header_chars) << RightEnd
Key = WS >> String(key_chars) << WS
Sep = InSet(sep_chars)
Value = WS >> HangingString(value_chars)
KVPair = WithIndent(Key + Opt(Sep >> Value))
Comment = WS >> (OneLineComment("#") | OneLineComment(";")).map(lambda x: None)
Line = Comment | KVPair.map(tuple)
Sect = (Header + Many(Line).map(skip_none).map(dict)).map(tuple)
Doc = Many(Comment | Sect).map(skip_none).map(dict)
Top = Doc << WS << EOF
def parse_yum_repos(content):
doc = Top(content)
for k, v in doc.items():
for special in ("baseurl", "gpgkey"):
if special in v:
v[special] = [i.strip() for i in re.split(",| ", v[special])]
return doc
@parser(Specs.yum_repos_d)
class YumReposD(LegacyItemAccess, Parser):
def loads(data):
return Entry(children=Top(data))
def load(f):
return loads(f.read())
def to_entry(name, rest):
if isinstance(rest, list):
return Section(name=name.value, children=rest, lineno=name.lineno)
return Directive(name=name.value, attrs=[rest], lineno=name.lineno)
Sep = InSet(":=")
Stmt = Forward()
Num = Number & (WSChar | LineEnd)
NULL = Literal("none", value=None)
Comment = (WS >> OneLineComment("#").map(lambda x: None))
BeginBlock = (WS >> LeftCurly << WS)
EndBlock = (WS >> RightCurly << WS)
Bare = String(set(string.printable) - (set(string.whitespace) | set("#{}'\"")))
Name = WS >> PosMarker(
String(string.ascii_letters + "_" + string.digits)) << WS
Value = WS >> (Num | NULL | QuotedString | Bare) << WS
Block = BeginBlock >> Many(Stmt).map(skip_none) << EndBlock
Stanza = (Lift(to_entry) * Name * (Block | (Sep >> Value))) | Comment
Stmt <= WS >> Stanza << WS
Doc = Many(Stmt).map(skip_none)
Top = Doc << EOF
# Operator precedence is handled by having different declarations for each # prededence level. expr handles low level operations, term handles high level # operations, and factor handles simple numbers or subexpressions between # parentheses. Since the first element in expr is term and the first element in # term is factor, factors are evaluated first, then terms, and then exprs. # # The Top declaration ensures that the entire input is valid with no extra data # at the end. # We have to declare expr before its definition since it's used recursively. expr = Forward() % "expr forward" # A factor is a simple number or a subexpression between parentheses factor = WS >> (Number % "Number" | (LeftParen >> expr << RightParen)) << WS # A term handles strings of multiplication and division. As written, it would # convert "1 + 2 - 3 + 4" into [1, [['+', 2], ['-', 3], ['+', 4]]]. The first # element in the outer list is the initial factor. The second element of the # outer list is another list, which is the result of the Many. The Many's list # contains several two-element lists generated from each match of # (HighOps + factor). We pass the entire structure into the op function with # map. term = (factor + Many(HighOps + factor)).map(op) % "term" # expr has the same form as term. expr <= (term + Many(LowOps + term)).map(op) % "expr" # Top returns [result, None] on success and raises an Exception on failure. Top = (expr + EOF) % "Top"
return Not(p) if op else p
def oper(args):
left, rest = args
for op, right in rest:
if op == "&":
left = And(left, right)
if op in ",|":
left = Or(left, right)
return left
expr = Forward()
bare = String(set(string.printable) - (set(string.whitespace) | set(")&,|")))
tag = (QuotedString | bare).map(Eq)
regex_body = QuotedString | String(
set(string.printable) - set(string.whitespace))
regex = Char("/") >> regex_body.map(Regex)
factor_body = ((Char("(") >> expr << Char(")")) | regex | tag)
factor = (WS >> Opt(Char("!")) + factor_body << WS).map(negate)
term = (factor + Many(Char("&") + factor)).map(oper)
expr <= (term + Many(InSet(",|") + term)).map(oper)
parse = expr << EOF
ret = []
for i in x:
name, attrs, body = i
if body:
for n in [name.value] + attrs:
ret.append(Section(name=n, children=body, lineno=name.lineno))
else:
attrs = [attrs] if not isinstance(attrs, list) else attrs
ret.append(
Directive(name=name.value, attrs=attrs, lineno=name.lineno))
return ret
scripts = set("postrotate prerotate firstaction lastaction preremove".split())
Stanza = Forward()
Spaces = Many(WSChar)
Bare = String(set(string.printable) - (set(string.whitespace) | set("#{}'\"")))
Num = Number & (WSChar | LineEnd)
Comment = OneLineComment("#").map(lambda x: None)
ScriptStart = WS >> PosMarker(Choice([Literal(s) for s in scripts])) << WS
ScriptEnd = Literal("endscript")
Line = (WS >> AnyChar.until(EOL) << WS).map(lambda x: "".join(x))
Lines = Line.until(ScriptEnd).map(lambda x: "\n".join(x))
Script = ScriptStart + Lines << ScriptEnd
Script = Script.map(lambda x: [x[0], x[1], None])
BeginBlock = WS >> LeftCurly << WS
EndBlock = WS >> RightCurly
First = PosMarker((Bare | QuotedString)) << Spaces
Attr = Spaces >> (Num | Bare | QuotedString) << Spaces
Rest = Many(Attr)
Block = BeginBlock >> Many(Stanza).map(skip_none).map(to_entries) << EndBlock
def _parse_doc(self):
def to_directive(name, attrs):
return Directive(name=name.value,
attrs=attrs,
lineno=name.lineno,
src=self)
def to_directive_noval(name, sep):
return Directive(name=name.value,
attrs=[],
lineno=name.lineno,
src=self)
def to_section(name, attrs, body):
return Section(name=name.value,
attrs=attrs,
children=body,
lineno=name.lineno,
src=self)
sep_chars = set("=")
Sep = InSet(sep_chars, "Sep")
name_chars = string.ascii_letters + "_/"
Name = Many(WSChar) >> PosMarker(
String(name_chars) | EmptyQuotedString(name_chars)) << Many(WSChar)
BareStringDir = String(set(string.printable) - set("\r\n"))
BareStringBlk = String(
set(string.printable) - (set(string.whitespace) | set("{}'\"")))
BlockBeg = Many(WSChar) >> LeftCurly << WS
BlockEnd = Many(WSChar) >> RightCurly << (EOL | EOF)
Comment = OneLineComment("#").map(lambda x: None) << (EOL | EOF)
Stmt = Forward()
_AttrDir = Many(WSChar) >> (BareStringDir | QuotedString)
AttrDir = Sep >> Many(_AttrDir)
AttrDirNoVal = Sep << Many(WSChar)
Dir = (Lift(to_directive) * Name * AttrDir) << (EOL | EOF)
DirNoVal = (Lift(to_directive_noval) * Name * AttrDirNoVal) << (EOL
| EOF)
_AttrBlk = Many(WSChar) >> (BareStringBlk | QuotedString)
AttrsBlk = Many(_AttrBlk)
BlockBody = BlockBeg >> Many(Stmt).map(skip_none) << BlockEnd
Block = (Lift(to_section) * Name * AttrsBlk * BlockBody)
Stmt <= WS >> (Block | Dir | DirNoVal | Comment) << WS
Doc = Many(Stmt).map(skip_none)
Top = Doc + EOF
return Top
