def main():
args = argparser.parse_args()
filename = args.program
startname = args.start
modname, classname = args.grammar.rsplit(".", 1)
try:
mod = importlib.import_module(modname)
except ImportError:
sys.exit(f"Cannot import {modname}")
try:
cls = getattr(mod, classname)
except AttributeError:
sys.exit(f"Module {modname} has no attribute {classname}")
if not isinstance(cls, type):
sys.exit(f"Object {modname}.{classname} is not a class ({cls!r})")
if not issubclass(cls, Parser):
sys.exit(f"Object {modname}.{classname} is not a subclass of Parser")
with open(filename) as f:
tokengen = generate_tokens(f.readline)
vis = Visualizer()
tok = Tokenizer(tokengen, vis)
p = cls(tok)
start = getattr(p, startname)
try:
tree = start()
vis.done()
finally:
vis.close()
def test_failure():
program = ("stmt: asmt | expr\n" "asmt: NAME '=' expr 42\n" "expr: NAME\n")
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = GrammarParser(tok)
rules = p.grammar()
assert rules is None
def test_toy():
program = "x - (y + z)"
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = ToyParser(tok)
tree = p.statement()
print(tree)
assert tree and tree.type == "statement"
assert tree.children[0].type == "expr"
assert tree.children[0].children[0].type == "expr"
def test_grammar():
program = ("stmt: asmt | expr\n" "asmt: NAME '=' expr\n" "expr: NAME\n")
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = GrammarParser(tok)
rules = p.grammar()
assert rules == [
Rule('stmt', [Alt(['asmt']), Alt(['expr'])]),
Rule('asmt', [Alt(['NAME', "'='", 'expr'])]),
Rule('expr', [Alt(['NAME'])])
]
def test_action():
program = "start: NAME { foo + bar } | NUMBER { -baz }\n"
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = GrammarParser(tok)
rules = p.grammar()
assert rules == [
Rule("start", [Alt(["NAME"], "foo + bar"),
Alt(["NUMBER"], "- baz")])
]
assert rules != [
Rule("start", [Alt(["NAME"], "foo + bar"),
Alt(["NUMBER"], "baz")])
]
def test_basic():
program = "f(42)"
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
def get():
return tok.get_token()[:2]
assert get() == (NAME, "f")
assert get() == (OP, "(")
assert get() == (NUMBER, "42")
assert get() == (OP, ")")
assert get() == (NEWLINE, "")
assert get() == (ENDMARKER, "")
def test_indents():
program = ("stmt: foo | bar\n" " | baz\n" " | booh | bah\n")
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = GrammarParser(tok)
rules = p.grammar()
assert rules == [
Rule('stmt', [
Alt(['foo']),
Alt(['bar']),
Alt(['baz']),
Alt(['booh']),
Alt(['bah'])
])
]
def main():
args = argparser.parse_args()
file = args.grammar
outfile = args.output
if not outfile:
head, tail = os.path.split(file)
base, ext = os.path.splitext(tail)
outfile = os.path.join(head, base + ".py")
classname = args.classname
if not classname:
tail = os.path.basename(file)
base, ext = os.path.splitext(tail)
classname = base.title() + "Parser"
print("Reading", file)
with open(file) as f:
tokengen = generate_tokens(f.readline)
vis = None
if args.visualize:
vis = Visualizer()
tok = Tokenizer(tokengen, vis)
p = GrammarParser(tok)
try:
rules = p.grammar()
if vis:
vis.done()
finally:
if vis:
vis.close()
if not rules:
sys.exit("Fail")
print("[")
for rule in rules:
print(f" {rule},")
print("]")
for rule in rules:
print(rule.name, end=": ", file=sys.stderr)
print(*rule.alts, sep=" | ", file=sys.stderr)
print("writing class", classname, "to", outfile, file=sys.stderr)
with open(outfile, "w") as stream:
generate(rules, classname, stream)
def test_basic():
program = "f(42)"
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
p = Parser(tok)
t = p.expect(NAME)
assert t and t.string == "f"
pos = p.mark()
assert p.expect("(")
t = p.expect(NUMBER)
assert t and t.string == "42"
assert p.expect(")")
pos2 = p.mark()
p.reset(pos)
assert p.expect("(")
assert p.expect(NUMBER)
assert p.expect(")")
p.reset(pos)
assert p.expect("(")
p.reset(pos2)
assert p.expect(NEWLINE)
assert p.expect(ENDMARKER)
def test_mark_reset():
program = "f(42) + abc"
file = StringIO(program)
tokengen = generate_tokens(file.readline)
tok = Tokenizer(tokengen)
def get():
return tok.get_token()[:2]
assert get() == (NAME, "f")
pos = tok.mark()
assert get() == (OP, "(")
assert get() == (NUMBER, "42")
assert get() == (OP, ")")
pos2 = tok.mark()
tok.reset(pos)
assert get() == (OP, "(")
assert get() == (NUMBER, "42")
assert get() == (OP, ")")
tok.reset(pos)
assert get() == (OP, "(")
tok.reset(pos2) # Forward
assert get() == (OP, "+")
assert get() == (NAME, "abc")
tok.reset(pos)
assert get() == (OP, "(")
assert get() == (NUMBER, "42")
assert get() == (OP, ")")
assert get() == (OP, "+")
assert get() == (NAME, "abc")