def main(argv):
action = argv[1]
argv = argv[2:]
# Used at grammar BUILD time.
OPS = {
'.': Id.Expr_Dot,
'->': Id.Expr_RArrow,
'::': Id.Expr_DColon,
'@': Id.Expr_At,
'...': Id.Expr_Ellipsis,
'$': Id.Expr_Dollar, # Only for legacy eggex /d+$/
}
# Note: We have two lists of ops because Id.Op_Semi is used, not
# Id.Arith_Semi.
for _, token_str, id_ in lex.EXPR_OPS:
assert token_str not in OPS, token_str
OPS[token_str] = id_
# Tokens that look like / or ${ or @{
triples = (
meta.ID_SPEC.LexerPairs(Kind.Arith) +
lex.OIL_LEFT_SUBS +
lex.OIL_LEFT_UNQUOTED +
lex.EXPR_WORDS
)
more_ops = {}
for _, token_str, id_ in triples:
assert token_str not in more_ops, token_str
more_ops[token_str] = id_
# Tokens that look like 'for'
keyword_ops = {}
for _, token_str, id_ in lex.EXPR_WORDS: # for, in, etc.
assert token_str not in keyword_ops, token_str
keyword_ops[token_str] = id_
if 0:
from pprint import pprint
pprint(OPS)
print('---')
pprint(more_ops)
print('---')
pprint(keyword_ops)
print('---')
tok_def = OilTokenDef(OPS, more_ops, keyword_ops)
if action == 'marshal': # generate the grammar and parse it
grammar_path = argv[0]
out_dir = argv[1]
basename, _ = os.path.splitext(os.path.basename(grammar_path))
# HACK for find:
if basename == 'find':
from tools.find import tokenizer as find_tokenizer
tok_def = find_tokenizer.TokenDef()
with open(grammar_path) as f:
gr = pgen.MakeGrammar(f, tok_def=tok_def)
marshal_path = os.path.join(out_dir, basename + '.marshal')
with open(marshal_path, 'wb') as out_f:
gr.dump(out_f)
nonterm_path = os.path.join(out_dir, basename + '_nt.py')
with open(nonterm_path, 'w') as out_f:
gr.dump_nonterminals(out_f)
log('Compiled %s -> %s and %s', grammar_path, marshal_path, nonterm_path)
#gr.report()
elif action == 'parse': # generate the grammar and parse it
# Remove build dependency
from frontend import parse_lib
from oil_lang import expr_parse
grammar_path = argv[0]
start_symbol = argv[1]
code_str = argv[2]
# For choosing lexer and semantic actions
grammar_name, _ = os.path.splitext(os.path.basename(grammar_path))
with open(grammar_path) as f:
gr = pgen.MakeGrammar(f, tok_def=tok_def)
arena = alloc.Arena()
lex_ = MakeOilLexer(code_str, arena)
is_expr = grammar_name in ('calc', 'grammar')
parse_opts = parse_lib.OilParseOptions()
parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, gr)
p = expr_parse.ExprParser(parse_ctx, gr)
try:
pnode, _ = p.Parse(lex_, gr.symbol2number[start_symbol])
except parse.ParseError as e:
log('Parse Error: %s', e)
return 1
names = parse_lib.MakeGrammarNames(gr)
p_printer = expr_parse.ParseTreePrinter(names) # print raw nodes
p_printer.Print(pnode)
if is_expr:
from oil_lang import expr_to_ast
tr = expr_to_ast.Transformer(gr)
if start_symbol == 'eval_input':
ast_node = tr.Expr(pnode)
else:
ast_node = tr.VarDecl(pnode)
ast_node.PrettyPrint()
print()
elif action == 'stdlib-test':
# This shows how deep Python's parse tree is. It doesn't use semantic
# actions to prune on the fly!
import parser # builtin module
t = parser.expr('1+2')
print(t)
t2 = parser.st2tuple(t)
print(t2)
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
action = argv[1]
argv = argv[2:]
# Common initialization
arith_ops = {}
for _, token_str, id_ in meta.ID_SPEC.LexerPairs(Kind.Arith):
arith_ops[token_str] = id_
if 0:
from pprint import pprint
pprint(arith_ops)
tok_def = OilTokenDef(arith_ops)
if action == 'marshal': # generate the grammar and parse it
grammar_path = argv[0]
out_dir = argv[1]
basename, _ = os.path.splitext(os.path.basename(grammar_path))
# HACK for find:
if basename == 'find':
from tools.find import tokenizer as find_tokenizer
tok_def = find_tokenizer.TokenDef()
with open(grammar_path) as f:
gr = pgen.MakeGrammar(f, tok_def=tok_def)
marshal_path = os.path.join(out_dir, basename + '.marshal')
with open(marshal_path, 'wb') as out_f:
gr.dump(out_f)
nonterm_path = os.path.join(out_dir, basename + '_nt.py')
with open(nonterm_path, 'w') as out_f:
gr.dump_nonterminals(out_f)
log('Compiled %s -> %s and %s', grammar_path, marshal_path, nonterm_path)
#gr.report()
elif action == 'parse': # generate the grammar and parse it
# Remove build dependency
from frontend import parse_lib
from oil_lang import expr_parse
grammar_path = argv[0]
start_symbol = argv[1]
code_str = argv[2]
# For choosing lexer and semantic actions
grammar_name, _ = os.path.splitext(os.path.basename(grammar_path))
with open(grammar_path) as f:
gr = pgen.MakeGrammar(f, tok_def=tok_def)
arena = alloc.Arena()
lex = MakeOilLexer(code_str, arena)
is_expr = grammar_name in ('calc', 'grammar')
parse_opts = parse_lib.OilParseOptions()
parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, gr)
p = expr_parse.ExprParser(parse_ctx, gr)
try:
pnode, _ = p.Parse(lex, gr.symbol2number[start_symbol])
except parse.ParseError as e:
log('Parse Error: %s', e)
return 1
names = parse_lib.MakeGrammarNames(gr)
p_printer = expr_parse.ParseTreePrinter(names) # print raw nodes
p_printer.Print(pnode)
if is_expr:
from oil_lang import expr_to_ast
tr = expr_to_ast.Transformer(gr)
if start_symbol == 'eval_input':
ast_node = tr.Expr(pnode)
else:
ast_node = tr.OilAssign(pnode)
ast_node.PrettyPrint()
print()
elif action == 'stdlib-test':
# This shows how deep Python's parse tree is. It doesn't use semantic
# actions to prune on the fly!
import parser # builtin module
t = parser.expr('1+2')
print(t)
t2 = parser.st2tuple(t)
print(t2)
else:
raise RuntimeError('Invalid action %r' % action)