def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# NOTE: This initialization must be identical to the one in core/meta.py. We
# do it here to avoid circular dependencies.
ID_SPEC = id_kind.IdSpec({}, {})
id_kind.AddKinds(ID_SPEC)
id_kind.AddBoolKinds(ID_SPEC, bool_arg_type_e) # must come second
id_kind.SetupTestBuiltin(ID_SPEC, {}, {}, {}, bool_arg_type_e)
ids = ID_SPEC.id_str2int.items()
ids.sort(key=lambda pair: pair[1]) # Sort by ID
if action == 'c':
for name, id_int in ids:
print('#define id__%s %s' % (name, id_int))
elif action == 'mypy':
from asdl import asdl_
from asdl import gen_python
#
# Create a SYNTHETIC ASDL module, and generate code from it.
#
id_sum = asdl_.Sum([asdl_.Constructor(name) for name, _ in ids])
variants2 = [
asdl_.Constructor(name) for name in ID_SPEC.kind_name_list
]
kind_sum = asdl_.Sum(variants2)
id_ = asdl_.Type('Id', id_sum)
kind_ = asdl_.Type('Kind', kind_sum)
schema_ast = asdl_.Module('id_kind', [id_, kind_])
#print(schema_ast)
f = sys.stdout
f.write("""\
from asdl import runtime
""")
# Minor style issue: we want Id and Kind, not Id_e and Kind_e
v = gen_python.GenMyPyVisitor(f, None, e_suffix=False)
v.VisitModule(schema_ast)
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
try:
schema_path = argv[2]
except IndexError:
raise RuntimeError('Schema path required')
schema_filename = os.path.basename(schema_path)
if schema_filename in ('syntax.asdl', 'runtime.asdl'):
app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')}
else:
app_types = {}
if action == 'c': # Generate C code for the lexer
with open(schema_path) as f:
schema_ast, _ = front_end.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(schema_ast)
elif action == 'cpp': # Generate C++ code for ASDL schemas
out_prefix = argv[3]
pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes'))
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
# asdl/typed_arith.asdl -> typed_arith_asdl
ns = os.path.basename(schema_path).replace('.', '_')
with open(out_prefix + '.h', 'w') as f:
guard = ns.upper()
f.write("""\
#ifndef %s
#define %s
""" % (guard, guard))
f.write("""\
#include <cstdint>
#include "mylib.h" // for Str, List, etc.
""")
if pretty_print_methods:
f.write("""
#include "hnode_asdl.h"
using hnode_asdl::hnode_t;
""")
f.write("""\
namespace %s {
""" % ns)
v = gen_cpp.ForwardDeclareVisitor(f)
v.VisitModule(schema_ast)
v2 = gen_cpp.ClassDefVisitor(
f, type_lookup, pretty_print_methods=pretty_print_methods)
v2.VisitModule(schema_ast)
f.write("""
} // namespace %s
#endif // %s
""" % (ns, guard))
with open(out_prefix + '.cc', 'w') as f:
# HACK until we support 'use'
if schema_filename == 'syntax.asdl':
f.write('#include "id_kind_asdl.h" // hack\n')
f.write('using id_kind_asdl::Id_t; // hack\n')
f.write("""
#include <assert.h>
#include "hnode_asdl.h"
using hnode_asdl::hnode__Record;
using hnode_asdl::hnode__Array;
using hnode_asdl::hnode__External;
using hnode_asdl::hnode__Leaf;
using hnode_asdl::field;
using hnode_asdl::color_e;
// TODO: Generate this asdl/runtime.h header and include it?
namespace runtime { // declare
hnode_asdl::hnode__Record* NewRecord(Str* node_type);
hnode_asdl::hnode__Leaf* NewLeaf(Str* s, hnode_asdl::color_t e_color);
extern Str* TRUE_STR;
extern Str* FALSE_STR;
} // declare namespace runtime
""")
f.write("""
#include "%s.h"
namespace %s {
""" % (ns, ns))
v3 = gen_cpp.MethodDefVisitor(
f, type_lookup, pretty_print_methods=pretty_print_methods)
v3.VisitModule(schema_ast)
f.write("""
} // namespace %s
""" % ns)
elif action == 'mypy': # Generated typed MyPy code
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
try:
abbrev_module_name = argv[3]
except IndexError:
abbrev_mod = None
else:
# Weird Python rule for importing: fromlist needs to be non-empty.
abbrev_mod = __import__(abbrev_module_name, fromlist=['.'])
f = sys.stdout
for typ in app_types.itervalues():
if isinstance(typ, meta.UserType):
f.write('from _devbuild.gen.%s import %s\n' %
(typ.mod_name, typ.type_name))
# HACK
f.write('from _devbuild.gen.%s import Id_str\n' % typ.mod_name)
f.write('\n')
# NOTE: Dict, Any are for AssocArray with 'dict' type.
f.write("""\
from asdl import pybase
from typing import Optional, List, Tuple, Dict, Any, cast
""")
pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes'))
optional_fields = bool(os.getenv('OPTIONAL_FIELDS', 'yes'))
if pretty_print_methods:
f.write("""
from asdl import runtime # For runtime.NO_SPID
from asdl.runtime import NewRecord, NewLeaf
from _devbuild.gen.hnode_asdl import color_e, hnode, hnode_e, hnode_t, field
""")
abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else []
v = gen_python.GenMyPyVisitor(
f,
type_lookup,
abbrev_mod_entries,
pretty_print_methods=pretty_print_methods,
optional_fields=optional_fields)
v.VisitModule(schema_ast)
if abbrev_mod:
f.write("""\
#
# CONCATENATED FILE
#
""")
package, module = abbrev_module_name.split('.')
path = os.path.join(package, module + '.py')
with open(path) as in_f:
f.write(in_f.read())
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# TODO:
# generate builtin::echo, etc.
#
# And in Python do the same.
option = _CreateSum(_OPT_ENUM, [opt.name for opt in option_def.All()])
builtin = _CreateSum(_BUILTIN_ENUM,
[b.enum_name for b in builtin_def.All()])
# TODO: could shrink array later.
# [opt.name for opt in option_def.All() if opt.implemented])
schema_ast = asdl_.Module('option', [], [option, builtin])
if action == 'cpp':
from asdl import gen_cpp
out_prefix = argv[2]
with open(out_prefix + '.h', 'w') as f:
f.write("""\
#ifndef OPTION_ASDL_H
#define OPTION_ASDL_H
namespace option_asdl {
""")
# TODO: Could suppress option_str
v = gen_cpp.ClassDefVisitor(f, {}, simple_int_sums=_SIMPLE)
v.VisitModule(schema_ast)
f.write("""
} // namespace option_asdl
#endif // OPTION_ASDL_H
""")
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
#include <assert.h>
#include "option_asdl.h"
namespace option_asdl {
""")
v = gen_cpp.MethodDefVisitor(f, {}, simple_int_sums=_SIMPLE)
v.VisitModule(schema_ast)
f.write('} // namespace option_asdl\n')
elif action == 'mypy':
from asdl import gen_python
f = sys.stdout
f.write("""\
from asdl import pybase
""")
# option_i type
v = gen_python.GenMyPyVisitor(f, None, simple_int_sums=_SIMPLE)
v.VisitModule(schema_ast)
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
try:
schema_path = argv[2]
except IndexError:
raise RuntimeError('Schema path required')
schema_filename = os.path.basename(schema_path)
if schema_filename in ('syntax.asdl', 'runtime.asdl'):
app_types = {'id': UserType('id_kind_asdl', 'Id_t')}
else:
app_types = {}
if action == 'c': # Generate C code for the lexer
with open(schema_path) as f:
schema_ast = front_end.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(schema_ast)
elif action == 'cpp': # Generate C++ code for ASDL schemas
out_prefix = argv[3]
pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes'))
with open(schema_path) as f:
schema_ast = front_end.LoadSchema(f, app_types)
# asdl/typed_arith.asdl -> typed_arith_asdl
ns = os.path.basename(schema_path).replace('.', '_')
with open(out_prefix + '.h', 'w') as f:
guard = ns.upper()
f.write("""\
// %s.h is generated by asdl/tool.py
#ifndef %s
#define %s
""" % (out_prefix, guard, guard))
f.write("""\
#include <cstdint>
#include "mylib.h" // for Str, List, etc.
""")
if pretty_print_methods:
f.write("""\
#include "hnode_asdl.h"
using hnode_asdl::hnode_t;
""")
if app_types:
f.write("""\
#include "id_kind_asdl.h"
using id_kind_asdl::Id_t;
""")
for use in schema_ast.uses:
# Forward declarations in the header, like
# namespace syntax_asdl { class command_t; }
# must come BEFORE namespace, so it can't be in the visitor.
# assume sum type for now!
cpp_names = [
'class %s;' % asdl_.TypeNameHeuristic(n)
for n in use.type_names
]
f.write('namespace %s_asdl { %s }\n' %
(use.mod_name, ' '.join(cpp_names)))
f.write('\n')
f.write("""\
namespace %s {
""" % ns)
v = gen_cpp.ForwardDeclareVisitor(f)
v.VisitModule(schema_ast)
debug_info = {}
v2 = gen_cpp.ClassDefVisitor(
f,
pretty_print_methods=pretty_print_methods,
simple_int_sums=_SIMPLE,
debug_info=debug_info)
v2.VisitModule(schema_ast)
f.write("""
} // namespace %s
#endif // %s
""" % (ns, guard))
try:
debug_info_path = argv[4]
except IndexError:
pass
else:
with open(debug_info_path, 'w') as f:
from pprint import pformat
f.write('''\
cpp_namespace = %r
tags_to_types = \\
%s
''' % (ns, pformat(debug_info)))
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
// %s.cc is generated by asdl/tool.py
#include "%s.h"
#include <assert.h>
#include "asdl_runtime.h" // generated code uses wrappers here
""" % (out_prefix, ns))
# To call pretty-printing methods
for use in schema_ast.uses:
f.write('#include "%s_asdl.h" // ASDL use\n' %
use.mod_name)
f.write("""\
// Generated code uses these types
using hnode_asdl::hnode__Record;
using hnode_asdl::hnode__Array;
using hnode_asdl::hnode__External;
using hnode_asdl::hnode__Leaf;
using hnode_asdl::field;
using hnode_asdl::color_e;
""")
if app_types:
f.write('using id_kind_asdl::Id_str;\n')
f.write("""
namespace %s {
""" % ns)
v3 = gen_cpp.MethodDefVisitor(
f,
pretty_print_methods=pretty_print_methods,
simple_int_sums=_SIMPLE)
v3.VisitModule(schema_ast)
f.write("""
} // namespace %s
""" % ns)
elif action == 'mypy': # Generated typed MyPy code
with open(schema_path) as f:
schema_ast = front_end.LoadSchema(f, app_types)
try:
abbrev_module_name = argv[3]
except IndexError:
abbrev_mod = None
else:
# Weird Python rule for importing: fromlist needs to be non-empty.
abbrev_mod = __import__(abbrev_module_name, fromlist=['.'])
f = sys.stdout
# TODO: Remove Any once we stop using it
f.write("""\
from asdl import pybase
from typing import Optional, List, Tuple, Dict, Any, cast, TYPE_CHECKING
""")
if schema_ast.uses:
f.write('\n')
f.write('if TYPE_CHECKING:\n')
for use in schema_ast.uses:
py_names = [asdl_.TypeNameHeuristic(n) for n in use.type_names]
# indented
f.write(' from _devbuild.gen.%s_asdl import %s\n' %
(use.mod_name, ', '.join(py_names)))
f.write('\n')
for typ in app_types.itervalues():
if isinstance(typ, UserType):
f.write('from _devbuild.gen.%s import %s\n' %
(typ.mod_name, typ.type_name))
# HACK
f.write('from _devbuild.gen.%s import Id_str\n' % typ.mod_name)
f.write('\n')
pretty_print_methods = bool(os.getenv('PRETTY_PRINT_METHODS', 'yes'))
optional_fields = bool(os.getenv('OPTIONAL_FIELDS', 'yes'))
if pretty_print_methods:
f.write("""
from asdl import runtime # For runtime.NO_SPID
from asdl.runtime import NewRecord, NewLeaf
from _devbuild.gen.hnode_asdl import color_e, hnode, hnode_e, hnode_t, field
""")
abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else []
v = gen_python.GenMyPyVisitor(
f,
abbrev_mod_entries,
pretty_print_methods=pretty_print_methods,
optional_fields=optional_fields,
simple_int_sums=_SIMPLE)
v.VisitModule(schema_ast)
if abbrev_mod:
f.write("""\
#
# CONCATENATED FILE
#
""")
package, module = abbrev_module_name.split('.')
path = os.path.join(package, module + '.py')
with open(path) as in_f:
f.write(in_f.read())
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# TODO: Remove duplication in core/meta.py
ID_TO_KIND = {}
BOOL_ARG_TYPES = {}
TEST_UNARY_LOOKUP = {}
TEST_BINARY_LOOKUP = {}
TEST_OTHER_LOOKUP = {}
ID_SPEC = id_kind_def.IdSpec(ID_TO_KIND, BOOL_ARG_TYPES)
id_kind_def.AddKinds(ID_SPEC)
id_kind_def.AddBoolKinds(ID_SPEC) # must come second
id_kind_def.SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP,
TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP)
ids = ID_SPEC.id_str2int.items()
ids.sort(key=lambda pair: pair[1]) # Sort by ID
if action == 'c':
for name, id_int in ids:
print('#define id__%s %s' % (name, id_int))
elif action == 'cpp':
from asdl import gen_cpp
schema_ast = _CreateModule(ID_SPEC, ids)
out_prefix = argv[2]
with open(out_prefix + '.h', 'w') as f:
f.write("""\
#ifndef ID_KIND_ASDL_H
#define ID_KIND_ASDL_H
namespace id_kind_asdl {
""")
v = gen_cpp.ClassDefVisitor(f,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write("""
} // namespace id_kind_asdl
#endif // ID_KIND_ASDL_H
""")
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
#include <assert.h>
#include "id_kind_asdl.h"
namespace id_kind_asdl {
""")
v = gen_cpp.MethodDefVisitor(f,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write('} // namespace id_kind_asdl\n')
elif action == 'mypy':
from asdl import gen_python
schema_ast = _CreateModule(ID_SPEC, ids)
#print(schema_ast)
f = sys.stdout
f.write("""\
from asdl import pybase
""")
# Minor style issue: we want Id and Kind, not Id_e and Kind_e
v = gen_python.GenMyPyVisitor(f,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
elif action == 'cpp-consts':
from frontend import consts
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
LIST_INT = [
'STRICT_ALL',
'OIL_BASIC',
'OIL_ALL',
'DEFAULT_TRUE',
'PARSE_OPTION_NUMS',
'SHOPT_OPTION_NUMS',
'SET_OPTION_NUMS',
]
# TODO: These could be changed to numbers
LIST_STR = ['VISIBLE_SHOPT_NAMES']
prefix = argv[2]
with open(prefix + '.h', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#ifndef CONSTS_H
#define CONSTS_H
#include "mylib.h"
#include "id_kind_asdl.h"
#include "option_asdl.h"
#include "runtime_asdl.h"
#include "types_asdl.h"
namespace consts {
""")
for name in LIST_INT:
out('extern List<int>* %s;', name)
for name in LIST_STR:
out('extern List<Str*>* %s;', name)
out("""\
extern int NO_INDEX;
int RedirDefaultFd(id_kind_asdl::Id_t id);
types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id);
types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id);
id_kind_asdl::Kind GetKind(id_kind_asdl::Id_t id);
option_asdl::builtin_t LookupNormalBuiltin(Str* s);
option_asdl::builtin_t LookupAssignBuiltin(Str* s);
option_asdl::builtin_t LookupSpecialBuiltin(Str* s);
bool IsControlFlow(Str* s);
bool IsKeyword(Str* s);
Str* LookupCharC(Str* c);
Str* LookupCharPrompt(Str* c);
Str* OptionName(option_asdl::option_t opt_num);
Tuple2<runtime_asdl::state_t, runtime_asdl::emit_t> IfsEdge(runtime_asdl::state_t state, runtime_asdl::char_kind_t ch);
} // namespace consts
#endif // CONSTS_H
""")
with open(prefix + '.cc', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#include "consts.h"
namespace Id = id_kind_asdl::Id;
using id_kind_asdl::Kind;
using types_asdl::redir_arg_type_e;
using types_asdl::bool_arg_type_e;
using option_asdl::builtin_t;
namespace consts {
int NO_INDEX = 0; // duplicated from frontend/consts.py
""")
# Note: could use opt_num:: instead of raw ints
for name in LIST_INT:
val = getattr(consts, name)
val_str = ', '.join(str(i) for i in val)
out('List<int>* %s = new List<int>({%s});', name, val_str)
for name in LIST_STR:
val = getattr(consts, name)
val_str = '/* TODO */'
out('List<Str*>* %s = new List<Str*>({%s});', name, val_str)
out("""\
int RedirDefaultFd(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(consts.REDIR_DEFAULT_FD):
a = Id_str(id_).replace('.', '::')
b = consts.REDIR_DEFAULT_FD[id_]
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(consts.REDIR_ARG_TYPES):
a = Id_str(id_).replace('.', '::')
# redir_arg_type_e::Path, etc.
b = redir_arg_type_str(consts.REDIR_ARG_TYPES[id_]).replace(
'.', '_e::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(BOOL_ARG_TYPES):
a = Id_str(id_).replace('.', '::')
# bool_arg_type_e::Str, etc.
b = bool_arg_type_str(BOOL_ARG_TYPES[id_]).replace('.', '_e::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
Kind GetKind(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(ID_TO_KIND):
a = Id_str(id_).replace('.', '::')
b = Kind_str(ID_TO_KIND[id_]).replace('.', '::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
b = builtin_def.BuiltinDict()
GenBuiltinLookup(b, 'LookupNormalBuiltin', 'normal', f)
GenBuiltinLookup(b, 'LookupAssignBuiltin', 'assign', f)
GenBuiltinLookup(b, 'LookupSpecialBuiltin', 'special', f)
# TODO: Fill these in
out("""\
bool IsControlFlow(Str* s) {
assert(0);
}
bool IsKeyword(Str* s) {
assert(0);
}
""")
GenCharLookup('LookupCharC', consts._ONE_CHAR_C, f, required=True)
GenCharLookup('LookupCharPrompt', consts._ONE_CHAR_PROMPT, f)
# OptionName() is a bit redundant with ADSL's option_str(), but we can
# remove that.
out("""\
Str* OptionName(option_asdl::option_t opt_num) {
const char* s;
switch (opt_num) {
""")
# These are the only ones we use
set_opts = [(opt.index, opt.name) for opt in option_def.All()
if opt.builtin == 'set']
for index, name in set_opts:
out(' case %s:' % index)
out(' s = "%s";' % name)
out(' break;')
out("""\
default:
assert(0);
}
return new Str(s); // TODO-intern
}
""")
#
# Generate a tightly packed 2D array for C, from a Python dict.
#
edges = consts._IFS_EDGES
max_state = max(edge[0] for edge in edges)
max_char_kind = max(edge[1] for edge in edges)
edge_array = []
for i in xrange(max_state + 1):
# unused cells get -1
edge_array.append(['-1'] * (max_char_kind + 1))
for i in xrange(max_state + 1):
for j in xrange(max_char_kind + 1):
entry = edges.get((i, j))
if entry is not None:
# pack (new_state, action) into 32 bits
edge_array[i][j] = '(%d<<16)|%d' % entry
parts = []
for i in xrange(max_state + 1):
parts.append(' {')
parts.append(', '.join('%10s' % cell
for cell in edge_array[i]))
parts.append(' },\n')
out("""\
int _IFS_EDGE[%d][%d] = {
%s
};
""" % (max_state + 1, max_char_kind + 1, ''.join(parts)))
out("""\
// Note: all of these are integers, e.g. state_i, emit_i, char_kind_i
using runtime_asdl::state_t;
using runtime_asdl::emit_t;
using runtime_asdl::char_kind_t;
Tuple2<state_t, emit_t> IfsEdge(state_t state, runtime_asdl::char_kind_t ch) {
int cell = _IFS_EDGE[state][ch];
state_t new_state = cell >> 16;
emit_t emit = cell & 0xFFFF;
return Tuple2<state_t, emit_t>(new_state, emit);
}
""")
out("""\
} // namespace consts
""")
elif action == 'py-consts':
# It's kind of weird to use the generated code to generate more code.
# Can we do this instead with the parsed module for "id" and "types.asdl"?
from frontend import consts
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
print("""
from _devbuild.gen.id_kind_asdl import Id, Kind
from _devbuild.gen.types_asdl import redir_arg_type_e, bool_arg_type_e
""")
if 0:
print('')
print('REDIR_DEFAULT_FD = {')
for id_ in sorted(consts.REDIR_DEFAULT_FD):
v = consts.REDIR_DEFAULT_FD[id_]
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('REDIR_ARG_TYPES = {')
for id_ in sorted(consts.REDIR_ARG_TYPES):
v = consts.REDIR_ARG_TYPES[id_]
# HACK
v = redir_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('BOOL_ARG_TYPES = {')
for id_ in sorted(BOOL_ARG_TYPES):
v = BOOL_ARG_TYPES[id_]
# HACK
v = bool_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('TEST_UNARY_LOOKUP = {')
for op_str in sorted(TEST_UNARY_LOOKUP):
v = Id_str(TEST_UNARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_BINARY_LOOKUP = {')
for op_str in sorted(TEST_BINARY_LOOKUP):
v = Id_str(TEST_BINARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_OTHER_LOOKUP = {')
for op_str in sorted(TEST_OTHER_LOOKUP):
v = Id_str(TEST_OTHER_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('ID_TO_KIND = {')
for id_ in sorted(ID_TO_KIND):
v = Kind_str(ID_TO_KIND[id_])
print(' %s: %s,' % (Id_str(id_), v))
print('}')
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
try:
schema_path = argv[2]
except IndexError:
raise RuntimeError('Schema path required')
if os.path.basename(schema_path) == 'types.asdl':
app_types = {}
else:
app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')}
if action == 'c': # Generate C code for the lexer
with open(schema_path) as f:
schema_ast, _ = front_end.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(schema_ast)
elif action == 'mypy': # typed mypy
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
try:
abbrev_module_name = argv[3]
except IndexError:
abbrev_mod = None
else:
# Weird Python rule for importing: fromlist needs to be non-empty.
abbrev_mod = __import__(abbrev_module_name, fromlist=['.'])
f = sys.stdout
for typ in app_types.itervalues():
if isinstance(typ, meta.UserType):
f.write('from _devbuild.gen.%s import %s\n' %
(typ.mod_name, typ.type_name))
f.write('\n')
f.write("""\
from asdl import const # For const.NO_INTEGER
from asdl import runtime
PrettyLeaf = runtime.PrettyLeaf
PrettyArray = runtime.PrettyArray
PrettyNode = runtime.PrettyNode
Color_TypeName = runtime.Color_TypeName
Color_StringConst = runtime.Color_StringConst
Color_OtherConst = runtime.Color_OtherConst
Color_UserType = runtime.Color_UserType
from typing import Optional, List, Tuple
""")
abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else []
v = gen_python.GenMyPyVisitor(f, type_lookup, abbrev_mod_entries)
v.VisitModule(schema_ast)
if abbrev_mod:
f.write("""\
#
# CONCATENATED FILE
#
""")
package, module = abbrev_module_name.split('.')
path = os.path.join(package, module + '.py')
with open(path) as in_f:
f.write(in_f.read())
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# TODO: Remove duplication in core/meta.py
ID_TO_KIND = {}
BOOL_ARG_TYPES = {}
TEST_UNARY_LOOKUP = {}
TEST_BINARY_LOOKUP = {}
TEST_OTHER_LOOKUP = {}
ID_SPEC = id_kind.IdSpec(ID_TO_KIND, BOOL_ARG_TYPES)
id_kind.AddKinds(ID_SPEC)
id_kind.AddBoolKinds(ID_SPEC) # must come second
id_kind.SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP, TEST_BINARY_LOOKUP,
TEST_OTHER_LOOKUP)
ids = ID_SPEC.id_str2int.items()
ids.sort(key=lambda pair: pair[1]) # Sort by ID
if action == 'c':
for name, id_int in ids:
print('#define id__%s %s' % (name, id_int))
elif action == 'cpp':
from asdl import gen_cpp
schema_ast = _CreateModule(ID_SPEC, ids)
out_prefix = argv[2]
with open(out_prefix + '.h', 'w') as f:
f.write("""
#ifndef ID_KIND_ASDL_H
#define ID_KIND_ASDL_H
namespace id_kind_asdl {
""")
v = gen_cpp.ClassDefVisitor(f, {},
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write("""
} // namespace id_kind_asdl
#endif // ID_KIND_ASDL_H
""")
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
#include <assert.h>
#include "id_kind_asdl.h"
namespace id_kind_asdl {
""")
v = gen_cpp.MethodDefVisitor(f, {},
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write('} // namespace id_kind_asdl\n')
elif action == 'mypy':
from asdl import gen_python
schema_ast = _CreateModule(ID_SPEC, ids)
#print(schema_ast)
f = sys.stdout
f.write("""\
from asdl import pybase
from typing import List
""")
# Minor style issue: we want Id and Kind, not Id_e and Kind_e
v = gen_python.GenMyPyVisitor(f,
None,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
elif action == 'cc-tables':
from frontend.lookup import REDIR_DEFAULT_FD, REDIR_ARG_TYPES
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
prefix = argv[2]
with open(prefix + '.h', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#ifndef LOOKUP_H
#define LOOKUP_H
#include "id_kind_asdl.h"
namespace lookup {
id_kind_asdl::Kind LookupKind(id_kind_asdl::Id_t id);
} // namespace lookup
#endif // LOOKUP_H
""")
with open(prefix + '.cc', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#include "lookup.h"
namespace Id = id_kind_asdl::Id;
using id_kind_asdl::Kind;
namespace lookup {
""")
out('Kind LookupKind(id_kind_asdl::Id_t id) {')
out(' // relies on "switch lowering"')
out(' switch (id) {')
for id_ in sorted(ID_TO_KIND):
a = Id_str(id_).replace('.', '::')
b = Kind_str(ID_TO_KIND[id_]).replace('.', '::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
} // namespace lookup
""")
elif action == 'py-tables':
# It's kind of weird to use the generated code to generate more code.
# Can we do this instead with the parsed module for "id" and "types.asdl"?
from frontend.lookup import REDIR_DEFAULT_FD, REDIR_ARG_TYPES
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
print("""
from _devbuild.gen.id_kind_asdl import Id, Kind
from _devbuild.gen.types_asdl import redir_arg_type_e, bool_arg_type_e
""")
print('')
print('REDIR_DEFAULT_FD = {')
for id_ in sorted(REDIR_DEFAULT_FD):
v = REDIR_DEFAULT_FD[id_]
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('REDIR_ARG_TYPES = {')
for id_ in sorted(REDIR_ARG_TYPES):
v = REDIR_ARG_TYPES[id_]
# HACK
v = redir_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('BOOL_ARG_TYPES = {')
for id_ in sorted(BOOL_ARG_TYPES):
v = BOOL_ARG_TYPES[id_]
# HACK
v = bool_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('TEST_UNARY_LOOKUP = {')
for op_str in sorted(TEST_UNARY_LOOKUP):
v = Id_str(TEST_UNARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_BINARY_LOOKUP = {')
for op_str in sorted(TEST_BINARY_LOOKUP):
v = Id_str(TEST_BINARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_OTHER_LOOKUP = {')
for op_str in sorted(TEST_OTHER_LOOKUP):
v = Id_str(TEST_OTHER_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('ID_TO_KIND = {')
for id_ in sorted(ID_TO_KIND):
v = Kind_str(ID_TO_KIND[id_])
print(' %s: %s,' % (Id_str(id_), v))
print('}')
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# TODO: Remove duplication in core/meta.py
ID_TO_KIND = {}
BOOL_ARG_TYPES = {}
TEST_UNARY_LOOKUP = {}
TEST_BINARY_LOOKUP = {}
TEST_OTHER_LOOKUP = {}
ID_SPEC = id_kind_def.IdSpec(ID_TO_KIND, BOOL_ARG_TYPES)
id_kind_def.AddKinds(ID_SPEC)
id_kind_def.AddBoolKinds(ID_SPEC) # must come second
id_kind_def.SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP,
TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP)
ids = ID_SPEC.id_str2int.items()
ids.sort(key=lambda pair: pair[1]) # Sort by ID
if action == 'c':
for name, id_int in ids:
print('#define id__%s %s' % (name, id_int))
elif action == 'cpp':
from asdl import gen_cpp
schema_ast = _CreateModule(ID_SPEC, ids)
out_prefix = argv[2]
with open(out_prefix + '.h', 'w') as f:
f.write("""\
#ifndef ID_KIND_ASDL_H
#define ID_KIND_ASDL_H
namespace id_kind_asdl {
""")
v = gen_cpp.ClassDefVisitor(f, {},
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write("""
} // namespace id_kind_asdl
#endif // ID_KIND_ASDL_H
""")
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
#include <assert.h>
#include "id_kind_asdl.h"
namespace id_kind_asdl {
""")
v = gen_cpp.MethodDefVisitor(f, {},
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write('} // namespace id_kind_asdl\n')
elif action == 'mypy':
from asdl import gen_python
schema_ast = _CreateModule(ID_SPEC, ids)
#print(schema_ast)
f = sys.stdout
f.write("""\
from asdl import pybase
""")
# Minor style issue: we want Id and Kind, not Id_e and Kind_e
v = gen_python.GenMyPyVisitor(f,
None,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
elif action == 'cpp-consts':
from frontend import consts
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
LIST_INT = ['STRICT_ALL', 'OIL_BASIC', 'OIL_ALL', 'DEFAULT_TRUE']
# TODO: These could be changed to numbers
LIST_STR = [
'SET_OPTION_NAMES', 'SHOPT_OPTION_NAMES', 'VISIBLE_SHOPT_NAMES',
'PARSE_OPTION_NAMES'
]
prefix = argv[2]
with open(prefix + '.h', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#ifndef LOOKUP_H
#define LOOKUP_H
#include "mylib.h"
#include "id_kind_asdl.h"
#include "option_asdl.h"
#include "runtime_asdl.h"
#include "types_asdl.h"
namespace consts {
""")
for name in LIST_INT:
out('extern List<int>* %s;', name)
for name in LIST_STR:
out('extern List<Str*>* %s;', name)
out("""\
extern int NO_INDEX;
int RedirDefaultFd(id_kind_asdl::Id_t id);
types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id);
types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id);
id_kind_asdl::Kind GetKind(id_kind_asdl::Id_t id);
option_asdl::builtin_t LookupNormalBuiltin(Str* s);
option_asdl::builtin_t LookupAssignBuiltin(Str* s);
option_asdl::builtin_t LookupSpecialBuiltin(Str* s);
Tuple2<runtime_asdl::state_t, runtime_asdl::emit_t> IfsEdge(runtime_asdl::state_t state, runtime_asdl::char_kind_t ch);
} // namespace consts
#endif // LOOKUP_H
""")
with open(prefix + '.cc', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#include "consts.h"
namespace Id = id_kind_asdl::Id;
using id_kind_asdl::Kind;
using types_asdl::redir_arg_type_e;
using types_asdl::bool_arg_type_e;
using option_asdl::builtin_t;
int NO_INDEX = 0; // duplicated from frontend/consts.py
namespace consts {
""")
# Note: could use opt_num:: instead of raw ints
for name in LIST_INT:
val = getattr(consts, name)
val_str = ', '.join(str(i) for i in val)
out('List<int>* %s = new List<int>({%s});', name, val_str)
for name in LIST_STR:
val = getattr(consts, name)
val_str = '/* TODO */'
out('List<Str*>* %s = new List<Str*>({%s});', name, val_str)
out("""\
int RedirDefaultFd(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(consts.REDIR_DEFAULT_FD):
a = Id_str(id_).replace('.', '::')
b = consts.REDIR_DEFAULT_FD[id_]
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(consts.REDIR_ARG_TYPES):
a = Id_str(id_).replace('.', '::')
# redir_arg_type_e::Path, etc.
b = redir_arg_type_str(consts.REDIR_ARG_TYPES[id_]).replace(
'.', '_e::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(BOOL_ARG_TYPES):
a = Id_str(id_).replace('.', '::')
# bool_arg_type_e::Str, etc.
b = bool_arg_type_str(BOOL_ARG_TYPES[id_]).replace('.', '_e::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
Kind GetKind(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(ID_TO_KIND):
a = Id_str(id_).replace('.', '::')
b = Kind_str(ID_TO_KIND[id_]).replace('.', '::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
builtin_t LookupNormalBuiltin(Str* s) {
assert(0);
}
builtin_t LookupAssignBuiltin(Str* s) {
assert(0);
}
builtin_t LookupSpecialBuiltin(Str* s) {
assert(0);
}
""")
out("""\
} // namespace consts
""")
elif action == 'py-consts':
# It's kind of weird to use the generated code to generate more code.
# Can we do this instead with the parsed module for "id" and "types.asdl"?
from frontend import consts
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
print("""
from _devbuild.gen.id_kind_asdl import Id, Kind
from _devbuild.gen.types_asdl import redir_arg_type_e, bool_arg_type_e
""")
if 0:
print('')
print('REDIR_DEFAULT_FD = {')
for id_ in sorted(consts.REDIR_DEFAULT_FD):
v = consts.REDIR_DEFAULT_FD[id_]
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('REDIR_ARG_TYPES = {')
for id_ in sorted(consts.REDIR_ARG_TYPES):
v = consts.REDIR_ARG_TYPES[id_]
# HACK
v = redir_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('BOOL_ARG_TYPES = {')
for id_ in sorted(BOOL_ARG_TYPES):
v = BOOL_ARG_TYPES[id_]
# HACK
v = bool_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('TEST_UNARY_LOOKUP = {')
for op_str in sorted(TEST_UNARY_LOOKUP):
v = Id_str(TEST_UNARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_BINARY_LOOKUP = {')
for op_str in sorted(TEST_BINARY_LOOKUP):
v = Id_str(TEST_BINARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_OTHER_LOOKUP = {')
for op_str in sorted(TEST_OTHER_LOOKUP):
v = Id_str(TEST_OTHER_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('ID_TO_KIND = {')
for id_ in sorted(ID_TO_KIND):
v = Kind_str(ID_TO_KIND[id_])
print(' %s: %s,' % (Id_str(id_), v))
print('}')
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
try:
schema_path = argv[2]
except IndexError:
raise RuntimeError('Schema path required')
if os.path.basename(schema_path) in ('syntax.asdl', 'runtime.asdl'):
app_types = {'id': meta.UserType('id_kind_asdl', 'Id_t')}
else:
app_types = {}
if action == 'c': # Generate C code for the lexer
with open(schema_path) as f:
schema_ast, _ = front_end.LoadSchema(f, app_types)
v = gen_cpp.CEnumVisitor(sys.stdout)
v.VisitModule(schema_ast)
elif action == 'cpp': # Generate C++ code for ASDL schemas
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
# asdl/typed_arith.asdl -> typed_arith_asdl
ns = os.path.basename(schema_path).replace('.', '_')
f = sys.stdout
f.write("""\
#include <cstdint>
#include "runtime.h" // for Str, List, etc.
namespace %s {
""" % ns)
v = gen_cpp.ForwardDeclareVisitor(f)
v.VisitModule(schema_ast)
v2 = gen_cpp.ClassDefVisitor(f, type_lookup)
v2.VisitModule(schema_ast)
f.write('} // namespace %s\n' % ns)
elif action == 'mypy': # Generated typed MyPy code
with open(schema_path) as f:
schema_ast, type_lookup = front_end.LoadSchema(f, app_types)
try:
abbrev_module_name = argv[3]
except IndexError:
abbrev_mod = None
else:
# Weird Python rule for importing: fromlist needs to be non-empty.
abbrev_mod = __import__(abbrev_module_name, fromlist=['.'])
f = sys.stdout
for typ in app_types.itervalues():
if isinstance(typ, meta.UserType):
f.write('from _devbuild.gen.%s import %s\n' %
(typ.mod_name, typ.type_name))
f.write('\n')
# NOTE: Dict, Any are for AssocArray with 'dict' type.
f.write("""\
from asdl import const # For const.NO_INTEGER
from asdl import runtime
from asdl.runtime import (
PrettyLeaf, PrettyArray, PrettyNode,
Color_TypeName, Color_StringConst, Color_OtherConst, Color_UserType,
)
from typing import Optional, List, Tuple, Dict, Any
""")
abbrev_mod_entries = dir(abbrev_mod) if abbrev_mod else []
v = gen_python.GenMyPyVisitor(f, type_lookup, abbrev_mod_entries)
v.VisitModule(schema_ast)
if abbrev_mod:
f.write("""\
#
# CONCATENATED FILE
#
""")
package, module = abbrev_module_name.split('.')
path = os.path.join(package, module + '.py')
with open(path) as in_f:
f.write(in_f.read())
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# NOTE: This initialization must be identical to the one in core/meta.py. We
# do it here to avoid circular dependencies.
ID_SPEC = id_kind.IdSpec({}, {})
id_kind.AddKinds(ID_SPEC)
id_kind.AddBoolKinds(ID_SPEC) # must come second
id_kind.SetupTestBuiltin(ID_SPEC, {}, {}, {})
ids = ID_SPEC.id_str2int.items()
ids.sort(key=lambda pair: pair[1]) # Sort by ID
if action == 'c':
for name, id_int in ids:
print('#define id__%s %s' % (name, id_int))
elif action == 'cpp':
from asdl import gen_cpp
schema_ast = _CreateModule(ID_SPEC, ids)
out_prefix = argv[2]
with open(out_prefix + '.h', 'w') as f:
f.write("""
#ifndef ID_KIND_ASDL_H
#define ID_KIND_ASDL_H
namespace id_kind_asdl {
""")
v = gen_cpp.ClassDefVisitor(f, {}, e_suffix=False)
v.VisitModule(schema_ast)
f.write("""
} // namespace id_kind_asdl
#endif // ID_KIND_ASDL_H
""")
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
#include <assert.h>
#include "id_kind_asdl.h"
namespace id_kind_asdl {
""")
v = gen_cpp.MethodDefVisitor(f, {}, e_suffix=False)
v.VisitModule(schema_ast)
f.write('} // namespace id_kind_asdl\n')
elif action == 'mypy':
from asdl import gen_python
schema_ast = _CreateModule(ID_SPEC, ids)
#print(schema_ast)
f = sys.stdout
f.write("""\
from asdl import pybase
from typing import List
""")
# Minor style issue: we want Id and Kind, not Id_e and Kind_e
v = gen_python.GenMyPyVisitor(f, None, e_suffix=False)
v.VisitModule(schema_ast)
f.write("""
ID_INSTANCES = [
None, # unused index 0
""")
for name, _ in ids:
f.write(' Id.%s,\n' % name)
f.write('] # type: List[Id_t]\n')
f.write("""
KIND_INSTANCES = [
None, # unused index 0
""")
for name in ID_SPEC.kind_name_list:
f.write(' Kind.%s,\n' % name)
f.write('] # type: List[Kind_t]\n')
else:
raise RuntimeError('Invalid action %r' % action)
