If this rule isn't followed, we would read unitialized memory past the
sentinel. Python's regex engine knows where the end of the input string is, so
it doesn't require need a sentinel like \0.
"""
from _devbuild.gen.id_kind_asdl import Id, Id_t, Kind
from _devbuild.gen.types_asdl import lex_mode_e
from frontend import id_kind_def
from typing import Tuple
# Initialize spec that the lexer depends on.
# NOTE: This is duplicated in frontend/id_kind_gen.py.
ID_SPEC = id_kind_def.IdSpec({}, {})
id_kind_def.AddKinds(ID_SPEC)
id_kind_def.AddBoolKinds(ID_SPEC) # must come second
id_kind_def.SetupTestBuiltin(ID_SPEC, {}, {}, {})
def C(pat, tok_type):
# type: (str, Id_t) -> Tuple[bool, str, Id_t]
""" Lexer rule with a constant string, e.g. C('$*', VSub_Star) """
return (False, pat, tok_type)
def R(pat, tok_type):
# type: (str, Id_t) -> Tuple[bool, str, Id_t]
""" Lexer rule with a regex string, e.g. R('\$[0-9]', VSub_Number) """
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# TODO: Remove duplication in core/meta.py
ID_TO_KIND = {}
BOOL_ARG_TYPES = {}
TEST_UNARY_LOOKUP = {}
TEST_BINARY_LOOKUP = {}
TEST_OTHER_LOOKUP = {}
ID_SPEC = id_kind_def.IdSpec(ID_TO_KIND, BOOL_ARG_TYPES)
id_kind_def.AddKinds(ID_SPEC)
id_kind_def.AddBoolKinds(ID_SPEC) # must come second
id_kind_def.SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP,
TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP)
ids = ID_SPEC.id_str2int.items()
ids.sort(key=lambda pair: pair[1]) # Sort by ID
if action == 'c':
for name, id_int in ids:
print('#define id__%s %s' % (name, id_int))
elif action == 'cpp':
from asdl import gen_cpp
schema_ast = _CreateModule(ID_SPEC, ids)
out_prefix = argv[2]
with open(out_prefix + '.h', 'w') as f:
f.write("""\
#ifndef ID_KIND_ASDL_H
#define ID_KIND_ASDL_H
namespace id_kind_asdl {
""")
v = gen_cpp.ClassDefVisitor(f,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write("""
} // namespace id_kind_asdl
#endif // ID_KIND_ASDL_H
""")
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
#include <assert.h>
#include "id_kind_asdl.h"
namespace id_kind_asdl {
""")
v = gen_cpp.MethodDefVisitor(f,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write('} // namespace id_kind_asdl\n')
elif action == 'mypy':
from asdl import gen_python
schema_ast = _CreateModule(ID_SPEC, ids)
#print(schema_ast)
f = sys.stdout
f.write("""\
from asdl import pybase
""")
# Minor style issue: we want Id and Kind, not Id_e and Kind_e
v = gen_python.GenMyPyVisitor(f,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
elif action == 'cpp-consts':
from frontend import consts
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
LIST_INT = [
'STRICT_ALL',
'OIL_BASIC',
'OIL_ALL',
'DEFAULT_TRUE',
'PARSE_OPTION_NUMS',
'SHOPT_OPTION_NUMS',
'SET_OPTION_NUMS',
]
# TODO: These could be changed to numbers
LIST_STR = ['VISIBLE_SHOPT_NAMES']
prefix = argv[2]
with open(prefix + '.h', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#ifndef CONSTS_H
#define CONSTS_H
#include "mylib.h"
#include "id_kind_asdl.h"
#include "option_asdl.h"
#include "runtime_asdl.h"
#include "types_asdl.h"
namespace consts {
""")
for name in LIST_INT:
out('extern List<int>* %s;', name)
for name in LIST_STR:
out('extern List<Str*>* %s;', name)
out("""\
extern int NO_INDEX;
int RedirDefaultFd(id_kind_asdl::Id_t id);
types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id);
types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id);
id_kind_asdl::Kind GetKind(id_kind_asdl::Id_t id);
option_asdl::builtin_t LookupNormalBuiltin(Str* s);
option_asdl::builtin_t LookupAssignBuiltin(Str* s);
option_asdl::builtin_t LookupSpecialBuiltin(Str* s);
bool IsControlFlow(Str* s);
bool IsKeyword(Str* s);
Str* LookupCharC(Str* c);
Str* LookupCharPrompt(Str* c);
Str* OptionName(option_asdl::option_t opt_num);
Tuple2<runtime_asdl::state_t, runtime_asdl::emit_t> IfsEdge(runtime_asdl::state_t state, runtime_asdl::char_kind_t ch);
} // namespace consts
#endif // CONSTS_H
""")
with open(prefix + '.cc', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#include "consts.h"
namespace Id = id_kind_asdl::Id;
using id_kind_asdl::Kind;
using types_asdl::redir_arg_type_e;
using types_asdl::bool_arg_type_e;
using option_asdl::builtin_t;
namespace consts {
int NO_INDEX = 0; // duplicated from frontend/consts.py
""")
# Note: could use opt_num:: instead of raw ints
for name in LIST_INT:
val = getattr(consts, name)
val_str = ', '.join(str(i) for i in val)
out('List<int>* %s = new List<int>({%s});', name, val_str)
for name in LIST_STR:
val = getattr(consts, name)
val_str = '/* TODO */'
out('List<Str*>* %s = new List<Str*>({%s});', name, val_str)
out("""\
int RedirDefaultFd(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(consts.REDIR_DEFAULT_FD):
a = Id_str(id_).replace('.', '::')
b = consts.REDIR_DEFAULT_FD[id_]
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(consts.REDIR_ARG_TYPES):
a = Id_str(id_).replace('.', '::')
# redir_arg_type_e::Path, etc.
b = redir_arg_type_str(consts.REDIR_ARG_TYPES[id_]).replace(
'.', '_e::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(BOOL_ARG_TYPES):
a = Id_str(id_).replace('.', '::')
# bool_arg_type_e::Str, etc.
b = bool_arg_type_str(BOOL_ARG_TYPES[id_]).replace('.', '_e::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
Kind GetKind(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(ID_TO_KIND):
a = Id_str(id_).replace('.', '::')
b = Kind_str(ID_TO_KIND[id_]).replace('.', '::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
b = builtin_def.BuiltinDict()
GenBuiltinLookup(b, 'LookupNormalBuiltin', 'normal', f)
GenBuiltinLookup(b, 'LookupAssignBuiltin', 'assign', f)
GenBuiltinLookup(b, 'LookupSpecialBuiltin', 'special', f)
# TODO: Fill these in
out("""\
bool IsControlFlow(Str* s) {
assert(0);
}
bool IsKeyword(Str* s) {
assert(0);
}
""")
GenCharLookup('LookupCharC', consts._ONE_CHAR_C, f, required=True)
GenCharLookup('LookupCharPrompt', consts._ONE_CHAR_PROMPT, f)
# OptionName() is a bit redundant with ADSL's option_str(), but we can
# remove that.
out("""\
Str* OptionName(option_asdl::option_t opt_num) {
const char* s;
switch (opt_num) {
""")
# These are the only ones we use
set_opts = [(opt.index, opt.name) for opt in option_def.All()
if opt.builtin == 'set']
for index, name in set_opts:
out(' case %s:' % index)
out(' s = "%s";' % name)
out(' break;')
out("""\
default:
assert(0);
}
return new Str(s); // TODO-intern
}
""")
#
# Generate a tightly packed 2D array for C, from a Python dict.
#
edges = consts._IFS_EDGES
max_state = max(edge[0] for edge in edges)
max_char_kind = max(edge[1] for edge in edges)
edge_array = []
for i in xrange(max_state + 1):
# unused cells get -1
edge_array.append(['-1'] * (max_char_kind + 1))
for i in xrange(max_state + 1):
for j in xrange(max_char_kind + 1):
entry = edges.get((i, j))
if entry is not None:
# pack (new_state, action) into 32 bits
edge_array[i][j] = '(%d<<16)|%d' % entry
parts = []
for i in xrange(max_state + 1):
parts.append(' {')
parts.append(', '.join('%10s' % cell
for cell in edge_array[i]))
parts.append(' },\n')
out("""\
int _IFS_EDGE[%d][%d] = {
%s
};
""" % (max_state + 1, max_char_kind + 1, ''.join(parts)))
out("""\
// Note: all of these are integers, e.g. state_i, emit_i, char_kind_i
using runtime_asdl::state_t;
using runtime_asdl::emit_t;
using runtime_asdl::char_kind_t;
Tuple2<state_t, emit_t> IfsEdge(state_t state, runtime_asdl::char_kind_t ch) {
int cell = _IFS_EDGE[state][ch];
state_t new_state = cell >> 16;
emit_t emit = cell & 0xFFFF;
return Tuple2<state_t, emit_t>(new_state, emit);
}
""")
out("""\
} // namespace consts
""")
elif action == 'py-consts':
# It's kind of weird to use the generated code to generate more code.
# Can we do this instead with the parsed module for "id" and "types.asdl"?
from frontend import consts
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
print("""
from _devbuild.gen.id_kind_asdl import Id, Kind
from _devbuild.gen.types_asdl import redir_arg_type_e, bool_arg_type_e
""")
if 0:
print('')
print('REDIR_DEFAULT_FD = {')
for id_ in sorted(consts.REDIR_DEFAULT_FD):
v = consts.REDIR_DEFAULT_FD[id_]
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('REDIR_ARG_TYPES = {')
for id_ in sorted(consts.REDIR_ARG_TYPES):
v = consts.REDIR_ARG_TYPES[id_]
# HACK
v = redir_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('BOOL_ARG_TYPES = {')
for id_ in sorted(BOOL_ARG_TYPES):
v = BOOL_ARG_TYPES[id_]
# HACK
v = bool_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('TEST_UNARY_LOOKUP = {')
for op_str in sorted(TEST_UNARY_LOOKUP):
v = Id_str(TEST_UNARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_BINARY_LOOKUP = {')
for op_str in sorted(TEST_BINARY_LOOKUP):
v = Id_str(TEST_BINARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_OTHER_LOOKUP = {')
for op_str in sorted(TEST_OTHER_LOOKUP):
v = Id_str(TEST_OTHER_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('ID_TO_KIND = {')
for id_ in sorted(ID_TO_KIND):
v = Kind_str(ID_TO_KIND[id_])
print(' %s: %s,' % (Id_str(id_), v))
print('}')
else:
raise RuntimeError('Invalid action %r' % action)
def main(argv):
try:
action = argv[1]
except IndexError:
raise RuntimeError('Action required')
# TODO: Remove duplication in core/meta.py
ID_TO_KIND = {}
BOOL_ARG_TYPES = {}
TEST_UNARY_LOOKUP = {}
TEST_BINARY_LOOKUP = {}
TEST_OTHER_LOOKUP = {}
ID_SPEC = id_kind_def.IdSpec(ID_TO_KIND, BOOL_ARG_TYPES)
id_kind_def.AddKinds(ID_SPEC)
id_kind_def.AddBoolKinds(ID_SPEC) # must come second
id_kind_def.SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP,
TEST_BINARY_LOOKUP, TEST_OTHER_LOOKUP)
ids = ID_SPEC.id_str2int.items()
ids.sort(key=lambda pair: pair[1]) # Sort by ID
if action == 'c':
for name, id_int in ids:
print('#define id__%s %s' % (name, id_int))
elif action == 'cpp':
from asdl import gen_cpp
schema_ast = _CreateModule(ID_SPEC, ids)
out_prefix = argv[2]
with open(out_prefix + '.h', 'w') as f:
f.write("""\
#ifndef ID_KIND_ASDL_H
#define ID_KIND_ASDL_H
namespace id_kind_asdl {
""")
v = gen_cpp.ClassDefVisitor(f, {},
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write("""
} // namespace id_kind_asdl
#endif // ID_KIND_ASDL_H
""")
with open(out_prefix + '.cc', 'w') as f:
f.write("""\
#include <assert.h>
#include "id_kind_asdl.h"
namespace id_kind_asdl {
""")
v = gen_cpp.MethodDefVisitor(f, {},
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
f.write('} // namespace id_kind_asdl\n')
elif action == 'mypy':
from asdl import gen_python
schema_ast = _CreateModule(ID_SPEC, ids)
#print(schema_ast)
f = sys.stdout
f.write("""\
from asdl import pybase
""")
# Minor style issue: we want Id and Kind, not Id_e and Kind_e
v = gen_python.GenMyPyVisitor(f,
None,
e_suffix=False,
simple_int_sums=['Id'])
v.VisitModule(schema_ast)
elif action == 'cpp-consts':
from frontend import consts
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
LIST_INT = ['STRICT_ALL', 'OIL_BASIC', 'OIL_ALL', 'DEFAULT_TRUE']
# TODO: These could be changed to numbers
LIST_STR = [
'SET_OPTION_NAMES', 'SHOPT_OPTION_NAMES', 'VISIBLE_SHOPT_NAMES',
'PARSE_OPTION_NAMES'
]
prefix = argv[2]
with open(prefix + '.h', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#ifndef LOOKUP_H
#define LOOKUP_H
#include "mylib.h"
#include "id_kind_asdl.h"
#include "option_asdl.h"
#include "runtime_asdl.h"
#include "types_asdl.h"
namespace consts {
""")
for name in LIST_INT:
out('extern List<int>* %s;', name)
for name in LIST_STR:
out('extern List<Str*>* %s;', name)
out("""\
extern int NO_INDEX;
int RedirDefaultFd(id_kind_asdl::Id_t id);
types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id);
types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id);
id_kind_asdl::Kind GetKind(id_kind_asdl::Id_t id);
option_asdl::builtin_t LookupNormalBuiltin(Str* s);
option_asdl::builtin_t LookupAssignBuiltin(Str* s);
option_asdl::builtin_t LookupSpecialBuiltin(Str* s);
Tuple2<runtime_asdl::state_t, runtime_asdl::emit_t> IfsEdge(runtime_asdl::state_t state, runtime_asdl::char_kind_t ch);
} // namespace consts
#endif // LOOKUP_H
""")
with open(prefix + '.cc', 'w') as f:
def out(fmt, *args):
print(fmt % args, file=f)
out("""\
#include "consts.h"
namespace Id = id_kind_asdl::Id;
using id_kind_asdl::Kind;
using types_asdl::redir_arg_type_e;
using types_asdl::bool_arg_type_e;
using option_asdl::builtin_t;
int NO_INDEX = 0; // duplicated from frontend/consts.py
namespace consts {
""")
# Note: could use opt_num:: instead of raw ints
for name in LIST_INT:
val = getattr(consts, name)
val_str = ', '.join(str(i) for i in val)
out('List<int>* %s = new List<int>({%s});', name, val_str)
for name in LIST_STR:
val = getattr(consts, name)
val_str = '/* TODO */'
out('List<Str*>* %s = new List<Str*>({%s});', name, val_str)
out("""\
int RedirDefaultFd(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(consts.REDIR_DEFAULT_FD):
a = Id_str(id_).replace('.', '::')
b = consts.REDIR_DEFAULT_FD[id_]
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(consts.REDIR_ARG_TYPES):
a = Id_str(id_).replace('.', '::')
# redir_arg_type_e::Path, etc.
b = redir_arg_type_str(consts.REDIR_ARG_TYPES[id_]).replace(
'.', '_e::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(BOOL_ARG_TYPES):
a = Id_str(id_).replace('.', '::')
# bool_arg_type_e::Str, etc.
b = bool_arg_type_str(BOOL_ARG_TYPES[id_]).replace('.', '_e::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
Kind GetKind(id_kind_asdl::Id_t id) {
// relies on "switch lowering"
switch (id) {
""")
for id_ in sorted(ID_TO_KIND):
a = Id_str(id_).replace('.', '::')
b = Kind_str(ID_TO_KIND[id_]).replace('.', '::')
out(' case %s: return %s;' % (a, b))
out("""\
}
}
""")
out("""\
builtin_t LookupNormalBuiltin(Str* s) {
assert(0);
}
builtin_t LookupAssignBuiltin(Str* s) {
assert(0);
}
builtin_t LookupSpecialBuiltin(Str* s) {
assert(0);
}
""")
out("""\
} // namespace consts
""")
elif action == 'py-consts':
# It's kind of weird to use the generated code to generate more code.
# Can we do this instead with the parsed module for "id" and "types.asdl"?
from frontend import consts
from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
print("""
from _devbuild.gen.id_kind_asdl import Id, Kind
from _devbuild.gen.types_asdl import redir_arg_type_e, bool_arg_type_e
""")
if 0:
print('')
print('REDIR_DEFAULT_FD = {')
for id_ in sorted(consts.REDIR_DEFAULT_FD):
v = consts.REDIR_DEFAULT_FD[id_]
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('REDIR_ARG_TYPES = {')
for id_ in sorted(consts.REDIR_ARG_TYPES):
v = consts.REDIR_ARG_TYPES[id_]
# HACK
v = redir_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('BOOL_ARG_TYPES = {')
for id_ in sorted(BOOL_ARG_TYPES):
v = BOOL_ARG_TYPES[id_]
# HACK
v = bool_arg_type_str(v).replace('.', '_e.')
print(' %s: %s,' % (Id_str(id_), v))
print('}')
print('')
print('TEST_UNARY_LOOKUP = {')
for op_str in sorted(TEST_UNARY_LOOKUP):
v = Id_str(TEST_UNARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_BINARY_LOOKUP = {')
for op_str in sorted(TEST_BINARY_LOOKUP):
v = Id_str(TEST_BINARY_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('TEST_OTHER_LOOKUP = {')
for op_str in sorted(TEST_OTHER_LOOKUP):
v = Id_str(TEST_OTHER_LOOKUP[op_str])
print(' %r: %s,' % (op_str, v))
print('}')
print('')
print('ID_TO_KIND = {')
for id_ in sorted(ID_TO_KIND):
v = Kind_str(ID_TO_KIND[id_])
print(' %s: %s,' % (Id_str(id_), v))
print('}')
else:
raise RuntimeError('Invalid action %r' % action)